Kleidukos/OakArchive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Services for scraping data from TCGdex

Browse Source
This commit is contained in:
Kleidukos
2026-05-23 16:24:51 +02:00
parent f71283d617
commit adbc3a28d6
3 changed files with 424 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OakArchive/Services/MemoryCacheService.cs
+50
View File
@@ -0,0 +1,50 @@
using Microsoft.Extensions.Caching.Memory;
namespace OakArchive.Services;
public interface IMemoryCacheService
{
Task<T?> GetOrCreateAsync<T>(string cacheKey, Func<Task<T?>> createItem, TimeSpan? expiration = null)
where T : class;
void Remove(string cacheKey);
}
public class MemoryCacheService : IMemoryCacheService
{
private readonly IMemoryCache _memoryCache;
private readonly TimeSpan _defaultExpiration = TimeSpan.FromHours(1);
public MemoryCacheService(IMemoryCache memoryCache)
{
_memoryCache = memoryCache;
}
public async Task<T?> GetOrCreateAsync<T>(string cacheKey, Func<Task<T?>> createItem, TimeSpan? expiration = null)
where T : class
{
if (_memoryCache.TryGetValue(cacheKey, out T? cachedItem))
{
return cachedItem;
}
var item = await createItem();
if (item == null)
{
return null;
}
var cacheOptions = new MemoryCacheEntryOptions()
.SetAbsoluteExpiration(expiration ?? _defaultExpiration);
_memoryCache.Set(cacheKey, item, cacheOptions);
return item!;
}
public void Remove(string cacheKey)
{
_memoryCache.Remove(cacheKey);
}
}
OakArchive/Services/TcgDexDataUpdater.cs
+261
View File
@@ -0,0 +1,261 @@
using Microsoft.EntityFrameworkCore;
using OakArchive.Database;
using OakArchive.Entities.CardMarket;
using OakArchive.Entities.Cards;
using OakArchive.Entities.Enums;
using OakArchive.Entities.TCGdex;
using OakArchive.Util;
using OakArchive.Util.Extensions;
using Card = OakArchive.Entities.Cards.Card;
using CardCount = OakArchive.Entities.Set.CardCount;
using Legal = OakArchive.Entities.Set.Legal;
using Set = OakArchive.Entities.Set.Set;
namespace OakArchive.Services;
public interface ITcgDexDataUpdater
{
public Task<bool> SyncBaseDataFromTcgDex();
public Task<Card>
UpdateCard(string set, int id, string language);
}
public class TcgDexDataUpdater : ITcgDexDataUpdater
{
private readonly ITcgDexService _tcgDexService;
private readonly ApplicationDbContext _dbContext;
private readonly List<Language> _languages = Enum.GetValues<Language>().ToList();
public TcgDexDataUpdater(ITcgDexService tcgDexService, ApplicationDbContext dbContext)
{
_tcgDexService = tcgDexService;
_dbContext = dbContext;
}
public async Task<bool> SyncBaseDataFromTcgDex()
{
var hasChanges = false;
int i = 0;
foreach (var language in _languages)
{
var apiSeriesBriefs = await _tcgDexService.GetSeries(language.ToApiCode());
await Task.Delay(500);
foreach (var serieBrief in apiSeriesBriefs)
{
if (i == 1) return true;
i += 1;
if (string.IsNullOrEmpty(serieBrief.Id)) continue;
var dbSerie = await EnsureSerieExistsAsync(serieBrief, language);
var fullApiSerie = await _tcgDexService.GetSetsOfSeries(serieBrief.Id, language.ToApiCode());
if (fullApiSerie?.Sets == null) continue;
await Task.Delay(300);
foreach (var setBrief in fullApiSerie.Sets)
{
if (string.IsNullOrEmpty(setBrief.Id)) continue;
var isSetInDb = dbSerie.Sets.Any(s => s.SetInfo?.SetId == setBrief.Id);
if (isSetInDb) continue;
var fullApiSet = await _tcgDexService.GetSet(setBrief.Id, language.ToApiCode());
if (fullApiSet == null) continue;
var dbSet = await BuildSetEntityWithCardsAsync(setBrief, fullApiSet, language);
dbSerie.Sets.Add(dbSet);
await Task.Delay(500);
}
if (await _dbContext.SaveChangesAsync() > 0)
{
hasChanges = true;
}
}
}
return hasChanges;
}
public async Task<Card> UpdateCard(string set, int id, string language)
{
var tcgDexCardId = $"{set.ToLower()}-{id}";
if (!Enum.TryParse<Language>(language, true, out var langEnum))
{
langEnum = Language.En;
}
var dbCard = await _dbContext.Cards
.Include(c => c.TcgDexCardInfo)
.Include(c => c.Attacks)
.Include(c => c.Weaknesses)
.FirstOrDefaultAsync(c => c.TcgDexCardInfo.FullId == tcgDexCardId && c.Set.Language == langEnum);
if (dbCard == null)
{
throw new KeyNotFoundException(
$"Die Basis-Karte mit der TCGdex-ID '{tcgDexCardId}' wurde für die Sprache '{language}' nicht in der Datenbank gefunden. Führe zuerst den Basis-Sync aus.");
}
var apiCard = await _tcgDexService.GetCard(set, id, language);
if (apiCard == null)
{
throw new HttpRequestException(
$"Die Karte '{tcgDexCardId}' konnte von der TCGdex-API nicht geladen werden.");
}
dbCard.Name = apiCard.Name ?? dbCard.Name;
dbCard.Hp = apiCard.Hp;
dbCard.Retreat = apiCard.Retreat;
dbCard.Illustrator = apiCard.Illustrator;
dbCard.EvolveFrom = apiCard.EvolveFrom;
dbCard.Description = apiCard.Description;
dbCard.RegulationMark = apiCard.RegulationMark;
dbCard.Types = apiCard.Types;
if (!string.IsNullOrEmpty(apiCard.Rarity) &&
Enum.TryParse<Rarity>(apiCard.Rarity.Replace(" ", ""), true, out var parsedRarity))
{
dbCard.Rarity = parsedRarity;
}
if (!string.IsNullOrEmpty(apiCard.Category) &&
Enum.TryParse<Category>(apiCard.Category, true, out var parsedCategory))
{
dbCard.Category = parsedCategory;
}
dbCard.Variants = MapApiVariantsToEnumList(apiCard.Variants);
dbCard.Attacks.Clear();
if (apiCard.Attacks != null)
{
foreach (var apiAttack in apiCard.Attacks)
{
dbCard.Attacks.Add(new Attack
{
Name = apiAttack.Name ?? "Unknown",
Damage = apiAttack.Damage,
Effect = apiAttack.Effect ?? string.Empty,
Cost = apiAttack.Cost ?? []
});
}
}
dbCard.Weaknesses.Clear();
if (apiCard.Weaknesses != null)
{
foreach (var apiWeakness in apiCard.Weaknesses)
{
dbCard.Weaknesses.Add(new Weakness
{
Type = apiWeakness.Type ?? "Unknown",
Value = apiWeakness.Value ?? string.Empty
});
}
}
await _dbContext.SaveChangesAsync();
return dbCard;
}
private List<Variant> MapApiVariantsToEnumList(Models.TcgDex.Variants? apiVariants)
{
var list = new List<Variant>();
if (apiVariants == null) return list;
if (apiVariants.FirstEdition == true) list.Add(Variant.FirstEdition);
if (apiVariants.Holo == true) list.Add(Variant.Holo);
if (apiVariants.Normal == true) list.Add(Variant.Normal);
if (apiVariants.Reverse == true) list.Add(Variant.Reverse);
if (apiVariants.WPromo == true) list.Add(Variant.WPromo);
return list;
}
private async Task<Entities.Serie.Serie> EnsureSerieExistsAsync(Models.TcgDex.SerieBrief brief, Language language)
{
var dbSerie = await _dbContext.Series
.Include(s => s.Sets)
.ThenInclude(s => s.SetInfo)
.FirstOrDefaultAsync(s => s.TcgDexSeriesInfo.SeriesId == brief.Id && s.Language == language);
if (dbSerie != null) return dbSerie;
dbSerie = new Entities.Serie.Serie
{
Language = language,
Name = brief.Name ?? "Unknown",
Logo = brief.Logo,
TcgDexSeriesInfo = new TcgDexSeriesInfo { SeriesId = brief.Id! }
};
_dbContext.Series.Add(dbSerie);
return dbSerie;
}
private async Task<Set> BuildSetEntityWithCardsAsync(Models.TcgDex.SetBrief setBrief, Models.TcgDex.Set fullApiSet,
Language language)
{
DateTime? utcReleaseDate = fullApiSet.ReleaseDate != null
? DateTime.SpecifyKind(fullApiSet.ReleaseDate, DateTimeKind.Utc)
: null;
var dbSet = new Set
{
Language = language,
Name = setBrief.Name ?? "Unknown",
Logo = setBrief.Logo,
ReleaseDate = utcReleaseDate,
Symbol = setBrief.Symbol,
SetInfo = new TcgDexSetInfo { SetId = fullApiSet.Id },
CardMarketSetInfo = new CardMarketSetInfo { ExpansionId = "0" }, // TODO: Späterer Price-Scraper
Legal = new Legal
{
Expanded = fullApiSet.Legal?.Expanded ?? false,
Standard = fullApiSet.Legal?.Standard ?? false
},
CardCount = new CardCount
{
FirstEd = fullApiSet.CardCount?.FirstEd ?? 0,
Holo = fullApiSet.CardCount?.Holo ?? 0,
Normal = fullApiSet.CardCount?.Normal ?? 0,
Official = fullApiSet.CardCount?.Official ?? 0,
Reverse = fullApiSet.CardCount?.Reverse ?? 0,
Total = fullApiSet.CardCount?.Total ?? 0
}
};
if (fullApiSet.Cards is not { Count: > 0 }) return dbSet;
var cardTasks = fullApiSet.Cards.Select(async c =>
{
var localPath = await _tcgDexService.DownloadCardImage(c, language.ToApiCode());
return new Card
{
Name = c.Name ?? "Unknown",
Image = localPath == null ? c.Image : localPath.Item1,
CardNumber = c.LocalId ?? 0,
PHash = localPath == null ? 0 : PerceptualHashHelper.GenerateHash(localPath.Item2),
TcgDexCardInfo = new TcgDexCardInfo
{
FullId = c.Id ?? string.Empty,
LocalId = c.LocalId ?? 0
}
};
});
dbSet.Cards = (await Task.WhenAll(cardTasks)).ToList();
return dbSet;
}
}
OakArchive/Services/TcgDexService.cs
+113
View File
@@ -0,0 +1,113 @@
using OakArchive.Models.TcgDex;
namespace OakArchive.Services;
public interface ITcgDexService
{
public Task<List<SerieBrief>> GetSeries(string language);
public Task<Serie?> GetSetsOfSeries(string series, string language);
public Task<Set?> GetSet(string set, string language);
public Task<Card?> GetCard(string set, int id, string language);
public Task<Tuple<string, byte[]>?> DownloadCardImage(Card card, string language);
}
public class TcgDexService : ITcgDexService
{
private readonly HttpClient _httpClient;
private readonly IMemoryCacheService _cacheService;
private readonly string _baseFolder = Path.Combine(Directory.GetCurrentDirectory(), "Cards");
private readonly SemaphoreSlim _downloadSemaphore = new SemaphoreSlim(3, 3);
public TcgDexService(HttpClient httpClient, IMemoryCacheService cacheService)
{
_httpClient = httpClient;
_cacheService = cacheService;
}
public async Task<List<SerieBrief>> GetSeries(string language)
{
var cacheKey = $"tcgdex_series_{language.ToLower()}";
async Task<List<SerieBrief>> FetchFromApi()
{
var res = await _httpClient.GetFromJsonAsync<List<SerieBrief>>($"{language}/series");
return res ?? [];
}
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
}
public async Task<Serie?> GetSetsOfSeries(string series, string language)
{
var cacheKey = $"tcgdex_serie_{series.ToLower()}_{language.ToLower()}";
async Task<Serie?> FetchFromApi()
{
var res = await _httpClient.GetFromJsonAsync<Serie>($"{language}/series/{series}");
return res;
}
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
}
public async Task<Set?> GetSet(string set, string language)
{
var cacheKey = $"tcgdex_set_{set.ToLower()}_{language.ToLower()}";
async Task<Set?> FetchFromApi()
{
var res = await _httpClient.GetFromJsonAsync<Set>($"{language}/sets/{set}");
return res;
}
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
}
public async Task<Card?> GetCard(string set, int id, string language)
{
var cacheKey = $"tcgdex_card_{set.ToLower()}_{language.ToLower()}";
async Task<Card?> FetchFromApi()
{
var res = await _httpClient.GetFromJsonAsync<Card?>($"{language}/cards/{set}-{id}");
return res;
}
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
}
public async Task<Tuple<string, byte[]>?> DownloadCardImage(Card card, string language)
{
var split = card.Id!.Split("-");
var filePath = Path.Combine(_baseFolder, split[0], language,$"{split[1]}.webp");
if (File.Exists(filePath))
{
var res = await File.ReadAllBytesAsync(filePath);
return Tuple.Create(filePath, res);
}
await _downloadSemaphore.WaitAsync();
try
{
var res = await _httpClient.GetByteArrayAsync(card.Image + "/high.webp");
Directory.CreateDirectory(Path.GetDirectoryName(filePath)!);
await File.WriteAllBytesAsync(filePath, res);
await Task.Delay(200);
return Tuple.Create(filePath, res);
}
finally
{
_downloadSemaphore.Release();
}
}
}