Services for scraping data from TCGdex
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace OakArchive.Services;
|
||||
|
||||
public interface IMemoryCacheService
|
||||
{
|
||||
Task<T?> GetOrCreateAsync<T>(string cacheKey, Func<Task<T?>> createItem, TimeSpan? expiration = null)
|
||||
where T : class;
|
||||
|
||||
void Remove(string cacheKey);
|
||||
}
|
||||
|
||||
public class MemoryCacheService : IMemoryCacheService
|
||||
{
|
||||
private readonly IMemoryCache _memoryCache;
|
||||
private readonly TimeSpan _defaultExpiration = TimeSpan.FromHours(1);
|
||||
|
||||
public MemoryCacheService(IMemoryCache memoryCache)
|
||||
{
|
||||
_memoryCache = memoryCache;
|
||||
}
|
||||
|
||||
public async Task<T?> GetOrCreateAsync<T>(string cacheKey, Func<Task<T?>> createItem, TimeSpan? expiration = null)
|
||||
where T : class
|
||||
{
|
||||
if (_memoryCache.TryGetValue(cacheKey, out T? cachedItem))
|
||||
{
|
||||
return cachedItem;
|
||||
}
|
||||
|
||||
var item = await createItem();
|
||||
|
||||
if (item == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var cacheOptions = new MemoryCacheEntryOptions()
|
||||
.SetAbsoluteExpiration(expiration ?? _defaultExpiration);
|
||||
|
||||
_memoryCache.Set(cacheKey, item, cacheOptions);
|
||||
|
||||
return item!;
|
||||
}
|
||||
|
||||
public void Remove(string cacheKey)
|
||||
{
|
||||
_memoryCache.Remove(cacheKey);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using OakArchive.Database;
|
||||
using OakArchive.Entities.CardMarket;
|
||||
using OakArchive.Entities.Cards;
|
||||
using OakArchive.Entities.Enums;
|
||||
using OakArchive.Entities.TCGdex;
|
||||
using OakArchive.Util;
|
||||
using OakArchive.Util.Extensions;
|
||||
using Card = OakArchive.Entities.Cards.Card;
|
||||
using CardCount = OakArchive.Entities.Set.CardCount;
|
||||
using Legal = OakArchive.Entities.Set.Legal;
|
||||
using Set = OakArchive.Entities.Set.Set;
|
||||
|
||||
namespace OakArchive.Services;
|
||||
|
||||
public interface ITcgDexDataUpdater
|
||||
{
|
||||
public Task<bool> SyncBaseDataFromTcgDex();
|
||||
|
||||
public Task<Card>
|
||||
UpdateCard(string set, int id, string language);
|
||||
}
|
||||
|
||||
public class TcgDexDataUpdater : ITcgDexDataUpdater
|
||||
{
|
||||
private readonly ITcgDexService _tcgDexService;
|
||||
private readonly ApplicationDbContext _dbContext;
|
||||
|
||||
private readonly List<Language> _languages = Enum.GetValues<Language>().ToList();
|
||||
|
||||
public TcgDexDataUpdater(ITcgDexService tcgDexService, ApplicationDbContext dbContext)
|
||||
{
|
||||
_tcgDexService = tcgDexService;
|
||||
_dbContext = dbContext;
|
||||
}
|
||||
|
||||
public async Task<bool> SyncBaseDataFromTcgDex()
|
||||
{
|
||||
var hasChanges = false;
|
||||
int i = 0;
|
||||
foreach (var language in _languages)
|
||||
{
|
||||
var apiSeriesBriefs = await _tcgDexService.GetSeries(language.ToApiCode());
|
||||
|
||||
await Task.Delay(500);
|
||||
|
||||
foreach (var serieBrief in apiSeriesBriefs)
|
||||
{
|
||||
if (i == 1) return true;
|
||||
i += 1;
|
||||
if (string.IsNullOrEmpty(serieBrief.Id)) continue;
|
||||
|
||||
var dbSerie = await EnsureSerieExistsAsync(serieBrief, language);
|
||||
|
||||
var fullApiSerie = await _tcgDexService.GetSetsOfSeries(serieBrief.Id, language.ToApiCode());
|
||||
if (fullApiSerie?.Sets == null) continue;
|
||||
|
||||
await Task.Delay(300);
|
||||
|
||||
foreach (var setBrief in fullApiSerie.Sets)
|
||||
{
|
||||
if (string.IsNullOrEmpty(setBrief.Id)) continue;
|
||||
|
||||
var isSetInDb = dbSerie.Sets.Any(s => s.SetInfo?.SetId == setBrief.Id);
|
||||
if (isSetInDb) continue;
|
||||
|
||||
var fullApiSet = await _tcgDexService.GetSet(setBrief.Id, language.ToApiCode());
|
||||
if (fullApiSet == null) continue;
|
||||
|
||||
var dbSet = await BuildSetEntityWithCardsAsync(setBrief, fullApiSet, language);
|
||||
|
||||
dbSerie.Sets.Add(dbSet);
|
||||
|
||||
await Task.Delay(500);
|
||||
}
|
||||
|
||||
if (await _dbContext.SaveChangesAsync() > 0)
|
||||
{
|
||||
hasChanges = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return hasChanges;
|
||||
}
|
||||
|
||||
public async Task<Card> UpdateCard(string set, int id, string language)
|
||||
{
|
||||
var tcgDexCardId = $"{set.ToLower()}-{id}";
|
||||
if (!Enum.TryParse<Language>(language, true, out var langEnum))
|
||||
{
|
||||
langEnum = Language.En;
|
||||
}
|
||||
|
||||
var dbCard = await _dbContext.Cards
|
||||
.Include(c => c.TcgDexCardInfo)
|
||||
.Include(c => c.Attacks)
|
||||
.Include(c => c.Weaknesses)
|
||||
.FirstOrDefaultAsync(c => c.TcgDexCardInfo.FullId == tcgDexCardId && c.Set.Language == langEnum);
|
||||
|
||||
if (dbCard == null)
|
||||
{
|
||||
throw new KeyNotFoundException(
|
||||
$"Die Basis-Karte mit der TCGdex-ID '{tcgDexCardId}' wurde für die Sprache '{language}' nicht in der Datenbank gefunden. Führe zuerst den Basis-Sync aus.");
|
||||
}
|
||||
|
||||
var apiCard = await _tcgDexService.GetCard(set, id, language);
|
||||
if (apiCard == null)
|
||||
{
|
||||
throw new HttpRequestException(
|
||||
$"Die Karte '{tcgDexCardId}' konnte von der TCGdex-API nicht geladen werden.");
|
||||
}
|
||||
|
||||
dbCard.Name = apiCard.Name ?? dbCard.Name;
|
||||
dbCard.Hp = apiCard.Hp;
|
||||
dbCard.Retreat = apiCard.Retreat;
|
||||
dbCard.Illustrator = apiCard.Illustrator;
|
||||
dbCard.EvolveFrom = apiCard.EvolveFrom;
|
||||
dbCard.Description = apiCard.Description;
|
||||
dbCard.RegulationMark = apiCard.RegulationMark;
|
||||
dbCard.Types = apiCard.Types;
|
||||
|
||||
if (!string.IsNullOrEmpty(apiCard.Rarity) &&
|
||||
Enum.TryParse<Rarity>(apiCard.Rarity.Replace(" ", ""), true, out var parsedRarity))
|
||||
{
|
||||
dbCard.Rarity = parsedRarity;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(apiCard.Category) &&
|
||||
Enum.TryParse<Category>(apiCard.Category, true, out var parsedCategory))
|
||||
{
|
||||
dbCard.Category = parsedCategory;
|
||||
}
|
||||
|
||||
dbCard.Variants = MapApiVariantsToEnumList(apiCard.Variants);
|
||||
|
||||
dbCard.Attacks.Clear();
|
||||
if (apiCard.Attacks != null)
|
||||
{
|
||||
foreach (var apiAttack in apiCard.Attacks)
|
||||
{
|
||||
dbCard.Attacks.Add(new Attack
|
||||
{
|
||||
Name = apiAttack.Name ?? "Unknown",
|
||||
Damage = apiAttack.Damage,
|
||||
Effect = apiAttack.Effect ?? string.Empty,
|
||||
Cost = apiAttack.Cost ?? []
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
dbCard.Weaknesses.Clear();
|
||||
if (apiCard.Weaknesses != null)
|
||||
{
|
||||
foreach (var apiWeakness in apiCard.Weaknesses)
|
||||
{
|
||||
dbCard.Weaknesses.Add(new Weakness
|
||||
{
|
||||
Type = apiWeakness.Type ?? "Unknown",
|
||||
Value = apiWeakness.Value ?? string.Empty
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await _dbContext.SaveChangesAsync();
|
||||
|
||||
return dbCard;
|
||||
}
|
||||
|
||||
private List<Variant> MapApiVariantsToEnumList(Models.TcgDex.Variants? apiVariants)
|
||||
{
|
||||
var list = new List<Variant>();
|
||||
if (apiVariants == null) return list;
|
||||
|
||||
if (apiVariants.FirstEdition == true) list.Add(Variant.FirstEdition);
|
||||
if (apiVariants.Holo == true) list.Add(Variant.Holo);
|
||||
if (apiVariants.Normal == true) list.Add(Variant.Normal);
|
||||
if (apiVariants.Reverse == true) list.Add(Variant.Reverse);
|
||||
if (apiVariants.WPromo == true) list.Add(Variant.WPromo);
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private async Task<Entities.Serie.Serie> EnsureSerieExistsAsync(Models.TcgDex.SerieBrief brief, Language language)
|
||||
{
|
||||
var dbSerie = await _dbContext.Series
|
||||
.Include(s => s.Sets)
|
||||
.ThenInclude(s => s.SetInfo)
|
||||
.FirstOrDefaultAsync(s => s.TcgDexSeriesInfo.SeriesId == brief.Id && s.Language == language);
|
||||
|
||||
if (dbSerie != null) return dbSerie;
|
||||
|
||||
dbSerie = new Entities.Serie.Serie
|
||||
{
|
||||
Language = language,
|
||||
Name = brief.Name ?? "Unknown",
|
||||
Logo = brief.Logo,
|
||||
TcgDexSeriesInfo = new TcgDexSeriesInfo { SeriesId = brief.Id! }
|
||||
};
|
||||
_dbContext.Series.Add(dbSerie);
|
||||
|
||||
return dbSerie;
|
||||
}
|
||||
|
||||
private async Task<Set> BuildSetEntityWithCardsAsync(Models.TcgDex.SetBrief setBrief, Models.TcgDex.Set fullApiSet,
|
||||
Language language)
|
||||
{
|
||||
DateTime? utcReleaseDate = fullApiSet.ReleaseDate != null
|
||||
? DateTime.SpecifyKind(fullApiSet.ReleaseDate, DateTimeKind.Utc)
|
||||
: null;
|
||||
|
||||
var dbSet = new Set
|
||||
{
|
||||
Language = language,
|
||||
Name = setBrief.Name ?? "Unknown",
|
||||
Logo = setBrief.Logo,
|
||||
ReleaseDate = utcReleaseDate,
|
||||
Symbol = setBrief.Symbol,
|
||||
SetInfo = new TcgDexSetInfo { SetId = fullApiSet.Id },
|
||||
CardMarketSetInfo = new CardMarketSetInfo { ExpansionId = "0" }, // TODO: Späterer Price-Scraper
|
||||
Legal = new Legal
|
||||
{
|
||||
Expanded = fullApiSet.Legal?.Expanded ?? false,
|
||||
Standard = fullApiSet.Legal?.Standard ?? false
|
||||
},
|
||||
CardCount = new CardCount
|
||||
{
|
||||
FirstEd = fullApiSet.CardCount?.FirstEd ?? 0,
|
||||
Holo = fullApiSet.CardCount?.Holo ?? 0,
|
||||
Normal = fullApiSet.CardCount?.Normal ?? 0,
|
||||
Official = fullApiSet.CardCount?.Official ?? 0,
|
||||
Reverse = fullApiSet.CardCount?.Reverse ?? 0,
|
||||
Total = fullApiSet.CardCount?.Total ?? 0
|
||||
}
|
||||
};
|
||||
|
||||
if (fullApiSet.Cards is not { Count: > 0 }) return dbSet;
|
||||
|
||||
var cardTasks = fullApiSet.Cards.Select(async c =>
|
||||
{
|
||||
var localPath = await _tcgDexService.DownloadCardImage(c, language.ToApiCode());
|
||||
|
||||
return new Card
|
||||
{
|
||||
Name = c.Name ?? "Unknown",
|
||||
Image = localPath == null ? c.Image : localPath.Item1,
|
||||
CardNumber = c.LocalId ?? 0,
|
||||
PHash = localPath == null ? 0 : PerceptualHashHelper.GenerateHash(localPath.Item2),
|
||||
TcgDexCardInfo = new TcgDexCardInfo
|
||||
{
|
||||
FullId = c.Id ?? string.Empty,
|
||||
LocalId = c.LocalId ?? 0
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
dbSet.Cards = (await Task.WhenAll(cardTasks)).ToList();
|
||||
|
||||
return dbSet;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using OakArchive.Models.TcgDex;
|
||||
|
||||
namespace OakArchive.Services;
|
||||
|
||||
public interface ITcgDexService
|
||||
{
|
||||
public Task<List<SerieBrief>> GetSeries(string language);
|
||||
|
||||
public Task<Serie?> GetSetsOfSeries(string series, string language);
|
||||
|
||||
public Task<Set?> GetSet(string set, string language);
|
||||
|
||||
public Task<Card?> GetCard(string set, int id, string language);
|
||||
|
||||
public Task<Tuple<string, byte[]>?> DownloadCardImage(Card card, string language);
|
||||
}
|
||||
|
||||
public class TcgDexService : ITcgDexService
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly IMemoryCacheService _cacheService;
|
||||
|
||||
private readonly string _baseFolder = Path.Combine(Directory.GetCurrentDirectory(), "Cards");
|
||||
private readonly SemaphoreSlim _downloadSemaphore = new SemaphoreSlim(3, 3);
|
||||
|
||||
public TcgDexService(HttpClient httpClient, IMemoryCacheService cacheService)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
_cacheService = cacheService;
|
||||
}
|
||||
|
||||
public async Task<List<SerieBrief>> GetSeries(string language)
|
||||
{
|
||||
var cacheKey = $"tcgdex_series_{language.ToLower()}";
|
||||
|
||||
async Task<List<SerieBrief>> FetchFromApi()
|
||||
{
|
||||
var res = await _httpClient.GetFromJsonAsync<List<SerieBrief>>($"{language}/series");
|
||||
return res ?? [];
|
||||
}
|
||||
|
||||
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
|
||||
}
|
||||
|
||||
public async Task<Serie?> GetSetsOfSeries(string series, string language)
|
||||
{
|
||||
var cacheKey = $"tcgdex_serie_{series.ToLower()}_{language.ToLower()}";
|
||||
|
||||
async Task<Serie?> FetchFromApi()
|
||||
{
|
||||
var res = await _httpClient.GetFromJsonAsync<Serie>($"{language}/series/{series}");
|
||||
return res;
|
||||
}
|
||||
|
||||
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
|
||||
}
|
||||
|
||||
public async Task<Set?> GetSet(string set, string language)
|
||||
{
|
||||
var cacheKey = $"tcgdex_set_{set.ToLower()}_{language.ToLower()}";
|
||||
|
||||
async Task<Set?> FetchFromApi()
|
||||
{
|
||||
var res = await _httpClient.GetFromJsonAsync<Set>($"{language}/sets/{set}");
|
||||
return res;
|
||||
}
|
||||
|
||||
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
|
||||
}
|
||||
|
||||
public async Task<Card?> GetCard(string set, int id, string language)
|
||||
{
|
||||
var cacheKey = $"tcgdex_card_{set.ToLower()}_{language.ToLower()}";
|
||||
|
||||
async Task<Card?> FetchFromApi()
|
||||
{
|
||||
var res = await _httpClient.GetFromJsonAsync<Card?>($"{language}/cards/{set}-{id}");
|
||||
return res;
|
||||
}
|
||||
|
||||
return await _cacheService.GetOrCreateAsync(cacheKey, FetchFromApi, TimeSpan.FromDays(7));
|
||||
}
|
||||
|
||||
public async Task<Tuple<string, byte[]>?> DownloadCardImage(Card card, string language)
|
||||
{
|
||||
var split = card.Id!.Split("-");
|
||||
var filePath = Path.Combine(_baseFolder, split[0], language,$"{split[1]}.webp");
|
||||
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
var res = await File.ReadAllBytesAsync(filePath);
|
||||
return Tuple.Create(filePath, res);
|
||||
}
|
||||
|
||||
await _downloadSemaphore.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
var res = await _httpClient.GetByteArrayAsync(card.Image + "/high.webp");
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(filePath)!);
|
||||
await File.WriteAllBytesAsync(filePath, res);
|
||||
|
||||
await Task.Delay(200);
|
||||
|
||||
return Tuple.Create(filePath, res);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_downloadSemaphore.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user