Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings

Browse Source
This commit is contained in:
Kleidukos
2026-08-16 14:05:57 +02:00
parent b0f8d4b78b
commit 2ba54e8057
17 changed files with 1791 additions and 563 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticFundamentals/Services/FundamentalsDbService.cs
+176 -69
View File
@@ -79,44 +79,53 @@ public class FundamentalsDbService : IFundamentalsDbService
await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken);
bool enableHtmlFallback =
await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken);
bool forceHtmlFallback =
await settingsService.GetSettingAsync(SettingKeys.ForceHtmlFallback, cancellationToken);
int validityDays =
await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken);
bool effectiveForceRefresh = forceRefresh && allowForceRefresh;
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtmlFallback: {Html}",
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback);
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtml: {Html} | ForceHtml: {ForceHtml}",
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback, forceHtmlFallback);
// 2. Entitäten aus DB laden
var assetData = await context.AssetData
.Include(a => a.AvailableTickers)
.Include(a => a.KeyExecutives)
.Include(a => a.AssetEvents)
.Include(a => a.FundamentalData)
.FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken);
var fundamentalData = await context.FundamentalData
.FirstOrDefaultAsync(f => f.Isin == cleanIsin, cancellationToken);
string targetTicker = !string.IsNullOrWhiteSpace(requestedTicker)
? requestedTicker
: (assetData?.PrimaryTicker?.Ticker ?? string.Empty);
var fundamentalData = assetData?.FundamentalData?
.FirstOrDefault(f => !string.IsNullOrWhiteSpace(targetTicker) && string.Equals(f.Ticker.Ticker, targetTicker, StringComparison.OrdinalIgnoreCase))
?? (string.IsNullOrWhiteSpace(requestedTicker) ? assetData?.FundamentalData?.FirstOrDefault() : null);
// 3. Prüfen, was aktualisiert werden muss
bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name);
bool executivesMissing = assetData == null || assetData.KeyExecutives == null ||
assetData.KeyExecutives.Count == 0;
bool fundamentalsExpired = fundamentalData == null ||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
bool fundamentalsMissingOrExpired = fundamentalData == null ||
(fundamentalData.MarketCap == null && fundamentalData.TrailingPe == null) ||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
bool tickersCorruptOrMissing = assetData?.AvailableTickers == null ||
assetData.AvailableTickers.Count == 0 ||
assetData.AvailableTickers.Any(t => t.Ticker != null && t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase));
// Wenn ein expliziter Ticker übergeben wurde und sich vom gespeicherten unterscheidet,
// müssen Asset-Daten und Fundamentals mit dem neuen Ticker neu abgerufen werden.
bool tickerChanged = !string.IsNullOrWhiteSpace(requestedTicker)
&& assetData?.PrimaryTicker != null
&& !string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker,
StringComparison.OrdinalIgnoreCase);
bool shouldUpdate = assetDataMissing || executivesMissing || fundamentalsMissingOrExpired || tickersCorruptOrMissing || effectiveForceRefresh || forceHtmlFallback;
bool shouldUpdateAssetData = assetDataMissing || effectiveForceRefresh || tickerChanged;
bool shouldUpdateExecutives = executivesMissing || effectiveForceRefresh;
bool shouldUpdateFundamentals = fundamentalsExpired || effectiveForceRefresh || tickerChanged;
if (shouldUpdateAssetData || shouldUpdateExecutives || shouldUpdateFundamentals)
if (!shouldUpdate && fundamentalData != null)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[FundamentalsDbService] Returning valid cached fundamental data for ISIN {Isin} (Ticker: {Ticker}, Updated: {UpdatedUtc}). External API fetch skipped.",
cleanIsin, fundamentalData.Ticker.Ticker, fundamentalData.LastUpdatedUtc.ToString("o"));
}
else
{
// --- STEP 1: Trade Republic Details ---
TradeRepublicStockDetailsResponse? trDetails = null;
@@ -150,8 +159,10 @@ public class FundamentalsDbService : IFundamentalsDbService
TickerInfoDto activeQueryTicker;
if (!string.IsNullOrWhiteSpace(requestedTicker))
{
var matchDto = resolvedTickers.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchDto = resolvedTickers.FirstOrDefault(t =>
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(t =>
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
if (matchDto != null)
{
@@ -197,12 +208,18 @@ public class FundamentalsDbService : IFundamentalsDbService
yahooPrimaryTicker.Ticker, activeQueryTicker.Ticker, cleanIsin);
// --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper ---
// Profile (Sektor, Industrie, Vorstände) wird gescrapt, wenn weder in DB noch in TR Vorstände/Beschreibungen vorliegen
bool hasProfileInDb = assetData != null && !string.IsNullOrWhiteSpace(assetData.Description) && assetData.KeyExecutives != null && assetData.KeyExecutives.Count > 0;
bool hasCeoInTr = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
bool needProfile = !hasProfileInDb && !hasCeoInTr;
YahooQuoteSummaryModulesDto? modulesDto = null;
if (!string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) && activeQueryTicker.Ticker != cleanIsin)
{
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
activeQueryTicker.Ticker,
forceHtmlScrape: false,
forceHtmlScrape: forceHtmlFallback,
includeProfile: needProfile,
cancellationToken: cancellationToken);
}
else
@@ -211,51 +228,96 @@ public class FundamentalsDbService : IFundamentalsDbService
"[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", activeQueryTicker.Ticker);
}
// --- Update AssetDataEntity ---
if (shouldUpdateAssetData)
// Falls der Sekundär-Ticker (z. B. APC.DE) überhaupt keine Daten liefert, nutze den PrimaryTicker (z. B. AAPL) als Fallback
if (modulesDto == null &&
!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) &&
yahooPrimaryTicker.Ticker != activeQueryTicker.Ticker &&
yahooPrimaryTicker.Ticker != cleanIsin)
{
if (assetData == null)
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-FALLBACK-PRIMARY] Sekundär-Ticker '{Active}' lieferte keine Daten. Versuche PrimaryTicker '{Primary}'...",
activeQueryTicker.Ticker, yahooPrimaryTicker.Ticker);
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
yahooPrimaryTicker.Ticker,
forceHtmlScrape: forceHtmlFallback,
includeProfile: needProfile,
cancellationToken: cancellationToken);
}
// --- Update AssetDataEntity ---
if (assetData == null)
{
assetData = new AssetDataEntity
{
assetData = new AssetDataEntity
Isin = cleanIsin,
PrimaryTicker = new TickerEntity
{
Isin = cleanIsin,
PrimaryTicker = new TickerEntity
{
Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
},
KeyExecutives = new List<KeyExecutiveEntity>(),
AssetEvents = new List<AssetEventEntity>()
};
context.AssetData.Add(assetData);
}
Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
},
KeyExecutives = new List<KeyExecutiveEntity>(),
AssetEvents = new List<AssetEventEntity>()
};
context.AssetData.Add(assetData);
}
string trName = trDetails?.Company?.Name ?? string.Empty;
string trDescription = trDetails?.Company?.Description ?? string.Empty;
string trName = trDetails?.Company?.Name?.Trim() ?? string.Empty;
string trDescription = trDetails?.Company?.Description?.Trim() ?? string.Empty;
string fallbackName = modulesDto?.QuoteType?.ShortName
?? modulesDto?.QuoteType?.LongName
?? activeQueryTicker.Ticker;
string yahooName = modulesDto?.QuoteType?.LongName?.Trim()
?? modulesDto?.QuoteType?.ShortName?.Trim()
?? string.Empty;
string yahooDesc = modulesDto?.AssetProfile?.LongBusinessSummary?.Trim() ?? string.Empty;
assetData.Name = !string.IsNullOrWhiteSpace(trName) ? trName : fallbackName;
assetData.Description = !string.IsNullOrWhiteSpace(trDescription)
? trDescription
: (modulesDto?.AssetProfile?.LongBusinessSummary ?? string.Empty);
// Name nur aktualisieren, wenn ein echter Name vorliegt (Bestandsdaten niemals mit ISIN/Ticker überschreiben)
if (!string.IsNullOrWhiteSpace(trName))
{
assetData.Name = trName;
}
else if (!string.IsNullOrWhiteSpace(yahooName))
{
assetData.Name = yahooName;
}
else if (string.IsNullOrWhiteSpace(assetData.Name))
{
assetData.Name = !string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) ? activeQueryTicker.Ticker : cleanIsin;
}
// PrimaryTicker ist FEST der erste von Yahoo Finance
// Description nur aktualisieren, wenn neue Beschreibung vorhanden ist
if (!string.IsNullOrWhiteSpace(trDescription))
{
assetData.Description = trDescription;
}
else if (!string.IsNullOrWhiteSpace(yahooDesc))
{
assetData.Description = yahooDesc;
}
// PrimaryTicker aktualisieren falls vorhanden
if (!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) && yahooPrimaryTicker.Ticker != cleanIsin)
{
assetData.PrimaryTicker = new TickerEntity
{
Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
};
}
if (!resolvedTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
// AvailableTickers aktualisieren (nur echte Börsenticker, keine ISINs)
var validTickers = resolvedTickers
.Where(t => !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase))
.ToList();
if (validTickers.Count > 0)
{
if (!validTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
{
resolvedTickers.Insert(0, yahooPrimaryTicker);
validTickers.Insert(0, yahooPrimaryTicker);
}
assetData.AvailableTickers.Clear();
foreach (var a in resolvedTickers)
foreach (var a in validTickers)
{
assetData.AvailableTickers.Add(new TickerEntity
{
@@ -263,21 +325,19 @@ public class FundamentalsDbService : IFundamentalsDbService
Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker)
});
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}'",
assetData.Name, assetData.PrimaryTicker.Ticker);
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}' | AvailableTickers: {Count}",
assetData.Name, assetData.PrimaryTicker?.Ticker ?? "NULL", assetData.AvailableTickers.Count);
// --- Process Trade Republic Corporate Events ---
if (trDetails != null && (shouldUpdateAssetData || effectiveForceRefresh) && assetData != null)
if (trDetails != null && assetData != null)
{
// 1. Alte Events direkt in der DB löschen (bypasses Change Tracker)
await context.AssetEvents
.Where(e => e.AssetDataIsin == cleanIsin)
.ExecuteDeleteAsync(cancellationToken);
// 2. ALLE tracked AssetEventEntity-Einträge aus dem Change Tracker entfernen
foreach (var entry in context.ChangeTracker.Entries<AssetEventEntity>()
.Where(e => e.Entity.AssetDataIsin == cleanIsin)
.ToList())
@@ -285,7 +345,6 @@ public class FundamentalsDbService : IFundamentalsDbService
entry.State = EntityState.Detached;
}
// 3. Navigation-Collection zurücksetzen
assetData.AssetEvents = new List<AssetEventEntity>();
var trEventList = new List<TradeRepublicEventDto>();
@@ -324,18 +383,18 @@ public class FundamentalsDbService : IFundamentalsDbService
}
// --- Process Modules DTO (Executives & Fundamental Data) ---
if (modulesDto != null)
if (modulesDto != null || trDetails?.Company != null)
{
// Update KeyExecutives
if (shouldUpdateExecutives && assetData != null)
// Update KeyExecutives wenn Executives aus TR oder Yahoo vorliegen
var yahooOfficers = modulesDto?.AssetProfile?.CompanyOfficers;
bool hasTrOfficers = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
if (((yahooOfficers != null && yahooOfficers.Count > 0) || hasTrOfficers) && assetData != null)
{
// 1. Alte Executives direkt in der DB löschen (bypasses Change Tracker)
await context.KeyExecutives
.Where(e => e.AssetDataIsin == cleanIsin)
.ExecuteDeleteAsync(cancellationToken);
// 2. ALLE tracked KeyExecutiveEntity-Einträge aus dem Change Tracker entfernen
// (nicht nur die in der Navigation-Collection — der Tracker kann mehr halten)
foreach (var entry in context.ChangeTracker.Entries<KeyExecutiveEntity>()
.Where(e => e.Entity.AssetDataIsin == cleanIsin)
.ToList())
@@ -343,14 +402,12 @@ public class FundamentalsDbService : IFundamentalsDbService
entry.State = EntityState.Detached;
}
// 3. Navigation-Collection zurücksetzen
assetData.KeyExecutives = new List<KeyExecutiveEntity>();
// 4. Neue Executives aufbauen und direkt über den DbSet hinzufügen
if (modulesDto.AssetProfile?.CompanyOfficers != null)
if (yahooOfficers != null && yahooOfficers.Count > 0)
{
int sortIdx = 0;
foreach (var officer in modulesDto.AssetProfile.CompanyOfficers)
foreach (var officer in yahooOfficers)
{
if (!string.IsNullOrWhiteSpace(officer.Name))
{
@@ -368,6 +425,49 @@ public class FundamentalsDbService : IFundamentalsDbService
}
}
}
else if (hasTrOfficers && trDetails?.Company != null)
{
int sortIdx = 0;
if (!string.IsNullOrWhiteSpace(trDetails.Company.CeoName))
{
var ceo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CeoName,
Title = "CEO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(ceo);
assetData.KeyExecutives.Add(ceo);
}
if (!string.IsNullOrWhiteSpace(trDetails.Company.CfoName))
{
var cfo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CfoName,
Title = "CFO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(cfo);
assetData.KeyExecutives.Add(cfo);
}
if (!string.IsNullOrWhiteSpace(trDetails.Company.CooName))
{
var coo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CooName,
Title = "COO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(coo);
assetData.KeyExecutives.Add(coo);
}
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.",
@@ -375,16 +475,23 @@ public class FundamentalsDbService : IFundamentalsDbService
}
// Update FundamentalDataEntity
if (shouldUpdateFundamentals)
if (modulesDto != null && (modulesDto.SummaryDetail != null || modulesDto.DefaultKeyStatistics != null || modulesDto.FinancialData != null))
{
if (fundamentalData == null || !string.Equals(fundamentalData.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase))
{
fundamentalData = assetData?.FundamentalData?
.FirstOrDefault(f => string.Equals(f.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase));
}
if (fundamentalData == null)
{
fundamentalData = new FundamentalDataEntity
{
Isin = cleanIsin,
Id = Guid.NewGuid(),
AssetDataIsin = cleanIsin
};
context.FundamentalData.Add(fundamentalData);
assetData?.FundamentalData.Add(fundamentalData);
}
fundamentalData.Ticker = new TickerEntity
@@ -542,7 +649,7 @@ public class FundamentalsDbService : IFundamentalsDbService
: (assetData.PrimaryTicker != null ? new List<TickerEntity> { assetData.PrimaryTicker } : new List<TickerEntity>());
var tickerDtos = tickerEntities
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker))
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(assetData.Isin, StringComparison.OrdinalIgnoreCase))
.Select(a => new TickerInfoDto
{
Ticker = a.Ticker,