feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings

This commit is contained in:
2026-08-16 14:05:57 +02:00
parent b0f8d4b78b
commit 2ba54e8057
17 changed files with 1791 additions and 563 deletions
@@ -191,13 +191,17 @@ class FundamentalDataModel extends Equatable {
} }
final totalRev = parseNullableDouble(fundMap?['totalRevenue'] ?? json['totalRevenue']); final totalRev = parseNullableDouble(fundMap?['totalRevenue'] ?? json['totalRevenue']);
final grossProf = parseNullableDouble(fundMap?['grossProfit'] ?? json['grossProfit']); final rawGrossProfit = parseNullableDouble(fundMap?['grossProfit'] ?? json['grossProfit']);
double? grossMarginVal = parseNullableDouble(fundMap?['grossMargin'] ?? json['grossMargin']); double? grossMarginVal = parseNullableDouble(fundMap?['grossMargins'] ?? fundMap?['grossMargin'] ?? json['grossMargin']);
if (grossMarginVal == null && grossProf != null) { double? grossProfVal = rawGrossProfit;
if (grossProf <= 1.0 && grossProf >= 0.0) { if (rawGrossProfit != null) {
grossMarginVal = grossProf; if (rawGrossProfit <= 1.0 && rawGrossProfit >= 0.0) {
grossMarginVal ??= rawGrossProfit;
if (totalRev != null && totalRev > 0) {
grossProfVal = rawGrossProfit * totalRev;
}
} else if (totalRev != null && totalRev > 0) { } else if (totalRev != null && totalRev > 0) {
grossMarginVal = grossProf / totalRev; grossMarginVal ??= rawGrossProfit / totalRev;
} }
} }
@@ -207,6 +211,50 @@ class FundamentalDataModel extends Equatable {
evToRevVal = evVal / totalRev; evToRevVal = evVal / totalRev;
} }
String? exDivDateStr = fundMap?['exDividendDate']?.toString() ?? json['exDividendDate']?.toString();
String? nextEarningsDateStr = fundMap?['nextEarningsDate']?.toString() ?? json['nextEarningsDate']?.toString();
final rawEvents = json['events'];
if (rawEvents is List) {
final now = DateTime.now();
final divEvents = rawEvents.whereType<Map<String, dynamic>>().where((e) {
final t = e['type']?.toString().toUpperCase() ?? '';
return t == 'DIVIDEND' || t == 'EX_DIVIDEND';
}).toList();
if (exDivDateStr == null && divEvents.isNotEmpty) {
divEvents.sort((a, b) {
final da = DateTime.tryParse(a['date']?.toString() ?? '') ?? DateTime(1970);
final db = DateTime.tryParse(b['date']?.toString() ?? '') ?? DateTime(1970);
return da.compareTo(db);
});
final upcoming = divEvents.firstWhere((e) {
final d = DateTime.tryParse(e['date']?.toString() ?? '');
return d != null && d.isAfter(now.subtract(const Duration(days: 7)));
}, orElse: () => divEvents.last);
exDivDateStr = upcoming['date']?.toString();
}
final earningsEvents = rawEvents.whereType<Map<String, dynamic>>().where((e) {
final t = e['type']?.toString().toUpperCase() ?? '';
return t.contains('EARNINGS');
}).toList();
if (nextEarningsDateStr == null && earningsEvents.isNotEmpty) {
earningsEvents.sort((a, b) {
final da = DateTime.tryParse(a['date']?.toString() ?? '') ?? DateTime(1970);
final db = DateTime.tryParse(b['date']?.toString() ?? '') ?? DateTime(1970);
return da.compareTo(db);
});
final upcoming = earningsEvents.firstWhere((e) {
final d = DateTime.tryParse(e['date']?.toString() ?? '');
return d != null && d.isAfter(now.subtract(const Duration(days: 1)));
}, orElse: () => earningsEvents.last);
nextEarningsDateStr = upcoming['date']?.toString();
}
}
return FundamentalDataModel( return FundamentalDataModel(
isin: isinVal, isin: isinVal,
primaryTicker: primaryTickerVal, primaryTicker: primaryTickerVal,
@@ -228,25 +276,25 @@ class FundamentalDataModel extends Equatable {
fiftyTwoWeekLow: parseNullableDouble(fundMap?['fiftyTwoWeekLow'] ?? json['fiftyTwoWeekLow']), fiftyTwoWeekLow: parseNullableDouble(fundMap?['fiftyTwoWeekLow'] ?? json['fiftyTwoWeekLow']),
marketCapitalization: parseNullableDouble(fundMap?['marketCap'] ?? fundMap?['marketCapitalization'] ?? json['marketCapitalization']), marketCapitalization: parseNullableDouble(fundMap?['marketCap'] ?? fundMap?['marketCapitalization'] ?? json['marketCapitalization']),
enterpriseValue: evVal, enterpriseValue: evVal,
peRatioTrailing: parseNullableDouble(fundMap?['trailingPE'] ?? fundMap?['peRatioTrailing'] ?? json['peRatioTrailing']), peRatioTrailing: parseNullableDouble(fundMap?['trailingPe'] ?? fundMap?['trailingPE'] ?? fundMap?['peRatioTrailing'] ?? json['peRatioTrailing'] ?? json['trailingPe']),
peRatioForward: parseNullableDouble(fundMap?['forwardPE'] ?? fundMap?['peRatioForward'] ?? json['peRatioForward']), peRatioForward: parseNullableDouble(fundMap?['forwardPe'] ?? fundMap?['forwardPE'] ?? fundMap?['peRatioForward'] ?? json['peRatioForward'] ?? json['forwardPe']),
pegRatio: parseNullableDouble(fundMap?['pegRatio'] ?? json['pegRatio']), pegRatio: parseNullableDouble(fundMap?['pegRatio'] ?? json['pegRatio']),
pbRatio: parseNullableDouble(fundMap?['priceToBook'] ?? fundMap?['pbRatio'] ?? json['pbRatio']), pbRatio: parseNullableDouble(fundMap?['priceToBook'] ?? fundMap?['pbRatio'] ?? json['pbRatio']),
psRatio: parseNullableDouble(fundMap?['priceToSalesTrailing12Months'] ?? fundMap?['psRatio'] ?? json['psRatio']), psRatio: parseNullableDouble(fundMap?['priceToSales'] ?? fundMap?['priceToSalesTrailing12Months'] ?? fundMap?['psRatio'] ?? json['psRatio']),
evToEbitda: parseNullableDouble(fundMap?['enterpriseToEbitda'] ?? fundMap?['evToEbitda'] ?? json['evToEbitda']), evToEbitda: parseNullableDouble(fundMap?['evToEbitda'] ?? fundMap?['enterpriseToEbitda'] ?? json['evToEbitda']),
evToRevenue: evToRevVal, evToRevenue: evToRevVal,
totalRevenue: totalRev, totalRevenue: totalRev,
revenueGrowthYoY: parseNullableDouble(fundMap?['revenueGrowth'] ?? fundMap?['revenueGrowthYoY'] ?? json['revenueGrowthYoY']), revenueGrowthYoY: parseNullableDouble(fundMap?['revenueGrowthYoY'] ?? fundMap?['revenueGrowth'] ?? json['revenueGrowthYoY']),
grossProfit: grossProf, grossProfit: grossProfVal,
ebitda: parseNullableDouble(fundMap?['ebitda'] ?? json['ebitda']), ebitda: parseNullableDouble(fundMap?['ebitda'] ?? json['ebitda']),
dilutedEps: parseNullableDouble(fundMap?['trailingEps'] ?? fundMap?['dilutedEps'] ?? json['dilutedEps']), dilutedEps: parseNullableDouble(fundMap?['dilutedEps'] ?? fundMap?['trailingEps'] ?? json['dilutedEps']),
totalCash: parseNullableDouble(fundMap?['totalCash'] ?? json['totalCash']), totalCash: parseNullableDouble(fundMap?['totalCash'] ?? json['totalCash']),
totalDebt: parseNullableDouble(fundMap?['totalDebt'] ?? json['totalDebt']), totalDebt: parseNullableDouble(fundMap?['totalDebt'] ?? json['totalDebt']),
operatingCashFlow: parseNullableDouble(fundMap?['operatingCashflow'] ?? fundMap?['operatingCashFlow'] ?? json['operatingCashFlow']), operatingCashFlow: parseNullableDouble(fundMap?['operatingCashFlow'] ?? fundMap?['operatingCashflow'] ?? json['operatingCashFlow']),
freeCashFlow: parseNullableDouble(fundMap?['freeCashflow'] ?? fundMap?['freeCashFlow'] ?? json['freeCashFlow']), freeCashFlow: parseNullableDouble(fundMap?['freeCashFlow'] ?? fundMap?['freeCashflow'] ?? json['freeCashFlow']),
grossMargin: grossMarginVal, grossMargin: grossMarginVal,
operatingMargin: parseNullableDouble(fundMap?['operatingMargins'] ?? fundMap?['operatingMargin'] ?? json['operatingMargin']), operatingMargin: parseNullableDouble(fundMap?['operatingIncome'] ?? fundMap?['operatingMargins'] ?? fundMap?['operatingMargin'] ?? json['operatingMargin']),
netProfitMargin: parseNullableDouble(fundMap?['profitMargins'] ?? fundMap?['netProfitMargin'] ?? json['netProfitMargin']), netProfitMargin: parseNullableDouble(fundMap?['netIncome'] ?? fundMap?['profitMargins'] ?? fundMap?['netProfitMargin'] ?? json['netProfitMargin']),
returnOnEquity: parseNullableDouble(fundMap?['returnOnEquity'] ?? json['returnOnEquity']), returnOnEquity: parseNullableDouble(fundMap?['returnOnEquity'] ?? json['returnOnEquity']),
returnOnAssets: parseNullableDouble(fundMap?['returnOnAssets'] ?? json['returnOnAssets']), returnOnAssets: parseNullableDouble(fundMap?['returnOnAssets'] ?? json['returnOnAssets']),
returnOnInvestedCapital: parseNullableDouble(fundMap?['returnOnInvestedCapital'] ?? json['returnOnInvestedCapital']), returnOnInvestedCapital: parseNullableDouble(fundMap?['returnOnInvestedCapital'] ?? json['returnOnInvestedCapital']),
@@ -254,10 +302,10 @@ class FundamentalDataModel extends Equatable {
currentRatio: parseNullableDouble(fundMap?['currentRatio'] ?? json['currentRatio']), currentRatio: parseNullableDouble(fundMap?['currentRatio'] ?? json['currentRatio']),
quickRatio: parseNullableDouble(fundMap?['quickRatio'] ?? json['quickRatio']), quickRatio: parseNullableDouble(fundMap?['quickRatio'] ?? json['quickRatio']),
interestCoverage: parseNullableDouble(fundMap?['interestCoverage'] ?? json['interestCoverage']), interestCoverage: parseNullableDouble(fundMap?['interestCoverage'] ?? json['interestCoverage']),
dividendYield: parseNullableDouble(fundMap?['dividendYield'] ?? json['dividendYield']), dividendYield: parseNullableDouble(fundMap?['forwardDividendYield'] ?? fundMap?['dividendYield'] ?? json['dividendYield']),
payoutRatio: parseNullableDouble(fundMap?['payoutRatio'] ?? json['payoutRatio']), payoutRatio: parseNullableDouble(fundMap?['payoutRatio'] ?? json['payoutRatio']),
exDividendDate: fundMap?['exDividendDate']?.toString() ?? json['exDividendDate']?.toString(), exDividendDate: exDivDateStr,
nextEarningsDate: fundMap?['nextEarningsDate']?.toString() ?? json['nextEarningsDate']?.toString(), nextEarningsDate: nextEarningsDateStr,
percentHeldByInstitutions: parseNullableDouble(fundMap?['percentHeldByInstitutions'] ?? json['percentHeldByInstitutions']), percentHeldByInstitutions: parseNullableDouble(fundMap?['percentHeldByInstitutions'] ?? json['percentHeldByInstitutions']),
percentHeldByInsiders: parseNullableDouble(fundMap?['percentHeldByInsiders'] ?? json['percentHeldByInsiders']), percentHeldByInsiders: parseNullableDouble(fundMap?['percentHeldByInsiders'] ?? json['percentHeldByInsiders']),
shortRatio: parseNullableDouble(fundMap?['shortRatio'] ?? json['shortRatio']), shortRatio: parseNullableDouble(fundMap?['shortRatio'] ?? json['shortRatio']),
@@ -45,7 +45,7 @@ class FundamentalCategoryPanels extends StatelessWidget {
String _fmtPercent(double? val) { String _fmtPercent(double? val) {
if (val == null) return 'N/A'; if (val == null) return 'N/A';
final p = (val.abs() <= 1.0 && val != 0.0) ? val * 100.0 : val; final p = (val.abs() <= 5.0 && val != 0.0) ? val * 100.0 : val;
return '${p.toStringAsFixed(2)}%'; return '${p.toStringAsFixed(2)}%';
} }
+416 -128
View File
@@ -9,6 +9,8 @@ using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo; using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings; using FinlyticCore.Models.Settings;
using FinlyticCore.Services; using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using Microsoft.Playwright;
namespace FinlyticCore.Services.Yahoo; namespace FinlyticCore.Services.Yahoo;
@@ -18,9 +20,11 @@ public class YahooFinanceClient
private readonly CookieContainer _cookieContainer; private readonly CookieContainer _cookieContainer;
private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger; private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
private readonly ISettingsService? _settingsService; private readonly ISettingsService? _settingsService;
private readonly IPlaywrightExecutionService? _playwrightService;
private readonly SemaphoreSlim _authLock = new(1, 1); private readonly SemaphoreSlim _authLock = new(1, 1);
private string? _crumb; private string? _crumb;
private string? _rawCookieHeader;
private DateTime _lastAuthTime = DateTime.MinValue; private DateTime _lastAuthTime = DateTime.MinValue;
/// <summary> /// <summary>
@@ -44,10 +48,12 @@ public class YahooFinanceClient
public YahooFinanceClient( public YahooFinanceClient(
IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null, IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
ISettingsService? settingsService = null, ISettingsService? settingsService = null,
IPlaywrightExecutionService? playwrightService = null,
HttpClient? httpClient = null) HttpClient? httpClient = null)
{ {
_finlyticLogger = finlyticLogger; _finlyticLogger = finlyticLogger;
_settingsService = settingsService; _settingsService = settingsService;
_playwrightService = playwrightService;
_cookieContainer = new CookieContainer(); _cookieContainer = new CookieContainer();
if (httpClient != null) if (httpClient != null)
@@ -97,6 +103,7 @@ public class YahooFinanceClient
{ {
RestoreCookies(cachedCookies); RestoreCookies(cachedCookies);
_crumb = cachedCrumb; _crumb = cachedCrumb;
_rawCookieHeader = cachedCookies;
_lastAuthTime = DateTime.UtcNow; _lastAuthTime = DateTime.UtcNow;
if (_finlyticLogger != null) if (_finlyticLogger != null)
@@ -115,70 +122,40 @@ public class YahooFinanceClient
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)..."); await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");
// 3. Send GET request to fc.yahoo.com to obtain session cookie A3 // 3. Send GET request to fc.yahoo.com to obtain session cookie A3
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com")) try
{ {
using var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com");
initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"); initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken); using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
} }
catch { }
// 4. Send GET request to getcrumb to obtain dynamic crumb token // 4. Send GET request to getcrumb to obtain dynamic crumb token
string[] crumbUrls = new[] var crumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
if (!string.IsNullOrWhiteSpace(crumb))
{ {
"https://query1.finance.yahoo.com/v1/test/getcrumb", _crumb = crumb;
"https://query2.finance.yahoo.com/v1/test/getcrumb" _lastAuthTime = DateTime.UtcNow;
}; await PersistSessionAsync(_crumb, cancellationToken);
return _crumb;
}
foreach (var url in crumbUrls) // 5. FALLBACK: Playwright Browser Authentication (Bypasses EU Consent Wall and 429)
if (_playwrightService != null)
{ {
try var (browserCrumb, browserCookies) = await AuthenticateViaPlaywrightAsync(cancellationToken);
if (!string.IsNullOrWhiteSpace(browserCrumb))
{ {
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url); _crumb = browserCrumb;
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); _lastAuthTime = DateTime.UtcNow;
crumbRequest.Headers.Add("Accept", "*/*"); if (!string.IsNullOrWhiteSpace(browserCookies))
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (crumbResponse.IsSuccessStatusCode)
{ {
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken); RestoreCookies(browserCookies);
if (!string.IsNullOrWhiteSpace(crumbText)) _rawCookieHeader = browserCookies;
{
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
// Persist to DB cache via SettingsService
if (_settingsService != null)
{
try
{
var serializedCookies = SerializeCookies();
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken);
if (!string.IsNullOrWhiteSpace(serializedCookies))
{
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
}
}
catch (Exception persistEx)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database.");
}
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb);
return _crumb;
}
} }
else await PersistSessionAsync(_crumb, cancellationToken);
{ return _crumb;
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode);
}
}
catch
{
// Fallthrough to next endpoint
} }
} }
@@ -198,13 +175,204 @@ public class YahooFinanceClient
} }
} }
private async Task<string?> FetchCrumbWithHttpClientAsync(CancellationToken cancellationToken)
{
string[] crumbUrls = new[]
{
"https://query1.finance.yahoo.com/v1/test/getcrumb",
"https://query2.finance.yahoo.com/v1/test/getcrumb"
};
foreach (var url in crumbUrls)
{
try
{
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
crumbRequest.Headers.Add("Accept", "*/*");
crumbRequest.Headers.Add("Origin", "https://finance.yahoo.com");
crumbRequest.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
crumbRequest.Headers.Add("Cookie", _rawCookieHeader);
}
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
if (crumbResponse.IsSuccessStatusCode && !string.IsNullOrWhiteSpace(crumbText))
{
var cleanCrumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
if (!cleanCrumb.Contains("<html", StringComparison.OrdinalIgnoreCase) && cleanCrumb.Length < 100)
{
return cleanCrumb;
}
}
else
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] Endpoint '{Url}' returned status {Status}. Response Body: {Body}",
url, crumbResponse.StatusCode, crumbText);
}
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during FetchCrumb on '{Url}'", url);
}
}
return null;
}
private async Task<(string? crumb, string? cookieStr)> AuthenticateViaPlaywrightAsync(CancellationToken cancellationToken)
{
if (_playwrightService == null) return (null, null);
try
{
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Launching Playwright browser to acquire valid EU Yahoo session and Crumb...");
return await _playwrightService.ExecuteInContextAsync(async context =>
{
var page = await context.NewPageAsync();
try
{
await page.GotoAsync("https://finance.yahoo.com/quote/AAPL/", new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30_000
});
// Handle EU Consent if redirected
var url = page.Url;
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
{
var selectors = new[]
{
"button[name='agree']",
"button[value='agree']",
"button.accept-all",
"button.btn.primary",
"button.btn.secondary.accept-all",
"form[action*='consent'] button[type='submit']",
"button:has-text('Alle akzeptieren')",
"button:has-text('Accept all')",
"button:has-text('Akzeptieren')",
"button:has-text('Agree')"
};
foreach (var sel in selectors)
{
var btn = page.Locator(sel);
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
{
await btn.First.ClickAsync();
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15_000 });
break;
}
}
}
// 1. Extract cookies from BrowserContext
var cookies = await context.CookiesAsync();
var pairs = new List<string>();
foreach (var c in cookies)
{
if (c.Domain.Contains("yahoo.com", StringComparison.OrdinalIgnoreCase))
{
pairs.Add($"{c.Name}={c.Value}");
}
}
var extractedCookies = string.Join(";", pairs);
// 2. Fetch Crumb from inside the authenticated page
string? extractedCrumb = null;
try
{
extractedCrumb = await page.EvaluateAsync<string>(@"async () => {
try {
const res = await fetch('/v1/test/getcrumb');
if (res.ok) {
return await res.text();
}
} catch {}
return null;
}");
}
catch { }
// 3. If in-page fetch was empty, use the extracted cookies with HttpClient
if (string.IsNullOrWhiteSpace(extractedCrumb) && !string.IsNullOrWhiteSpace(extractedCookies))
{
RestoreCookies(extractedCookies);
_rawCookieHeader = extractedCookies;
extractedCrumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
}
if (!string.IsNullOrWhiteSpace(extractedCrumb))
{
extractedCrumb = extractedCrumb.Trim('"', ' ', '\t', '\r', '\n');
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Playwright successfully acquired Yahoo Crumb token: {Crumb}", extractedCrumb);
}
return (extractedCrumb, extractedCookies);
}
finally
{
await page.CloseAsync();
}
}, cancellationToken: cancellationToken);
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Playwright-based authentication failed.");
return (null, null);
}
}
private async Task PersistSessionAsync(string crumb, CancellationToken cancellationToken)
{
if (_settingsService == null || string.IsNullOrWhiteSpace(crumb)) return;
try
{
var serializedCookies = SerializeCookies();
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, crumb, cancellationToken);
if (!string.IsNullOrWhiteSpace(serializedCookies))
{
_rawCookieHeader = serializedCookies;
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Persisted valid Yahoo session & crumb to database.");
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Failed to persist Yahoo session to database.");
}
}
private string SerializeCookies() private string SerializeCookies()
{ {
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
return _rawCookieHeader;
}
try try
{ {
var cookies = _cookieContainer.GetAllCookies(); var cookies = _cookieContainer.GetAllCookies();
var pairs = new List<string>(); var pairs = new List<string>();
foreach (Cookie cookie in cookies) foreach (System.Net.Cookie cookie in cookies)
{ {
pairs.Add($"{cookie.Name}={cookie.Value}"); pairs.Add($"{cookie.Name}={cookie.Value}");
} }
@@ -220,9 +388,20 @@ public class YahooFinanceClient
{ {
if (string.IsNullOrWhiteSpace(serializedCookies)) return; if (string.IsNullOrWhiteSpace(serializedCookies)) return;
_rawCookieHeader = serializedCookies;
try try
{ {
var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var uris = new[]
{
new Uri("https://yahoo.com"),
new Uri("https://finance.yahoo.com"),
new Uri("https://query1.finance.yahoo.com"),
new Uri("https://query2.finance.yahoo.com"),
new Uri("https://fc.yahoo.com")
};
foreach (var part in parts) foreach (var part in parts)
{ {
var eqIdx = part.IndexOf('='); var eqIdx = part.IndexOf('=');
@@ -230,7 +409,14 @@ public class YahooFinanceClient
{ {
var name = part.Substring(0, eqIdx).Trim(); var name = part.Substring(0, eqIdx).Trim();
var val = part.Substring(eqIdx + 1).Trim(); var val = part.Substring(eqIdx + 1).Trim();
_cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com")); foreach (var uri in uris)
{
try
{
_cookieContainer.Add(uri, new System.Net.Cookie(name, val));
}
catch { }
}
} }
} }
} }
@@ -242,8 +428,6 @@ public class YahooFinanceClient
/// <summary> /// <summary>
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API. /// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
/// Note: Does not require Cookie/Crumb authentication.
/// </summary> /// </summary>
public async Task<YahooSearchResponseDto?> SearchAsync( public async Task<YahooSearchResponseDto?> SearchAsync(
string query, string query,
@@ -253,33 +437,55 @@ public class YahooFinanceClient
{ {
if (string.IsNullOrWhiteSpace(query)) return null; if (string.IsNullOrWhiteSpace(query)) return null;
try string[] endpoints = new[]
{ {
var url = $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}",
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}"; $"https://query1.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}"
using var response = await _httpClient.GetAsync(url, cancellationToken); };
if (!response.IsSuccessStatusCode) foreach (var url in endpoints)
{
try
{
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var json = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
if (dto != null)
{
return dto;
}
}
else
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] Search endpoint '{Url}' for '{Query}' returned status {Status}. Response Body: {Body}",
url, query, response.StatusCode, json);
}
}
catch (Exception ex)
{ {
if (_finlyticLogger != null) if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode); await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search endpoint '{Url}'", url);
return null;
} }
}
var json = await response.Content.ReadAsStringAsync(cancellationToken); return null;
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
return null;
}
} }
/// <summary> /// <summary>
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint. /// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&amp;modules={modules}
/// </summary> /// </summary>
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync( public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol, string symbol,
@@ -291,22 +497,49 @@ public class YahooFinanceClient
var moduleList = string.Join(",", modules); var moduleList = string.Join(",", modules);
return await ExecuteWithRetryAsync(async (crumb) => return await ExecuteWithRetryAsync(async (crumb) =>
{ {
var url = string[] baseUrls = new[]
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{ {
"https://query2.finance.yahoo.com/v10/finance/quoteSummary",
"https://query1.finance.yahoo.com/v10/finance/quoteSummary"
};
foreach (var baseUrl in baseUrls)
{
var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Origin", "https://finance.yahoo.com");
request.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(responseContent, GetJsonOptions());
return (false, dto);
}
if (_finlyticLogger != null) if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode); await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
return ( "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}",
response.StatusCode == HttpStatusCode.Unauthorized || symbol, response.StatusCode, url, responseContent);
response.StatusCode == HttpStatusCode.Forbidden, null);
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.TooManyRequests)
{
return (true, null);
}
} }
var json = await response.Content.ReadAsStringAsync(cancellationToken); return (false, null);
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken); }, cancellationToken);
} }
@@ -321,7 +554,6 @@ public class YahooFinanceClient
/// <summary> /// <summary>
/// Retrieves historical OHLCV chart data for a given symbol. /// Retrieves historical OHLCV chart data for a given symbol.
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&amp;interval={interval}&amp;crumb={crumb}
/// </summary> /// </summary>
public async Task<YahooChartResponseDto?> GetChartAsync( public async Task<YahooChartResponseDto?> GetChartAsync(
string symbol, string symbol,
@@ -333,28 +565,54 @@ public class YahooFinanceClient
return await ExecuteWithRetryAsync(async (crumb) => return await ExecuteWithRetryAsync(async (crumb) =>
{ {
var url = string[] baseUrls = new[]
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{ {
"https://query1.finance.yahoo.com/v8/finance/chart",
"https://query2.finance.yahoo.com/v8/finance/chart"
};
foreach (var baseUrl in baseUrls)
{
var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Origin", "https://finance.yahoo.com");
request.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(responseContent, GetJsonOptions());
return (false, dto);
}
if (_finlyticLogger != null) if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode); await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
return ( "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}",
response.StatusCode == HttpStatusCode.Unauthorized || symbol, response.StatusCode, url, responseContent);
response.StatusCode == HttpStatusCode.Forbidden, null);
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.TooManyRequests)
{
return (true, null);
}
} }
var json = await response.Content.ReadAsStringAsync(cancellationToken); return (false, null);
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken); }, cancellationToken);
} }
/// <summary> /// <summary>
/// Retrieves quick real-time price quotes for one or more symbols. /// Retrieves quick real-time price quotes for one or more symbols.
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&amp;crumb={crumb}
/// </summary> /// </summary>
public async Task<YahooQuoteResponseDto?> GetQuotesAsync( public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols, IEnumerable<string> symbols,
@@ -366,22 +624,49 @@ public class YahooFinanceClient
var symbolsParam = string.Join(",", symbolList); var symbolsParam = string.Join(",", symbolList);
return await ExecuteWithRetryAsync(async (crumb) => return await ExecuteWithRetryAsync(async (crumb) =>
{ {
var url = string[] baseUrls = new[]
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{ {
"https://query1.finance.yahoo.com/v7/finance/quote",
"https://query2.finance.yahoo.com/v7/finance/quote"
};
foreach (var baseUrl in baseUrls)
{
var url = $"{baseUrl}?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Origin", "https://finance.yahoo.com");
request.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(responseContent, GetJsonOptions());
return (false, dto);
}
if (_finlyticLogger != null) if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode); await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
return ( "[YahooFinanceClient] GetQuotes failed with status {Status}. URL: {Url}. Response Body: {Body}",
response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode, url, responseContent);
response.StatusCode == HttpStatusCode.Forbidden, null);
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.TooManyRequests)
{
return (true, null);
}
} }
var json = await response.Content.ReadAsStringAsync(cancellationToken); return (false, null);
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken); }, cancellationToken);
} }
@@ -408,26 +693,29 @@ public class YahooFinanceClient
CancellationToken cancellationToken) where T : class CancellationToken cancellationToken) where T : class
{ {
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken); var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
if (!string.IsNullOrEmpty(crumb))
{
var (isAuthError, result) = await action(crumb);
if (!isAuthError && result != null)
{
return result;
}
if (!isAuthError)
{
return result;
}
}
// Re-authenticate when auth error (401/403/429) or empty crumb occurs
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error or invalid crumb encountered (401/403/429). Force re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null; if (string.IsNullOrEmpty(crumb)) return null;
var (isAuthError, result) = await action(crumb); var (_, retryResult) = await action(crumb);
if (!isAuthError && result != null) return retryResult;
{
return result;
}
if (isAuthError)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (_, retryResult) = await action(crumb);
return retryResult;
}
return result;
} }
private static JsonSerializerOptions GetJsonOptions() private static JsonSerializerOptions GetJsonOptions()
+374 -249
View File
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization; using System.Globalization;
using System.Linq; using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -17,14 +18,15 @@ public interface IYahooFinanceHtmlClient
{ {
Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync( Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
string isinOrSymbol, string isinOrSymbol,
bool includeProfile = true,
CancellationToken cancellationToken = default); CancellationToken cancellationToken = default);
} }
public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
{ {
private const string _serviceName = nameof(YahooFinanceHtmlClient);
private readonly IPlaywrightExecutionService _playwrightService; private readonly IPlaywrightExecutionService _playwrightService;
private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger; private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
private const string _serviceName = "YahooFinanceHtmlClient";
public YahooFinanceHtmlClient( public YahooFinanceHtmlClient(
IPlaywrightExecutionService playwrightService, IPlaywrightExecutionService playwrightService,
@@ -36,55 +38,93 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync( public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
string isinOrSymbol, string isinOrSymbol,
bool includeProfile = true,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null; if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;
var symbol = isinOrSymbol.Trim().ToUpperInvariant(); var symbol = isinOrSymbol.Trim().ToUpperInvariant();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'..."); await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Starting parallel fast Playwright HTML DOM scrape for symbol '{symbol}' (IncludeProfile: {includeProfile})...");
return await _playwrightService.ExecuteInContextAsync(async context => return await _playwrightService.ExecuteInContextAsync(async context =>
{ {
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); var summaryUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/";
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var analysisData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
ProfileExtractionResult? profileResult = null;
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/"; var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/"; var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/"; var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken); // 1. Initial Page: Authenticate session and pass Cookie Consent once for the entire context
var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken); var initialPage = await context.NewPageAsync();
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken); var summaryData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken);
try try
{ {
await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask); await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [1/2] Loading Summary & passing Consent: {summaryUrl}");
await initialPage.GotoAsync(summaryUrl, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 15_000
});
keyStatsData = await statsTask; await HandleConsentAsync(initialPage);
profileResult = await profileTask; await WaitForContentAsync(initialPage);
financialsData = await financialsTask; summaryData = await ExtractKeyValuePairsFromPageAsync(initialPage, summaryUrl);
analysisData = await analysisTask;
} }
catch (Exception ex) catch (Exception ex)
{ {
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}."); await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping initial Summary page for {symbol}.");
}
finally
{
await initialPage.CloseAsync();
}
// 2. Parallel Sub-Pages (Stats, Financials, and conditionally Profile)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [2/2] Fetching Sub-Pages in parallel (IncludeProfile: {includeProfile})...");
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
Task<ProfileExtractionResult?> profileTask = includeProfile
? ScrapeProfilePageAsync(context, profileUrl, cancellationToken)!
: Task.FromResult<ProfileExtractionResult?>(null);
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
ProfileExtractionResult? profileResult = null;
try
{
await Task.WhenAll(statsTask, financialsTask, profileTask);
keyStatsData = await statsTask;
financialsData = await financialsTask;
profileResult = await profileTask;
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error during parallel sub-page scrape for {symbol}.");
}
// Merge summary data into key stats
foreach (var (k, v) in summaryData)
{
if (!keyStatsData.ContainsKey(k))
{
keyStatsData[k] = v;
}
} }
var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>(); var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}"); await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Fast HTML DOM scrape complete for '{symbol}'. Summary: {summaryData.Count}, Stats: {keyStatsData.Count}, Financials: {financialsData.Count}, Profile: {profileDict.Count}, Officers: {officers.Count}");
return BuildModulesDto( return BuildModulesDto(
keyStatsData, keyStatsData,
profileDict, profileDict,
financialsData, financialsData,
analysisData, new Dictionary<string, string>(),
officers, officers,
profileResult?.Sector, profileResult?.Sector,
profileResult?.Industry, profileResult?.Industry,
@@ -107,47 +147,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
await page.GotoAsync(url, new PageGotoOptions await page.GotoAsync(url, new PageGotoOptions
{ {
WaitUntil = WaitUntilState.DOMContentLoaded, WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 20_000 Timeout = 10_000
}); });
await HandleConsentAsync(page); await WaitForContentAsync(page);
targetDict = await ExtractKeyValuePairsFromPageAsync(page, url);
var extracted = await page.EvaluateAsync<Dictionary<string, string>>(@"() => {
const results = {};
const cleanKey = (str) => {
return str.toLowerCase()
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
.replace(/\s*\d+\s*$/, '')
.replace(/\s+/g, ' ')
.trim();
};
document.querySelectorAll('table tr').forEach(tr => {
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
if (cells.length >= 2 && cells[0] && cells[1]) {
const key = cleanKey(cells[0]);
const val = cells[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-') {
results[key] = val;
}
}
});
return results;
}");
if (extracted != null)
{
foreach (var (k, v) in extracted)
{
targetDict[k] = v;
}
}
} }
catch (Exception ex) catch (Exception ex)
{ {
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}"); await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping URL {url}");
} }
finally finally
{ {
@@ -157,7 +165,7 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
return targetDict; return targetDict;
} }
private async Task<ProfileExtractionResult> ScrapeProfilePageAsync( private async Task<ProfileExtractionResult?> ScrapeProfilePageAsync(
IBrowserContext context, IBrowserContext context,
string url, string url,
CancellationToken cancellationToken) CancellationToken cancellationToken)
@@ -176,15 +184,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
await page.GotoAsync(url, new PageGotoOptions await page.GotoAsync(url, new PageGotoOptions
{ {
WaitUntil = WaitUntilState.DOMContentLoaded, WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 20_000 Timeout = 10_000
}); });
await HandleConsentAsync(page); await WaitForContentAsync(page);
var metaInfo = await page.EvaluateAsync<ProfileMetaJsResult>(@"() => { var jsonStr = await page.EvaluateAsync<string>(@"() => {
let sector = null, industry = null, employees = null, description = null; let sector = null, industry = null, employees = null, description = null;
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary'); const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary, section[data-testid=""asset-profile""] p');
if (descEl) description = descEl.innerText.trim(); if (descEl) description = descEl.innerText.trim();
const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main'); const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
@@ -219,40 +227,51 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
} }
}); });
return { sector, industry, employees, description, officers }; return JSON.stringify({ sector, industry, employees, description, officers });
}"); }");
if (metaInfo != null) if (!string.IsNullOrWhiteSpace(jsonStr))
{ {
sector = metaInfo.Sector; var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
industry = metaInfo.Industry; var metaInfo = JsonSerializer.Deserialize<ProfileMetaJsResult>(jsonStr, options);
fullTimeEmployees = metaInfo.Employees;
description = metaInfo.Description;
if (metaInfo.Officers != null) if (metaInfo != null)
{ {
foreach (var off in metaInfo.Officers) sector = metaInfo.Sector;
industry = metaInfo.Industry;
fullTimeEmployees = metaInfo.Employees;
description = metaInfo.Description;
if (metaInfo.Officers != null)
{ {
if (!string.IsNullOrWhiteSpace(off.Name)) foreach (var off in metaInfo.Officers)
{ {
companyOfficers.Add(new YahooCompanyOfficerDto( if (!string.IsNullOrWhiteSpace(off.Name))
Name: off.Name, {
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null, companyOfficers.Add(new YahooCompanyOfficerDto(
Title: off.Title, Name: off.Name,
YearBorn: off.YearBorn, Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
FiscalYear: null, Title: off.Title,
TotalPay: ParseYahooValue(off.Pay), YearBorn: off.YearBorn,
ExercisedValue: ParseYahooValue(off.Exercised), FiscalYear: null,
UnexercisedValue: null TotalPay: ParseYahooValue(off.Pay),
)); ExercisedValue: ParseYahooValue(off.Exercised),
UnexercisedValue: null
));
}
} }
} }
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] HTML Profile extracted -> Sector: '{sector}', Industry: '{industry}', Employees: {fullTimeEmployees}, Officers: {companyOfficers.Count}, Desc length: {description?.Length ?? 0}");
} }
} }
profileDict = await ExtractKeyValuePairsFromPageAsync(page, url);
} }
catch (Exception ex) catch (Exception ex)
{ {
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}"); await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping Profile data from {url}");
} }
finally finally
{ {
@@ -262,21 +281,151 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description); return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
} }
private static async Task HandleConsentAsync(IPage page) private static async Task WaitForContentAsync(IPage page)
{ {
try try
{ {
if (page.Url.Contains("consent.yahoo.com")) await page.WaitForSelectorAsync("table tr, ul li, div[data-testid], section, main", new PageWaitForSelectorOptions
{ {
var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button"); State = WaitForSelectorState.Attached,
if (await consentBtn.CountAsync() > 0) Timeout = 2_500
});
}
catch { }
}
private async Task<Dictionary<string, string>> ExtractKeyValuePairsFromPageAsync(IPage page, string pageUrl)
{
var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
try
{
var jsonStr = await page.EvaluateAsync<string>(@"() => {
const results = {};
const cleanKey = (str) => {
return str.toLowerCase()
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
.replace(/\s*\d+\s*$/, '')
.replace(/\s+/g, ' ')
.trim();
};
// 1. Standard HTML Table Rows (Financials, Balance Sheet, Key Stats)
document.querySelectorAll('table tr').forEach(tr => {
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
if (cells.length >= 2 && cells[0] && cells[1]) {
const key = cleanKey(cells[0]);
const val = cells[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
results[key] = val;
}
}
});
// 2. Modern Yahoo Finance List / Div pairs (Summary Table, Quote Statistics, Flex containers)
document.querySelectorAll('li, div[class*=""container""], div[class*=""row""], section div, div[data-testid]').forEach(el => {
const children = Array.from(el.querySelectorAll(':scope > span, :scope > div, :scope > p')).map(s => s.innerText.trim()).filter(Boolean);
if (children.length === 2) {
const key = cleanKey(children[0]);
const val = children[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
if (!results[key]) {
results[key] = val;
}
}
}
});
return JSON.stringify(results);
}");
if (!string.IsNullOrWhiteSpace(jsonStr))
{
var parsed = JsonSerializer.Deserialize<Dictionary<string, string>>(jsonStr);
if (parsed != null)
{ {
await consentBtn.First.ClickAsync(); foreach (var (k, v) in parsed)
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 }); {
targetDict[k] = v;
}
}
}
var sampleKeys = targetDict.Keys.Take(6).ToList();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Extracted {targetDict.Count} items from HTML of '{pageUrl}'. Sample keys: [{string.Join(", ", sampleKeys)}]");
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error extracting key-value pairs from HTML of '{pageUrl}'");
}
return targetDict;
}
private async Task HandleConsentAsync(IPage page)
{
try
{
var url = page.Url;
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Detected EU Cookie Consent redirect: '{url}'. Searching for Accept/Reject buttons...");
var selectors = new[]
{
"button[name='agree']",
"button[value='agree']",
"button.accept-all",
"button.btn.primary",
"button.btn.secondary.accept-all",
"button[name='reject']",
"button[value='reject']",
"button.reject-all",
"form[action*='consent'] button[type='submit']",
"button:has-text('Alle akzeptieren')",
"button:has-text('Accept all')",
"button:has-text('Alle ablehnen')",
"button:has-text('Reject all')",
"button:has-text('Akzeptieren')",
"button:has-text('Ablehnen')",
"button:has-text('Agree')",
"button:has-text('I agree')"
};
foreach (var sel in selectors)
{
var btn = page.Locator(sel);
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
{
var btnText = await btn.First.InnerTextAsync();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Found consent button '{btnText.Trim()}' with selector '{sel}'. Clicking...");
// Fast click and wait for DOMContentLoaded on finance.yahoo.com
await btn.First.ClickAsync();
try
{
await page.WaitForURLAsync(u => u.Contains("finance.yahoo.com", StringComparison.OrdinalIgnoreCase),
new PageWaitForURLOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 5_000 });
}
catch { }
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Successfully passed consent wall. Current URL: '{page.Url}'");
break;
}
} }
} }
} }
catch { /* Fallback */ } catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex,
$"[{_serviceName}] Exception in HandleConsentAsync.");
}
} }
private YahooQuoteSummaryModulesDto BuildModulesDto( private YahooQuoteSummaryModulesDto BuildModulesDto(
@@ -297,10 +446,8 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
var assetProfile = new YahooAssetProfileDto( var assetProfile = new YahooAssetProfileDto(
Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null, Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
Industry: industry ?? GetString(allStats, "industry"), Industry: industry, IndustryKey: null, IndustryDisp: null,
IndustryKey: null, IndustryDisp: null, Sector: sector, SectorKey: null, SectorDisp: null,
Sector: sector ?? GetString(allStats, "sector"),
SectorKey: null, SectorDisp: null,
LongBusinessSummary: description, LongBusinessSummary: description,
FullTimeEmployees: fullTimeEmployees, FullTimeEmployees: fullTimeEmployees,
CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null, CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
@@ -308,100 +455,104 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
GovernanceEpochDate: null, CompensationAsOfEpochDate: null GovernanceEpochDate: null, CompensationAsOfEpochDate: null
); );
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto( var financialData = new YahooFinancialDataDto(
PriceToBook: GetVal(allStats, "price/book", "price / book"), CurrentPrice: GetVal(allStats, "previous close", "current price", "price", "regular market price"),
EnterpriseValue: GetVal(allStats, "enterprise value"), TargetHighPrice: GetVal(allStats, "target high price", "target high"),
ForwardPE: GetVal(allStats, "forward p/e"), TargetLowPrice: GetVal(allStats, "target low price", "target low"),
ProfitMargins: GetVal(allStats, "profit margin"), TargetMeanPrice: GetVal(allStats, "1y target est", "target mean price", "target est"),
FloatShares: GetVal(allStats, "float"), TargetMedianPrice: GetVal(allStats, "target median price"),
SharesOutstanding: GetVal(allStats, "shares outstanding"), RecommendationMean: GetVal(allStats, "recommendation mean"),
SharesShort: GetVal(allStats, "shares short"), RecommendationKey: allStats.GetValueOrDefault("recommendation") ?? allStats.GetValueOrDefault("recommendation key"),
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"), NumberOfAnalystOpinions: GetVal(allStats, "number of analyst opinions", "analyst opinions"),
SharesShortPreviousMonthDate: null, DateShortInterest: null, TotalCash: GetVal(allStats, "total cash", "total cash (mrq)"),
SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"), TotalCashPerShare: GetVal(allStats, "total cash per share", "total cash per share (mrq)"),
HeldPercentInsiders: GetVal(allStats, "% held by insiders"), Ebitda: GetVal(allStats, "ebitda"),
HeldPercentInstitutions: GetVal(allStats, "% held by institutions"), TotalDebt: GetVal(allStats, "total debt", "total debt (mrq)"),
ShortRatio: GetVal(allStats, "short ratio"), QuickRatio: GetVal(allStats, "quick ratio"),
ShortPercentOfFloat: GetVal(allStats, "short % of float"), CurrentRatio: GetVal(allStats, "current ratio", "current ratio (mrq)"),
Beta: GetVal(allStats, "beta (5y monthly)", "beta"), TotalRevenue: GetVal(allStats, "total revenue", "revenue", "revenue (ttm)"),
Category: null, DebtToEquity: GetVal(allStats, "total debt/equity", "total debt/equity (mrq)", "debt to equity"),
BookValue: GetVal(allStats, "book value per share", "book value"), RevenuePerShare: GetVal(allStats, "revenue per share", "revenue per share (ttm)"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"), ReturnOnAssets: GetVal(allStats, "return on assets", "return on assets (ttm)"),
LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"), ReturnOnEquity: GetVal(allStats, "return on equity", "return on equity (ttm)"),
NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"), GrossProfits: GetVal(allStats, "gross profit", "gross profit (ttm)", "gross profits"),
MostRecentQuarter: GetVal(allStats, "most recent quarter"), FreeCashflow: GetVal(allStats, "levered free cash flow", "levered free cash flow (ttm)", "free cash flow"),
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"), OperatingCashflow: GetVal(allStats, "operating cash flow", "operating cash flow (ttm)"),
NetIncomeToCommon: GetVal(allStats, "net income avi to common"), RevenueGrowth: GetVal(allStats, "quarterly revenue growth", "quarterly revenue growth (yoy)", "revenue growth"),
TrailingEps: GetVal(allStats, "diluted eps"), GrossMargins: GetVal(allStats, "gross margin", "gross margins"),
ForwardEps: GetVal(allStats, "forward eps"), EbitdaMargins: GetVal(allStats, "ebitda margin", "ebitda margins"),
PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"), OperatingMargins: GetVal(allStats, "operating margin", "operating margin (ttm)", "operating margins"),
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"), ProfitMargins: GetVal(allStats, "profit margin", "profit margins"),
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"), FinancialCurrency: null
FiftyTwoWeekChange: GetVal(allStats, "52-week change"),
SandP52WeekChange: GetVal(allStats, "s&p500 52-week change")
); );
var financialData = new YahooFinancialDataDto( var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
CurrentPrice: GetVal(allStats, "current price", "price"), PriceToBook: GetVal(allStats, "price/book", "price to book", "kbv", "kurs-buchwert-verhältnis"),
TargetHighPrice: GetVal(allStats, "target high", "high target"), EnterpriseValue: GetVal(allStats, "enterprise value", "unternehmenswert"),
TargetLowPrice: GetVal(allStats, "target low", "low target"), ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
TargetMeanPrice: GetVal(allStats, "target mean", "target est"), ProfitMargins: GetVal(allStats, "profit margin", "gewinnmarge"),
TargetMedianPrice: GetVal(allStats, "target median"), FloatShares: GetVal(allStats, "float", "streubesitz"),
RecommendationMean: GetVal(allStats, "recommendation mean"), SharesOutstanding: GetVal(allStats, "shares outstanding", "ausstehende aktien"),
RecommendationKey: GetString(allStats, "recommendation key"), SharesShort: GetVal(allStats, "shares short", "leerverkaufte aktien"),
NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"), SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
TotalCash: GetVal(allStats, "total cash"), SharesShortPreviousMonthDate: null,
TotalCashPerShare: GetVal(allStats, "total cash per share"), DateShortInterest: null,
Ebitda: GetVal(allStats, "ebitda"), SharesPercentSharesOut: GetVal(allStats, "shares % of shares outstanding", "short % of shares outstanding"),
TotalDebt: GetVal(allStats, "total debt"), HeldPercentInsiders: GetVal(allStats, "% held by insiders", "insider anteil"),
QuickRatio: GetVal(allStats, "quick ratio"), HeldPercentInstitutions: GetVal(allStats, "% held by institutions", "institutioneller anteil"),
CurrentRatio: GetVal(allStats, "current ratio"), ShortRatio: GetVal(allStats, "short ratio"),
TotalRevenue: GetVal(allStats, "revenue", "total revenue"), ShortPercentOfFloat: GetVal(allStats, "short % of float", "short percent of float"),
DebtToEquity: GetVal(allStats, "total debt/equity"), Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
RevenuePerShare: GetVal(allStats, "revenue per share"), Category: null,
ReturnOnAssets: GetVal(allStats, "return on assets"), BookValue: GetVal(allStats, "book value per share", "book value per share (mrq)", "book value", "buchwert"),
ReturnOnEquity: GetVal(allStats, "return on equity"), PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price to sales", "kuv"),
GrossProfits: GetVal(allStats, "gross profit"), LastFiscalYearEnd: null,
FreeCashflow: GetVal(allStats, "levered free cash flow"), NextFiscalYearEnd: null,
OperatingCashflow: GetVal(allStats, "operating cash flow"), MostRecentQuarter: null,
RevenueGrowth: GetVal(allStats, "quarterly revenue growth"), EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth", "quarterly earnings growth (yoy)", "earnings growth"),
GrossMargins: GetVal(allStats, "gross margin"), NetIncomeToCommon: GetVal(allStats, "net income avi to common", "net income avi to common (ttm)", "net income avail. to common", "net income"),
EbitdaMargins: GetVal(allStats, "ebitda margin"), TrailingEps: GetVal(allStats, "diluted eps", "diluted eps (ttm)", "trailing eps", "eps (ttm)", "gewinn je aktie"),
OperatingMargins: GetVal(allStats, "operating margin"), ForwardEps: GetVal(allStats, "forward eps"),
ProfitMargins: GetVal(allStats, "profit margin"), PegRatio: GetVal(allStats, "peg ratio (5 yr expected)", "peg ratio (5yr expected)", "peg ratio", "peg-verhältnis"),
FinancialCurrency: "USD" EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue", "ev/revenue"),
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda", "ev/ebitda"),
FiftyTwoWeekChange: GetVal(allStats, "52 week change", "52-week change", "52-wochen-änderung"),
SandP52WeekChange: GetVal(allStats, "s&p 500 52-week change", "s&p500 52-week change", "s&p 500 52 week change")
); );
var summaryDetail = new YahooSummaryDetailDto( var summaryDetail = new YahooSummaryDetailDto(
MaxAge: 86400, PriceHint: null, MaxAge: null,
PreviousClose: GetVal(allStats, "previous close"), PriceHint: null,
Open: GetVal(allStats, "open"), PreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
DayLow: GetVal(allStats, "day low"), Open: GetVal(allStats, "open", "eröffnung"),
DayHigh: GetVal(allStats, "day high"), DayLow: null,
RegularMarketPreviousClose: GetVal(allStats, "previous close"), DayHigh: null,
RegularMarketOpen: GetVal(allStats, "open"), RegularMarketPreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
RegularMarketDayLow: GetVal(allStats, "day low"), RegularMarketOpen: GetVal(allStats, "open", "eröffnung"),
RegularMarketDayHigh: GetVal(allStats, "day high"), RegularMarketDayLow: null,
DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"), RegularMarketDayHigh: null,
DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"), DividendRate: GetVal(allStats, "forward annual dividend rate", "trailing annual dividend rate", "forward dividend & yield", "dividend rate", "dividende"),
ExDividendDate: GetVal(allStats, "ex-dividend date"), DividendYield: GetVal(allStats, "forward annual dividend yield", "trailing annual dividend yield", "dividend yield", "dividendenrendite"),
PayoutRatio: GetVal(allStats, "payout ratio"), ExDividendDate: null,
FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"), PayoutRatio: GetVal(allStats, "payout ratio", "ausschüttungsquote"),
Beta: GetVal(allStats, "beta"), FiveYearAvgDividendYield: GetVal(allStats, "5 year average dividend yield"),
TrailingPE: GetVal(allStats, "trailing p/e"), Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
ForwardPE: GetVal(allStats, "forward p/e"), TrailingPE: GetVal(allStats, "pe ratio (ttm)", "trailing p/e", "p/e ratio", "pe", "trailing pe", "kgv (ttm)", "kgv"),
Volume: GetVal(allStats, "volume"), ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
RegularMarketVolume: GetVal(allStats, "volume"), Volume: GetVal(allStats, "volume", "volumen"),
AverageVolume: GetVal(allStats, "avg. volume", "average volume"), RegularMarketVolume: GetVal(allStats, "volume", "volumen"),
AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"), AverageVolume: GetVal(allStats, "avg vol (3 month)", "avg. volume", "average volume", "durchschnittsvolumen"),
AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"), AverageVolume10days: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"), AverageDailyVolume10Day: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
BidSize: null, AskSize: null, Bid: GetVal(allStats, "bid", "geld"),
MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"), Ask: GetVal(allStats, "ask", "brief"),
FiftyTwoWeekLow: GetVal(allStats, "52 week low"), BidSize: null,
FiftyTwoWeekHigh: GetVal(allStats, "52 week high"), AskSize: null,
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"), MarketCap: GetVal(allStats, "market cap", "market capitalization", "market cap (intraday)", "marktkapitalisierung"),
Currency: "USD" FiftyTwoWeekLow: GetVal(allStats, "52 week low", "52-week low", "52 wochen tief"),
FiftyTwoWeekHigh: GetVal(allStats, "52 week high", "52-week high", "52 wochen hoch"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "kuv"),
Currency: null
); );
return new YahooQuoteSummaryModulesDto( return new YahooQuoteSummaryModulesDto(
@@ -420,66 +571,59 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
); );
} }
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] keys) private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] candidateKeys)
{ {
foreach (var k in keys) foreach (var candidate in candidateKeys)
{ {
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val)) if (dict.TryGetValue(candidate, out var valStr) && !string.IsNullOrWhiteSpace(valStr))
return ParseYahooValue(val); {
var parsed = ParseYahooValue(valStr);
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase)); if (parsed != null) return parsed;
if (!string.IsNullOrWhiteSpace(match.Value)) }
return ParseYahooValue(match.Value);
} }
return null; return null;
} }
private static string? GetString(Dictionary<string, string> dict, params string[] keys) private static YahooValueDto? ParseYahooValue(string? raw)
{ {
foreach (var k in keys) if (string.IsNullOrWhiteSpace(raw) || raw == "N/A" || raw == "--" || raw == "-") return null;
{
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
return val.Trim();
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase)); var clean = raw.Trim().Replace(" ", "").Replace("$", "").Replace("€", "").Replace("£", "");
if (!string.IsNullOrWhiteSpace(match.Value)) var isPercent = clean.EndsWith("%");
return match.Value.Trim(); if (isPercent) clean = clean.TrimEnd('%');
}
return null;
}
/// <summary>
/// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln.
/// </summary>
public static YahooValueDto? ParseYahooValue(string? text)
{
if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-")
return null;
var trimmed = text.Trim();
bool isPercent = trimmed.EndsWith("%");
double multiplier = 1.0; double multiplier = 1.0;
if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0; if (clean.EndsWith("T", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000_000.0; clean = clean[..^1]; }
else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0; else if (clean.EndsWith("B", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000.0; clean = clean[..^1]; }
else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0; else if (clean.EndsWith("M", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000.0; clean = clean[..^1]; }
else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0; else if (clean.EndsWith("K", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000.0; clean = clean[..^1]; }
// Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt if (double.TryParse(clean, NumberStyles.Any, CultureInfo.InvariantCulture, out var num))
var numPart = Regex.Replace(trimmed, @"[^\d.-]", "");
if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal))
{ {
double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier); var finalRaw = num * multiplier;
return new YahooValueDto if (isPercent) finalRaw /= 100.0;
{ return new YahooValueDto { Raw = finalRaw, Fmt = raw };
Raw = finalVal,
Fmt = trimmed,
LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture)
};
} }
return null; return new YahooValueDto { Raw = null, Fmt = raw };
}
private class ProfileMetaJsResult
{
public string? Sector { get; set; }
public string? Industry { get; set; }
public int? Employees { get; set; }
public string? Description { get; set; }
public List<JsOfficer>? Officers { get; set; }
}
private class JsOfficer
{
public string? Name { get; set; }
public string? Title { get; set; }
public string? Pay { get; set; }
public string? Exercised { get; set; }
public int? YearBorn { get; set; }
} }
private record ProfileExtractionResult( private record ProfileExtractionResult(
@@ -488,24 +632,5 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
string? Sector, string? Sector,
string? Industry, string? Industry,
int? Employees, int? Employees,
string? Description string? Description);
);
private class ProfileMetaJsResult
{
public string? Sector { get; set; }
public string? Industry { get; set; }
public int? Employees { get; set; }
public string? Description { get; set; }
public List<OfficerJsResult>? Officers { get; set; }
}
private class OfficerJsResult
{
public string? Name { get; set; }
public string? Title { get; set; }
public string? Pay { get; set; }
public string? Exercised { get; set; }
public int? YearBorn { get; set; }
}
} }
@@ -1,4 +1,4 @@
using System.Text.Json.Serialization; using System.Text.Json.Serialization;
namespace FinlyticCore.Dtos.Fundamentals; namespace FinlyticCore.Dtos.Fundamentals;
@@ -18,7 +18,7 @@ public record AssetHeaderDto
public string Description { get; init; } = string.Empty; public string Description { get; init; } = string.Empty;
[JsonPropertyName("primaryTicker")] [JsonPropertyName("primaryTicker")]
public TickerInfoDto PrimaryTicker { get; init; } public TickerInfoDto PrimaryTicker { get; init; } = new();
[JsonPropertyName("availableTickers")] [JsonPropertyName("availableTickers")]
public List<TickerInfoDto> AvailableTickers { get; init; } = []; public List<TickerInfoDto> AvailableTickers { get; init; } = [];
@@ -16,7 +16,7 @@ public record CorporateEventDto
public string? Isin { get; init; } public string? Isin { get; init; }
[JsonPropertyName("ticker")] [JsonPropertyName("ticker")]
public TickerInfoDto Ticker { get; init; } public TickerInfoDto Ticker { get; init; } = new();
[JsonPropertyName("companyName")] [JsonPropertyName("companyName")]
public string? CompanyName { get; init; } public string? CompanyName { get; init; }
@@ -9,7 +9,7 @@ namespace FinlyticCore.Dtos.Fundamentals;
public record FundamentalDataDto public record FundamentalDataDto
{ {
[JsonPropertyName("ticker")] [JsonPropertyName("ticker")]
public TickerInfoDto Ticker { get; init; } public TickerInfoDto Ticker { get; init; } = new();
// --- Valuation & Multiples --- // --- Valuation & Multiples ---
[JsonPropertyName("marketCap")] [JsonPropertyName("marketCap")]
@@ -32,6 +32,7 @@ public interface IYahooFinanceScraper
Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync( Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin, string symbolOrIsin,
bool forceHtmlScrape = false, bool forceHtmlScrape = false,
bool includeProfile = true,
CancellationToken cancellationToken = default); CancellationToken cancellationToken = default);
} }
@@ -98,7 +99,11 @@ public class YahooFinanceScraper : IYahooFinanceScraper
} }
} }
} }
catch { } catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] Crypto search failed for {cryptoSubtitle}");
}
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel, await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel,
$"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}"); $"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}");
@@ -131,7 +136,7 @@ public class YahooFinanceScraper : IYahooFinanceScraper
} }
} }
// 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind // 2. Falls Ticker gefunden, mit Unternehmensname noch mehr internationale Exchangeticker suchen (z.B. APC.DE)
if (validQuotes.Count > 0) if (validQuotes.Count > 0)
{ {
var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName; var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName;
@@ -155,16 +160,24 @@ public class YahooFinanceScraper : IYahooFinanceScraper
$"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'"); $"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'");
} }
return symbols var result = symbols
.OrderBy(s => s.priority) .OrderBy(s => s.priority)
.Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange }) .Select(s => new TickerInfoDto
{
Ticker = s.symbol,
Exchange = !string.IsNullOrWhiteSpace(s.exchange) ? s.exchange : "Unknown"
})
.DistinctBy(s => s.Ticker, StringComparer.OrdinalIgnoreCase)
.ToList(); .ToList();
return result;
} }
/// <inheritdoc /> /// <inheritdoc />
public async Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync( public async Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin, string symbolOrIsin,
bool forceHtmlScrape = false, bool forceHtmlScrape = false,
bool includeProfile = true,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null; if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null;
@@ -220,9 +233,9 @@ public class YahooFinanceScraper : IYahooFinanceScraper
try try
{ {
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}'..."); $"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}' (IncludeProfile: {includeProfile})...");
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken); htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, includeProfile, cancellationToken);
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -246,16 +259,20 @@ public class YahooFinanceScraper : IYahooFinanceScraper
modules.DefaultKeyStatistics != null; modules.DefaultKeyStatistics != null;
} }
private static YahooQuoteSummaryModulesDto MergeModules( public static YahooQuoteSummaryModulesDto? MergeModules(
YahooQuoteSummaryModulesDto primary, YahooQuoteSummaryModulesDto? primary,
YahooQuoteSummaryModulesDto secondary) YahooQuoteSummaryModulesDto? secondary)
{ {
if (primary == null && secondary == null) return null;
if (primary == null) return secondary;
if (secondary == null) return primary;
return new YahooQuoteSummaryModulesDto( return new YahooQuoteSummaryModulesDto(
QuoteType: primary.QuoteType ?? secondary.QuoteType, QuoteType: primary.QuoteType ?? secondary.QuoteType,
AssetProfile: primary.AssetProfile ?? secondary.AssetProfile, AssetProfile: primary.AssetProfile ?? secondary.AssetProfile,
FinancialData: primary.FinancialData ?? secondary.FinancialData, FinancialData: MergeFinancialData(primary.FinancialData, secondary.FinancialData),
DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics, DefaultKeyStatistics: MergeDefaultKeyStatistics(primary.DefaultKeyStatistics, secondary.DefaultKeyStatistics),
SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail, SummaryDetail: MergeSummaryDetail(primary.SummaryDetail, secondary.SummaryDetail),
IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory, IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory,
IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly, IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly,
BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory, BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory,
@@ -266,6 +283,124 @@ public class YahooFinanceScraper : IYahooFinanceScraper
); );
} }
private static YahooFinancialDataDto? MergeFinancialData(YahooFinancialDataDto? a, YahooFinancialDataDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooFinancialDataDto(
CurrentPrice: a.CurrentPrice ?? b.CurrentPrice,
TargetHighPrice: a.TargetHighPrice ?? b.TargetHighPrice,
TargetLowPrice: a.TargetLowPrice ?? b.TargetLowPrice,
TargetMeanPrice: a.TargetMeanPrice ?? b.TargetMeanPrice,
TargetMedianPrice: a.TargetMedianPrice ?? b.TargetMedianPrice,
RecommendationMean: a.RecommendationMean ?? b.RecommendationMean,
RecommendationKey: !string.IsNullOrWhiteSpace(a.RecommendationKey) && a.RecommendationKey != "none" ? a.RecommendationKey : b.RecommendationKey,
NumberOfAnalystOpinions: a.NumberOfAnalystOpinions ?? b.NumberOfAnalystOpinions,
TotalCash: a.TotalCash ?? b.TotalCash,
TotalCashPerShare: a.TotalCashPerShare ?? b.TotalCashPerShare,
Ebitda: a.Ebitda ?? b.Ebitda,
TotalDebt: a.TotalDebt ?? b.TotalDebt,
QuickRatio: a.QuickRatio ?? b.QuickRatio,
CurrentRatio: a.CurrentRatio ?? b.CurrentRatio,
TotalRevenue: a.TotalRevenue ?? b.TotalRevenue,
DebtToEquity: a.DebtToEquity ?? b.DebtToEquity,
RevenuePerShare: a.RevenuePerShare ?? b.RevenuePerShare,
ReturnOnAssets: a.ReturnOnAssets ?? b.ReturnOnAssets,
ReturnOnEquity: a.ReturnOnEquity ?? b.ReturnOnEquity,
GrossProfits: a.GrossProfits ?? b.GrossProfits,
FreeCashflow: a.FreeCashflow ?? b.FreeCashflow,
OperatingCashflow: a.OperatingCashflow ?? b.OperatingCashflow,
RevenueGrowth: a.RevenueGrowth ?? b.RevenueGrowth,
GrossMargins: a.GrossMargins ?? b.GrossMargins,
EbitdaMargins: a.EbitdaMargins ?? b.EbitdaMargins,
OperatingMargins: a.OperatingMargins ?? b.OperatingMargins,
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
FinancialCurrency: a.FinancialCurrency ?? b.FinancialCurrency
);
}
private static YahooDefaultKeyStatisticsDto? MergeDefaultKeyStatistics(YahooDefaultKeyStatisticsDto? a, YahooDefaultKeyStatisticsDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooDefaultKeyStatisticsDto(
PriceToBook: a.PriceToBook ?? b.PriceToBook,
EnterpriseValue: a.EnterpriseValue ?? b.EnterpriseValue,
ForwardPE: a.ForwardPE ?? b.ForwardPE,
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
FloatShares: a.FloatShares ?? b.FloatShares,
SharesOutstanding: a.SharesOutstanding ?? b.SharesOutstanding,
SharesShort: a.SharesShort ?? b.SharesShort,
SharesShortPriorMonth: a.SharesShortPriorMonth ?? b.SharesShortPriorMonth,
SharesShortPreviousMonthDate: a.SharesShortPreviousMonthDate ?? b.SharesShortPreviousMonthDate,
DateShortInterest: a.DateShortInterest ?? b.DateShortInterest,
SharesPercentSharesOut: a.SharesPercentSharesOut ?? b.SharesPercentSharesOut,
HeldPercentInsiders: a.HeldPercentInsiders ?? b.HeldPercentInsiders,
HeldPercentInstitutions: a.HeldPercentInstitutions ?? b.HeldPercentInstitutions,
ShortRatio: a.ShortRatio ?? b.ShortRatio,
ShortPercentOfFloat: a.ShortPercentOfFloat ?? b.ShortPercentOfFloat,
Beta: a.Beta ?? b.Beta,
Category: a.Category ?? b.Category,
BookValue: a.BookValue ?? b.BookValue,
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
LastFiscalYearEnd: a.LastFiscalYearEnd ?? b.LastFiscalYearEnd,
NextFiscalYearEnd: a.NextFiscalYearEnd ?? b.NextFiscalYearEnd,
MostRecentQuarter: a.MostRecentQuarter ?? b.MostRecentQuarter,
EarningsQuarterlyGrowth: a.EarningsQuarterlyGrowth ?? b.EarningsQuarterlyGrowth,
NetIncomeToCommon: a.NetIncomeToCommon ?? b.NetIncomeToCommon,
TrailingEps: a.TrailingEps ?? b.TrailingEps,
ForwardEps: a.ForwardEps ?? b.ForwardEps,
PegRatio: a.PegRatio ?? b.PegRatio,
EnterpriseToRevenue: a.EnterpriseToRevenue ?? b.EnterpriseToRevenue,
EnterpriseToEbitda: a.EnterpriseToEbitda ?? b.EnterpriseToEbitda,
FiftyTwoWeekChange: a.FiftyTwoWeekChange ?? b.FiftyTwoWeekChange,
SandP52WeekChange: a.SandP52WeekChange ?? b.SandP52WeekChange
);
}
private static YahooSummaryDetailDto? MergeSummaryDetail(YahooSummaryDetailDto? a, YahooSummaryDetailDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooSummaryDetailDto(
MaxAge: a.MaxAge ?? b.MaxAge,
PriceHint: a.PriceHint ?? b.PriceHint,
PreviousClose: a.PreviousClose ?? b.PreviousClose,
Open: a.Open ?? b.Open,
DayLow: a.DayLow ?? b.DayLow,
DayHigh: a.DayHigh ?? b.DayHigh,
RegularMarketPreviousClose: a.RegularMarketPreviousClose ?? b.RegularMarketPreviousClose,
RegularMarketOpen: a.RegularMarketOpen ?? b.RegularMarketOpen,
RegularMarketDayLow: a.RegularMarketDayLow ?? b.RegularMarketDayLow,
RegularMarketDayHigh: a.RegularMarketDayHigh ?? b.RegularMarketDayHigh,
DividendRate: a.DividendRate ?? b.DividendRate,
DividendYield: a.DividendYield ?? b.DividendYield,
ExDividendDate: a.ExDividendDate ?? b.ExDividendDate,
PayoutRatio: a.PayoutRatio ?? b.PayoutRatio,
FiveYearAvgDividendYield: a.FiveYearAvgDividendYield ?? b.FiveYearAvgDividendYield,
Beta: a.Beta ?? b.Beta,
TrailingPE: a.TrailingPE ?? b.TrailingPE,
ForwardPE: a.ForwardPE ?? b.ForwardPE,
Volume: a.Volume ?? b.Volume,
RegularMarketVolume: a.RegularMarketVolume ?? b.RegularMarketVolume,
AverageVolume: a.AverageVolume ?? b.AverageVolume,
AverageVolume10days: a.AverageVolume10days ?? b.AverageVolume10days,
AverageDailyVolume10Day: a.AverageDailyVolume10Day ?? b.AverageDailyVolume10Day,
Bid: a.Bid ?? b.Bid,
Ask: a.Ask ?? b.Ask,
BidSize: a.BidSize ?? b.BidSize,
AskSize: a.AskSize ?? b.AskSize,
MarketCap: a.MarketCap ?? b.MarketCap,
FiftyTwoWeekLow: a.FiftyTwoWeekLow ?? b.FiftyTwoWeekLow,
FiftyTwoWeekHigh: a.FiftyTwoWeekHigh ?? b.FiftyTwoWeekHigh,
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
Currency: a.Currency ?? b.Currency
);
}
private static bool IsIsin(string value) private static bool IsIsin(string value)
{ {
return value.Length == 12 && return value.Length == 12 &&
@@ -67,13 +67,16 @@ public class FundamentalsDbContext : DbContext, ISettingsDbContext
modelBuilder.Entity<FundamentalDataEntity>(entity => modelBuilder.Entity<FundamentalDataEntity>(entity =>
{ {
entity.HasKey(e => e.Isin); entity.HasKey(e => e.Id);
entity.OwnsOne(e => e.Ticker, t => entity.OwnsOne(e => e.Ticker, t =>
{ {
t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty); t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty);
t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty); t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty);
t.HasIndex(p => p.Ticker);
}); });
entity.HasIndex(e => new { e.AssetDataIsin });
}); });
modelBuilder.Entity<KeyExecutiveEntity>(entity => modelBuilder.Entity<KeyExecutiveEntity>(entity =>
@@ -6,7 +6,7 @@ namespace FinlyticFundamentals.Entities;
public class FundamentalDataEntity public class FundamentalDataEntity
{ {
[Key] [Key]
public string Isin { get; set; } = string.Empty; public Guid Id { get; set; } = Guid.NewGuid();
public TickerEntity Ticker { get; set; } = new(); public TickerEntity Ticker { get; set; } = new();
@@ -0,0 +1,433 @@
// <auto-generated />
using System;
using FinlyticFundamentals.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticFundamentals.Migrations
{
[DbContext(typeof(FundamentalsDbContext))]
[Migration("20260816103109_MakeFundamentalDataPerTicker")]
partial class MakeFundamentalDataPerTicker
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.Property<string>("Isin")
.HasColumnType("text");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.HasKey("Isin");
b.ToTable("AssetData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("Date")
.HasColumnType("timestamp with time zone");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AssetDataIsin");
b.ToTable("AssetEvents");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ConsensusRating")
.HasColumnType("text");
b.Property<decimal?>("CurrentRatio")
.HasColumnType("numeric");
b.Property<decimal?>("DebtToEquity")
.HasColumnType("numeric");
b.Property<decimal?>("DilutedEps")
.HasColumnType("numeric");
b.Property<decimal?>("Ebitda")
.HasColumnType("numeric");
b.Property<decimal?>("EnterpriseValue")
.HasColumnType("numeric");
b.Property<decimal?>("EvToEbitda")
.HasColumnType("numeric");
b.Property<decimal?>("FiftyTwoWeekHigh")
.HasColumnType("numeric");
b.Property<decimal?>("FiftyTwoWeekLow")
.HasColumnType("numeric");
b.Property<decimal?>("ForwardDividendYield")
.HasColumnType("numeric");
b.Property<decimal?>("ForwardPe")
.HasColumnType("numeric");
b.Property<decimal?>("FreeCashFlow")
.HasColumnType("numeric");
b.Property<decimal?>("GrossProfit")
.HasColumnType("numeric");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<decimal?>("MarketCap")
.HasColumnType("numeric");
b.Property<decimal?>("NetIncome")
.HasColumnType("numeric");
b.Property<decimal?>("OperatingCashFlow")
.HasColumnType("numeric");
b.Property<decimal?>("OperatingIncome")
.HasColumnType("numeric");
b.Property<decimal?>("PayoutRatio")
.HasColumnType("numeric");
b.Property<decimal?>("PegRatio")
.HasColumnType("numeric");
b.Property<decimal?>("PercentHeldByInsiders")
.HasColumnType("numeric");
b.Property<decimal?>("PercentHeldByInstitutions")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetHigh")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetLow")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetMean")
.HasColumnType("numeric");
b.Property<decimal?>("PriceToBook")
.HasColumnType("numeric");
b.Property<decimal?>("PriceToSales")
.HasColumnType("numeric");
b.Property<decimal?>("ReturnOnAssets")
.HasColumnType("numeric");
b.Property<decimal?>("ReturnOnEquity")
.HasColumnType("numeric");
b.Property<decimal?>("RevenueGrowthYoY")
.HasColumnType("numeric");
b.Property<decimal?>("ShortPercentOfFloat")
.HasColumnType("numeric");
b.Property<decimal?>("ShortRatio")
.HasColumnType("numeric");
b.Property<decimal?>("TotalCash")
.HasColumnType("numeric");
b.Property<decimal?>("TotalDebt")
.HasColumnType("numeric");
b.Property<decimal?>("TotalRevenue")
.HasColumnType("numeric");
b.Property<decimal?>("TrailingPe")
.HasColumnType("numeric");
b.HasKey("Id");
b.HasIndex("AssetDataIsin");
b.ToTable("FundamentalData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Payment")
.IsRequired()
.HasColumnType("text");
b.Property<int>("SortOrder")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0);
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AssetDataIsin", "SortOrder");
b.ToTable("KeyExecutives");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 =>
{
b1.Property<string>("AssetDataEntityIsin")
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("PrimaryTickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("PrimaryTicker");
b1.HasKey("AssetDataEntityIsin");
b1.ToTable("AssetData");
b1.WithOwner()
.HasForeignKey("AssetDataEntityIsin");
});
b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 =>
{
b1.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b1.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.HasColumnType("text")
.HasColumnName("Exchange");
b1.Property<string>("Ticker")
.IsRequired()
.HasColumnType("text")
.HasColumnName("Ticker");
b1.HasKey("Id");
b1.HasIndex("AssetDataIsin");
b1.HasIndex("Ticker");
b1.ToTable("Tickers", (string)null);
b1.WithOwner()
.HasForeignKey("AssetDataIsin");
});
b.Navigation("AvailableTickers");
b.Navigation("PrimaryTicker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("AssetEvents")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<Guid>("AssetEventEntityId")
.HasColumnType("uuid");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("TickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("AssetEventEntityId");
b1.ToTable("AssetEvents");
b1.WithOwner()
.HasForeignKey("AssetEventEntityId");
});
b.Navigation("AssetData");
b.Navigation("Ticker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("FundamentalData")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<Guid>("FundamentalDataEntityId")
.HasColumnType("uuid");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("TickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("FundamentalDataEntityId");
b1.HasIndex("Ticker");
b1.ToTable("FundamentalData");
b1.WithOwner()
.HasForeignKey("FundamentalDataEntityId");
});
b.Navigation("AssetData");
b.Navigation("Ticker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("KeyExecutives")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("AssetData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.Navigation("AssetEvents");
b.Navigation("FundamentalData");
b.Navigation("KeyExecutives");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,68 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticFundamentals.Migrations
{
/// <inheritdoc />
public partial class MakeFundamentalDataPerTicker : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData");
migrationBuilder.DropColumn(
name: "Isin",
table: "FundamentalData");
migrationBuilder.AddColumn<Guid>(
name: "Id",
table: "FundamentalData",
type: "uuid",
nullable: false,
defaultValueSql: "gen_random_uuid()");
migrationBuilder.AddPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData",
column: "Id");
migrationBuilder.CreateIndex(
name: "IX_FundamentalData_Ticker",
table: "FundamentalData",
column: "Ticker");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData");
migrationBuilder.DropIndex(
name: "IX_FundamentalData_Ticker",
table: "FundamentalData");
migrationBuilder.DropColumn(
name: "Id",
table: "FundamentalData");
migrationBuilder.AddColumn<string>(
name: "Isin",
table: "FundamentalData",
type: "text",
nullable: false,
defaultValue: "");
migrationBuilder.AddPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData",
column: "Isin");
}
}
}
@@ -97,8 +97,9 @@ namespace FinlyticFundamentals.Migrations
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{ {
b.Property<string>("Isin") b.Property<Guid>("Id")
.HasColumnType("text"); .ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin") b.Property<string>("AssetDataIsin")
.IsRequired() .IsRequired()
@@ -212,7 +213,7 @@ namespace FinlyticFundamentals.Migrations
b.Property<decimal?>("TrailingPe") b.Property<decimal?>("TrailingPe")
.HasColumnType("numeric"); .HasColumnType("numeric");
b.HasKey("Isin"); b.HasKey("Id");
b.HasIndex("AssetDataIsin"); b.HasIndex("AssetDataIsin");
@@ -371,8 +372,8 @@ namespace FinlyticFundamentals.Migrations
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{ {
b1.Property<string>("FundamentalDataEntityIsin") b1.Property<Guid>("FundamentalDataEntityId")
.HasColumnType("text"); .HasColumnType("uuid");
b1.Property<string>("Exchange") b1.Property<string>("Exchange")
.IsRequired() .IsRequired()
@@ -388,12 +389,14 @@ namespace FinlyticFundamentals.Migrations
.HasDefaultValue("") .HasDefaultValue("")
.HasColumnName("Ticker"); .HasColumnName("Ticker");
b1.HasKey("FundamentalDataEntityIsin"); b1.HasKey("FundamentalDataEntityId");
b1.HasIndex("Ticker");
b1.ToTable("FundamentalData"); b1.ToTable("FundamentalData");
b1.WithOwner() b1.WithOwner()
.HasForeignKey("FundamentalDataEntityIsin"); .HasForeignKey("FundamentalDataEntityId");
}); });
b.Navigation("AssetData"); b.Navigation("AssetData");
+2 -2
View File
@@ -31,8 +31,8 @@ builder.Services.AddHttpClient<IYahooFinanceScraper, YahooFinanceScraper>()
AllowAutoRedirect = true AllowAutoRedirect = true
}); });
builder.Services.AddScoped<IPlaywrightBrowserFactory, PlaywrightBrowserFactory>(); builder.Services.AddSingleton<IPlaywrightBrowserFactory, PlaywrightBrowserFactory>();
builder.Services.AddScoped<IPlaywrightExecutionService, PlaywrightExecutionService>(); builder.Services.AddSingleton<IPlaywrightExecutionService, PlaywrightExecutionService>();
builder.Services.AddTransient<IYahooFinanceHtmlClient, YahooFinanceHtmlClient>(); builder.Services.AddTransient<IYahooFinanceHtmlClient, YahooFinanceHtmlClient>();
// Register Application Services // Register Application Services
@@ -79,44 +79,53 @@ public class FundamentalsDbService : IFundamentalsDbService
await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken); await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken);
bool enableHtmlFallback = bool enableHtmlFallback =
await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken); await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken);
bool forceHtmlFallback =
await settingsService.GetSettingAsync(SettingKeys.ForceHtmlFallback, cancellationToken);
int validityDays = int validityDays =
await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken); await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken);
bool effectiveForceRefresh = forceRefresh && allowForceRefresh; bool effectiveForceRefresh = forceRefresh && allowForceRefresh;
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtmlFallback: {Html}", "[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtml: {Html} | ForceHtml: {ForceHtml}",
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback); cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback, forceHtmlFallback);
// 2. Entitäten aus DB laden // 2. Entitäten aus DB laden
var assetData = await context.AssetData var assetData = await context.AssetData
.Include(a => a.AvailableTickers) .Include(a => a.AvailableTickers)
.Include(a => a.KeyExecutives) .Include(a => a.KeyExecutives)
.Include(a => a.AssetEvents) .Include(a => a.AssetEvents)
.Include(a => a.FundamentalData)
.FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken); .FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken);
var fundamentalData = await context.FundamentalData string targetTicker = !string.IsNullOrWhiteSpace(requestedTicker)
.FirstOrDefaultAsync(f => f.Isin == cleanIsin, cancellationToken); ? requestedTicker
: (assetData?.PrimaryTicker?.Ticker ?? string.Empty);
var fundamentalData = assetData?.FundamentalData?
.FirstOrDefault(f => !string.IsNullOrWhiteSpace(targetTicker) && string.Equals(f.Ticker.Ticker, targetTicker, StringComparison.OrdinalIgnoreCase))
?? (string.IsNullOrWhiteSpace(requestedTicker) ? assetData?.FundamentalData?.FirstOrDefault() : null);
// 3. Prüfen, was aktualisiert werden muss // 3. Prüfen, was aktualisiert werden muss
bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name); bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name);
bool executivesMissing = assetData == null || assetData.KeyExecutives == null || bool executivesMissing = assetData == null || assetData.KeyExecutives == null ||
assetData.KeyExecutives.Count == 0; assetData.KeyExecutives.Count == 0;
bool fundamentalsExpired = fundamentalData == null || bool fundamentalsMissingOrExpired = fundamentalData == null ||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays; (fundamentalData.MarketCap == null && fundamentalData.TrailingPe == null) ||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
bool tickersCorruptOrMissing = assetData?.AvailableTickers == null ||
assetData.AvailableTickers.Count == 0 ||
assetData.AvailableTickers.Any(t => t.Ticker != null && t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase));
// Wenn ein expliziter Ticker übergeben wurde und sich vom gespeicherten unterscheidet, bool shouldUpdate = assetDataMissing || executivesMissing || fundamentalsMissingOrExpired || tickersCorruptOrMissing || effectiveForceRefresh || forceHtmlFallback;
// müssen Asset-Daten und Fundamentals mit dem neuen Ticker neu abgerufen werden.
bool tickerChanged = !string.IsNullOrWhiteSpace(requestedTicker)
&& assetData?.PrimaryTicker != null
&& !string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker,
StringComparison.OrdinalIgnoreCase);
bool shouldUpdateAssetData = assetDataMissing || effectiveForceRefresh || tickerChanged; if (!shouldUpdate && fundamentalData != null)
bool shouldUpdateExecutives = executivesMissing || effectiveForceRefresh; {
bool shouldUpdateFundamentals = fundamentalsExpired || effectiveForceRefresh || tickerChanged; await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[FundamentalsDbService] Returning valid cached fundamental data for ISIN {Isin} (Ticker: {Ticker}, Updated: {UpdatedUtc}). External API fetch skipped.",
if (shouldUpdateAssetData || shouldUpdateExecutives || shouldUpdateFundamentals) cleanIsin, fundamentalData.Ticker.Ticker, fundamentalData.LastUpdatedUtc.ToString("o"));
}
else
{ {
// --- STEP 1: Trade Republic Details --- // --- STEP 1: Trade Republic Details ---
TradeRepublicStockDetailsResponse? trDetails = null; TradeRepublicStockDetailsResponse? trDetails = null;
@@ -150,8 +159,10 @@ public class FundamentalsDbService : IFundamentalsDbService
TickerInfoDto activeQueryTicker; TickerInfoDto activeQueryTicker;
if (!string.IsNullOrWhiteSpace(requestedTicker)) if (!string.IsNullOrWhiteSpace(requestedTicker))
{ {
var matchDto = resolvedTickers.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); var matchDto = resolvedTickers.FirstOrDefault(t =>
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(t =>
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
if (matchDto != null) if (matchDto != null)
{ {
@@ -197,12 +208,18 @@ public class FundamentalsDbService : IFundamentalsDbService
yahooPrimaryTicker.Ticker, activeQueryTicker.Ticker, cleanIsin); yahooPrimaryTicker.Ticker, activeQueryTicker.Ticker, cleanIsin);
// --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper --- // --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper ---
// Profile (Sektor, Industrie, Vorstände) wird gescrapt, wenn weder in DB noch in TR Vorstände/Beschreibungen vorliegen
bool hasProfileInDb = assetData != null && !string.IsNullOrWhiteSpace(assetData.Description) && assetData.KeyExecutives != null && assetData.KeyExecutives.Count > 0;
bool hasCeoInTr = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
bool needProfile = !hasProfileInDb && !hasCeoInTr;
YahooQuoteSummaryModulesDto? modulesDto = null; YahooQuoteSummaryModulesDto? modulesDto = null;
if (!string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) && activeQueryTicker.Ticker != cleanIsin) if (!string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) && activeQueryTicker.Ticker != cleanIsin)
{ {
modulesDto = await _scraper.GetQuoteSummaryModulesAsync( modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
activeQueryTicker.Ticker, activeQueryTicker.Ticker,
forceHtmlScrape: false, forceHtmlScrape: forceHtmlFallback,
includeProfile: needProfile,
cancellationToken: cancellationToken); cancellationToken: cancellationToken);
} }
else else
@@ -211,51 +228,96 @@ public class FundamentalsDbService : IFundamentalsDbService
"[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", activeQueryTicker.Ticker); "[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", activeQueryTicker.Ticker);
} }
// --- Update AssetDataEntity --- // Falls der Sekundär-Ticker (z. B. APC.DE) überhaupt keine Daten liefert, nutze den PrimaryTicker (z. B. AAPL) als Fallback
if (shouldUpdateAssetData) if (modulesDto == null &&
!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) &&
yahooPrimaryTicker.Ticker != activeQueryTicker.Ticker &&
yahooPrimaryTicker.Ticker != cleanIsin)
{ {
if (assetData == null) await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-FALLBACK-PRIMARY] Sekundär-Ticker '{Active}' lieferte keine Daten. Versuche PrimaryTicker '{Primary}'...",
activeQueryTicker.Ticker, yahooPrimaryTicker.Ticker);
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
yahooPrimaryTicker.Ticker,
forceHtmlScrape: forceHtmlFallback,
includeProfile: needProfile,
cancellationToken: cancellationToken);
}
// --- Update AssetDataEntity ---
if (assetData == null)
{
assetData = new AssetDataEntity
{ {
assetData = new AssetDataEntity Isin = cleanIsin,
PrimaryTicker = new TickerEntity
{ {
Isin = cleanIsin, Ticker = yahooPrimaryTicker.Ticker,
PrimaryTicker = new TickerEntity Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
{ },
Ticker = yahooPrimaryTicker.Ticker, KeyExecutives = new List<KeyExecutiveEntity>(),
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown" AssetEvents = new List<AssetEventEntity>()
}, };
KeyExecutives = new List<KeyExecutiveEntity>(), context.AssetData.Add(assetData);
AssetEvents = new List<AssetEventEntity>() }
};
context.AssetData.Add(assetData);
}
string trName = trDetails?.Company?.Name ?? string.Empty; string trName = trDetails?.Company?.Name?.Trim() ?? string.Empty;
string trDescription = trDetails?.Company?.Description ?? string.Empty; string trDescription = trDetails?.Company?.Description?.Trim() ?? string.Empty;
string fallbackName = modulesDto?.QuoteType?.ShortName string yahooName = modulesDto?.QuoteType?.LongName?.Trim()
?? modulesDto?.QuoteType?.LongName ?? modulesDto?.QuoteType?.ShortName?.Trim()
?? activeQueryTicker.Ticker; ?? string.Empty;
string yahooDesc = modulesDto?.AssetProfile?.LongBusinessSummary?.Trim() ?? string.Empty;
assetData.Name = !string.IsNullOrWhiteSpace(trName) ? trName : fallbackName; // Name nur aktualisieren, wenn ein echter Name vorliegt (Bestandsdaten niemals mit ISIN/Ticker überschreiben)
assetData.Description = !string.IsNullOrWhiteSpace(trDescription) if (!string.IsNullOrWhiteSpace(trName))
? trDescription {
: (modulesDto?.AssetProfile?.LongBusinessSummary ?? string.Empty); assetData.Name = trName;
}
else if (!string.IsNullOrWhiteSpace(yahooName))
{
assetData.Name = yahooName;
}
else if (string.IsNullOrWhiteSpace(assetData.Name))
{
assetData.Name = !string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) ? activeQueryTicker.Ticker : cleanIsin;
}
// PrimaryTicker ist FEST der erste von Yahoo Finance // Description nur aktualisieren, wenn neue Beschreibung vorhanden ist
if (!string.IsNullOrWhiteSpace(trDescription))
{
assetData.Description = trDescription;
}
else if (!string.IsNullOrWhiteSpace(yahooDesc))
{
assetData.Description = yahooDesc;
}
// PrimaryTicker aktualisieren falls vorhanden
if (!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) && yahooPrimaryTicker.Ticker != cleanIsin)
{
assetData.PrimaryTicker = new TickerEntity assetData.PrimaryTicker = new TickerEntity
{ {
Ticker = yahooPrimaryTicker.Ticker, Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown" Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
}; };
}
if (!resolvedTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase))) // AvailableTickers aktualisieren (nur echte Börsenticker, keine ISINs)
var validTickers = resolvedTickers
.Where(t => !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase))
.ToList();
if (validTickers.Count > 0)
{
if (!validTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
{ {
resolvedTickers.Insert(0, yahooPrimaryTicker); validTickers.Insert(0, yahooPrimaryTicker);
} }
assetData.AvailableTickers.Clear(); assetData.AvailableTickers.Clear();
foreach (var a in resolvedTickers) foreach (var a in validTickers)
{ {
assetData.AvailableTickers.Add(new TickerEntity assetData.AvailableTickers.Add(new TickerEntity
{ {
@@ -263,21 +325,19 @@ public class FundamentalsDbService : IFundamentalsDbService
Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker) Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker)
}); });
} }
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}'",
assetData.Name, assetData.PrimaryTicker.Ticker);
} }
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}' | AvailableTickers: {Count}",
assetData.Name, assetData.PrimaryTicker?.Ticker ?? "NULL", assetData.AvailableTickers.Count);
// --- Process Trade Republic Corporate Events --- // --- Process Trade Republic Corporate Events ---
if (trDetails != null && (shouldUpdateAssetData || effectiveForceRefresh) && assetData != null) if (trDetails != null && assetData != null)
{ {
// 1. Alte Events direkt in der DB löschen (bypasses Change Tracker)
await context.AssetEvents await context.AssetEvents
.Where(e => e.AssetDataIsin == cleanIsin) .Where(e => e.AssetDataIsin == cleanIsin)
.ExecuteDeleteAsync(cancellationToken); .ExecuteDeleteAsync(cancellationToken);
// 2. ALLE tracked AssetEventEntity-Einträge aus dem Change Tracker entfernen
foreach (var entry in context.ChangeTracker.Entries<AssetEventEntity>() foreach (var entry in context.ChangeTracker.Entries<AssetEventEntity>()
.Where(e => e.Entity.AssetDataIsin == cleanIsin) .Where(e => e.Entity.AssetDataIsin == cleanIsin)
.ToList()) .ToList())
@@ -285,7 +345,6 @@ public class FundamentalsDbService : IFundamentalsDbService
entry.State = EntityState.Detached; entry.State = EntityState.Detached;
} }
// 3. Navigation-Collection zurücksetzen
assetData.AssetEvents = new List<AssetEventEntity>(); assetData.AssetEvents = new List<AssetEventEntity>();
var trEventList = new List<TradeRepublicEventDto>(); var trEventList = new List<TradeRepublicEventDto>();
@@ -324,18 +383,18 @@ public class FundamentalsDbService : IFundamentalsDbService
} }
// --- Process Modules DTO (Executives & Fundamental Data) --- // --- Process Modules DTO (Executives & Fundamental Data) ---
if (modulesDto != null) if (modulesDto != null || trDetails?.Company != null)
{ {
// Update KeyExecutives // Update KeyExecutives wenn Executives aus TR oder Yahoo vorliegen
if (shouldUpdateExecutives && assetData != null) var yahooOfficers = modulesDto?.AssetProfile?.CompanyOfficers;
bool hasTrOfficers = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
if (((yahooOfficers != null && yahooOfficers.Count > 0) || hasTrOfficers) && assetData != null)
{ {
// 1. Alte Executives direkt in der DB löschen (bypasses Change Tracker)
await context.KeyExecutives await context.KeyExecutives
.Where(e => e.AssetDataIsin == cleanIsin) .Where(e => e.AssetDataIsin == cleanIsin)
.ExecuteDeleteAsync(cancellationToken); .ExecuteDeleteAsync(cancellationToken);
// 2. ALLE tracked KeyExecutiveEntity-Einträge aus dem Change Tracker entfernen
// (nicht nur die in der Navigation-Collection — der Tracker kann mehr halten)
foreach (var entry in context.ChangeTracker.Entries<KeyExecutiveEntity>() foreach (var entry in context.ChangeTracker.Entries<KeyExecutiveEntity>()
.Where(e => e.Entity.AssetDataIsin == cleanIsin) .Where(e => e.Entity.AssetDataIsin == cleanIsin)
.ToList()) .ToList())
@@ -343,14 +402,12 @@ public class FundamentalsDbService : IFundamentalsDbService
entry.State = EntityState.Detached; entry.State = EntityState.Detached;
} }
// 3. Navigation-Collection zurücksetzen
assetData.KeyExecutives = new List<KeyExecutiveEntity>(); assetData.KeyExecutives = new List<KeyExecutiveEntity>();
// 4. Neue Executives aufbauen und direkt über den DbSet hinzufügen if (yahooOfficers != null && yahooOfficers.Count > 0)
if (modulesDto.AssetProfile?.CompanyOfficers != null)
{ {
int sortIdx = 0; int sortIdx = 0;
foreach (var officer in modulesDto.AssetProfile.CompanyOfficers) foreach (var officer in yahooOfficers)
{ {
if (!string.IsNullOrWhiteSpace(officer.Name)) if (!string.IsNullOrWhiteSpace(officer.Name))
{ {
@@ -368,6 +425,49 @@ public class FundamentalsDbService : IFundamentalsDbService
} }
} }
} }
else if (hasTrOfficers && trDetails?.Company != null)
{
int sortIdx = 0;
if (!string.IsNullOrWhiteSpace(trDetails.Company.CeoName))
{
var ceo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CeoName,
Title = "CEO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(ceo);
assetData.KeyExecutives.Add(ceo);
}
if (!string.IsNullOrWhiteSpace(trDetails.Company.CfoName))
{
var cfo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CfoName,
Title = "CFO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(cfo);
assetData.KeyExecutives.Add(cfo);
}
if (!string.IsNullOrWhiteSpace(trDetails.Company.CooName))
{
var coo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CooName,
Title = "COO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(coo);
assetData.KeyExecutives.Add(coo);
}
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.", "[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.",
@@ -375,16 +475,23 @@ public class FundamentalsDbService : IFundamentalsDbService
} }
// Update FundamentalDataEntity // Update FundamentalDataEntity
if (shouldUpdateFundamentals) if (modulesDto != null && (modulesDto.SummaryDetail != null || modulesDto.DefaultKeyStatistics != null || modulesDto.FinancialData != null))
{ {
if (fundamentalData == null || !string.Equals(fundamentalData.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase))
{
fundamentalData = assetData?.FundamentalData?
.FirstOrDefault(f => string.Equals(f.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase));
}
if (fundamentalData == null) if (fundamentalData == null)
{ {
fundamentalData = new FundamentalDataEntity fundamentalData = new FundamentalDataEntity
{ {
Isin = cleanIsin, Id = Guid.NewGuid(),
AssetDataIsin = cleanIsin AssetDataIsin = cleanIsin
}; };
context.FundamentalData.Add(fundamentalData); context.FundamentalData.Add(fundamentalData);
assetData?.FundamentalData.Add(fundamentalData);
} }
fundamentalData.Ticker = new TickerEntity fundamentalData.Ticker = new TickerEntity
@@ -542,7 +649,7 @@ public class FundamentalsDbService : IFundamentalsDbService
: (assetData.PrimaryTicker != null ? new List<TickerEntity> { assetData.PrimaryTicker } : new List<TickerEntity>()); : (assetData.PrimaryTicker != null ? new List<TickerEntity> { assetData.PrimaryTicker } : new List<TickerEntity>());
var tickerDtos = tickerEntities var tickerDtos = tickerEntities
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker)) .Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(assetData.Isin, StringComparison.OrdinalIgnoreCase))
.Select(a => new TickerInfoDto .Select(a => new TickerInfoDto
{ {
Ticker = a.Ticker, Ticker = a.Ticker,
+1
View File
@@ -13,6 +13,7 @@ public class SettingKeys
// --- Features & Toggles --- // --- Features & Toggles ---
public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true); public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true);
public static readonly SettingKey<bool> ForceHtmlFallback = new("Scraper.ForceHtmlFallback", false);
public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true); public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true);
public static readonly SettingKey<int> FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30); public static readonly SettingKey<int> FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30);
} }
@@ -154,58 +154,63 @@ public class TradeLifecycleService : ITradeLifecycleService
{ {
string targetUserId = !string.IsNullOrWhiteSpace(request.UserId) ? request.UserId : "default_user"; string targetUserId = !string.IsNullOrWhiteSpace(request.UserId) ? request.UserId : "default_user";
var existingTrade = await _dbContext.Trades // 1. Prüfen, ob DIESER spezifische Nutzer diesen Trade/AnalysisId bereits als aktiven Trade angenommen hat
var userExistingTrade = await _dbContext.Trades
.FirstOrDefaultAsync(t => .FirstOrDefaultAsync(t =>
(!string.IsNullOrEmpty(request.TradeId) && t.TradeId == request.TradeId) || !t.IsGlobalProposal &&
(!string.IsNullOrEmpty(request.AnalysisId) && t.AnalysisId == request.AnalysisId), cancellationToken); t.UserId == targetUserId &&
((!string.IsNullOrEmpty(request.TradeId) && t.TradeId == request.TradeId) ||
(!string.IsNullOrEmpty(request.AnalysisId) && t.AnalysisId == request.AnalysisId)),
cancellationToken);
if (existingTrade != null) if (userExistingTrade != null)
{ {
if (existingTrade.Status == TradeStatus.Closed) if (userExistingTrade.Status == TradeStatus.Closed)
{ {
await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Refused to accept trade {TradeId} because its status is CLOSED", existingTrade.TradeId); await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Refused to accept trade {TradeId} because user's trade is already CLOSED", userExistingTrade.TradeId);
return null; return null;
} }
existingTrade.Status = TradeStatus.Active; // Bestehenden User-Trade mit neuen Parametern aktualisieren
existingTrade.IsGlobalProposal = false; if (request.ActualEntryPrice > 0) userExistingTrade.ActualEntryPrice = request.ActualEntryPrice;
existingTrade.UserId = targetUserId; if (request.EntryPrice > 0) userExistingTrade.EntryPrice = request.EntryPrice.Value;
if (request.PositionSize > 0) userExistingTrade.PositionSize = request.PositionSize;
if (request.ActualEntryPrice > 0) existingTrade.ActualEntryPrice = request.ActualEntryPrice; if (request.LeverageUsed > 0) userExistingTrade.LeverageUsed = request.LeverageUsed;
if (request.EntryPrice > 0) existingTrade.EntryPrice = request.EntryPrice.Value; if (request.Quantity > 0) userExistingTrade.Quantity = request.Quantity;
if (request.PositionSize > 0) existingTrade.PositionSize = request.PositionSize; if (request.EntryFee.HasValue) userExistingTrade.EntryFee = request.EntryFee;
if (request.LeverageUsed > 0) existingTrade.LeverageUsed = request.LeverageUsed; if (request.ExitFee.HasValue) userExistingTrade.ExitFee = request.ExitFee;
if (request.Quantity > 0) existingTrade.Quantity = request.Quantity; if (request.StopLoss > 0) userExistingTrade.StopLoss = request.StopLoss.Value;
if (request.EntryFee.HasValue) existingTrade.EntryFee = request.EntryFee; if (request.TakeProfit > 0) userExistingTrade.TakeProfit = request.TakeProfit.Value;
if (request.ExitFee.HasValue) existingTrade.ExitFee = request.ExitFee; if (request.KnockoutThreshold > 0) userExistingTrade.KnockoutThreshold = request.KnockoutThreshold;
if (request.StopLoss > 0) existingTrade.StopLoss = request.StopLoss.Value; if (!string.IsNullOrWhiteSpace(request.Timeframe)) userExistingTrade.Timeframe = request.Timeframe;
if (request.TakeProfit > 0) existingTrade.TakeProfit = request.TakeProfit.Value; if (!string.IsNullOrWhiteSpace(request.DerivativeIsin)) userExistingTrade.DerivativeIsin = request.DerivativeIsin;
if (request.KnockoutThreshold > 0) existingTrade.KnockoutThreshold = request.KnockoutThreshold; if (!string.IsNullOrWhiteSpace(request.Reasoning)) userExistingTrade.Reasoning = request.Reasoning;
if (!string.IsNullOrWhiteSpace(request.Timeframe)) existingTrade.Timeframe = request.Timeframe;
if (!string.IsNullOrWhiteSpace(request.DerivativeIsin)) existingTrade.DerivativeIsin = request.DerivativeIsin;
if (!string.IsNullOrWhiteSpace(request.Reasoning)) existingTrade.Reasoning = request.Reasoning;
existingTrade.ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow; userExistingTrade.ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow;
existingTrade.PnlAbsolute = -(existingTrade.EntryFee ?? 0m) - (existingTrade.ExitFee ?? 0m); userExistingTrade.PnlAbsolute = -(userExistingTrade.EntryFee ?? 0m) - (userExistingTrade.ExitFee ?? 0m);
if (existingTrade.PositionSize > 0) if (userExistingTrade.PositionSize > 0)
{ {
existingTrade.PnlPercent = (existingTrade.PnlAbsolute / existingTrade.PositionSize) * 100m; userExistingTrade.PnlPercent = (userExistingTrade.PnlAbsolute / userExistingTrade.PositionSize) * 100m;
} }
_dbContext.Trades.Update(existingTrade); _dbContext.Trades.Update(userExistingTrade);
await _dbContext.SaveChangesAsync(cancellationToken); await _dbContext.SaveChangesAsync(cancellationToken);
await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully ACCEPTED and UPDATED trade {TradeId} for ISIN {Isin}, UserId: {UserId}", existingTrade.TradeId, existingTrade.Isin, existingTrade.UserId); await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully UPDATED existing trade {TradeId} for ISIN {Isin}, UserId: {UserId}", userExistingTrade.TradeId, userExistingTrade.Isin, userExistingTrade.UserId);
return existingTrade; return userExistingTrade;
} }
// 2. Globalen Trade-Vorschlag finden (dieser bleibt unverändert in der DB, damit andere Nutzer ihn ebenfalls annehmen können)
var proposal = await _dbContext.Trades var proposal = await _dbContext.Trades
.FirstOrDefaultAsync(t => t.IsGlobalProposal && .FirstOrDefaultAsync(t =>
(!string.IsNullOrEmpty(request.AnalysisId) ? t.AnalysisId == request.AnalysisId : t.Isin == request.Isin), (t.IsGlobalProposal || t.Status == TradeStatus.Proposed) &&
((!string.IsNullOrEmpty(request.AnalysisId) && t.AnalysisId == request.AnalysisId) ||
(!string.IsNullOrEmpty(request.TradeId) && t.TradeId == request.TradeId) ||
(!string.IsNullOrEmpty(request.Isin) && t.Isin == request.Isin)),
cancellationToken); cancellationToken);
var targetTradeId = !string.IsNullOrWhiteSpace(request.TradeId) ? request.TradeId : ("TRD-" + Guid.NewGuid().ToString("N")[..10].ToUpperInvariant()); var targetTradeId = "TRD-" + Guid.NewGuid().ToString("N")[..10].ToUpperInvariant();
var newTrade = new TradeEntity var newTrade = new TradeEntity
{ {
@@ -248,9 +253,10 @@ public class TradeLifecycleService : ITradeLifecycleService
EntryFee = request.EntryFee, EntryFee = request.EntryFee,
ExitFee = request.ExitFee, ExitFee = request.ExitFee,
ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow, ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow,
Quantity = request.Quantity, Quantity = request.Quantity > 0 ? request.Quantity : 1m,
KnockoutThreshold = request.KnockoutThreshold, KnockoutThreshold = request.KnockoutThreshold,
IsRecurring = request.IsRecurring IsRecurring = request.IsRecurring,
DerivativeProductCategories = proposal?.DerivativeProductCategories != null ? new List<string>(proposal.DerivativeProductCategories) : new List<string>()
}; };
newTrade.PnlAbsolute = -(newTrade.EntryFee ?? 0m) - (newTrade.ExitFee ?? 0m); newTrade.PnlAbsolute = -(newTrade.EntryFee ?? 0m) - (newTrade.ExitFee ?? 0m);
@@ -262,58 +268,69 @@ public class TradeLifecycleService : ITradeLifecycleService
_dbContext.Trades.Add(newTrade); _dbContext.Trades.Add(newTrade);
await _dbContext.SaveChangesAsync(cancellationToken); await _dbContext.SaveChangesAsync(cancellationToken);
await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully created active trade {TradeId} for ISIN {Isin}, UserId: {UserId}", newTrade.TradeId, request.Isin, newTrade.UserId); await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully CREATED individual active trade {TradeId} for ISIN {Isin}, UserId: {UserId} from proposal {AnalysisId}",
newTrade.TradeId, newTrade.Isin, newTrade.UserId, newTrade.AnalysisId);
return newTrade; return newTrade;
} }
public async Task AddHourlyUpdateAsync(TradeHourlyUpdateDto update, CancellationToken cancellationToken = default) public async Task AddHourlyUpdateAsync(TradeHourlyUpdateDto update, CancellationToken cancellationToken = default)
{ {
var trade = await _dbContext.Trades var matchedTrades = await _dbContext.Trades
.FirstOrDefaultAsync(t => t.TradeId == update.TradeId || t.Id.ToString() == update.TradeId, cancellationToken); .Where(t => t.TradeId == update.TradeId || (t.AnalysisId != null && t.AnalysisId == update.TradeId) || t.Id.ToString() == update.TradeId)
.ToListAsync(cancellationToken);
if (trade == null || (trade.Status != TradeStatus.Active && trade.Status != TradeStatus.Proposed)) if (matchedTrades.Count == 0)
{ {
await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Cannot add hourly update: Trade {TradeId} not found or not active/proposed.", update.TradeId); await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Cannot add hourly update: No active or proposed trades found for identifier {TradeId}.", update.TradeId);
return; return;
} }
var updateEntity = new TradeHourlyUpdateEntity foreach (var trade in matchedTrades)
{ {
TradeId = trade.Id, if (trade.Status != TradeStatus.Active && trade.Status != TradeStatus.Proposed)
Recommendation = update.Recommendation,
CurrentPrice = update.CurrentPrice,
SuggestedStopLoss = update.SuggestedStopLoss,
SuggestedTakeProfit = update.SuggestedTakeProfit,
VixValue = update.VixValue,
Reasoning = update.Reasoning,
Timestamp = update.Timestamp
};
_dbContext.TradeHourlyUpdates.Add(updateEntity);
if (update.SuggestedStopLoss.HasValue && update.SuggestedStopLoss > 0)
trade.StopLoss = update.SuggestedStopLoss.Value;
if (update.SuggestedTakeProfit.HasValue && update.SuggestedTakeProfit > 0)
trade.TakeProfit = update.SuggestedTakeProfit.Value;
if (string.Equals(update.Recommendation, "Close", StringComparison.OrdinalIgnoreCase))
{
if (trade.IsGlobalProposal || trade.Status == TradeStatus.Proposed)
{ {
trade.Status = TradeStatus.Invalidated; continue;
trade.CloseReason = "ProposalInvalidated";
trade.ClosedAt = DateTime.UtcNow;
} }
else
var updateEntity = new TradeHourlyUpdateEntity
{ {
await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Active trade {TradeId} received Close recommendation ({Reasoning}). Trade kept Active for user action.", TradeId = trade.Id,
trade.TradeId, update.Reasoning); Recommendation = update.Recommendation,
CurrentPrice = update.CurrentPrice,
SuggestedStopLoss = update.SuggestedStopLoss,
SuggestedTakeProfit = update.SuggestedTakeProfit,
VixValue = update.VixValue,
Reasoning = update.Reasoning,
Timestamp = update.Timestamp
};
_dbContext.TradeHourlyUpdates.Add(updateEntity);
if (update.SuggestedStopLoss.HasValue && update.SuggestedStopLoss > 0)
trade.StopLoss = update.SuggestedStopLoss.Value;
if (update.SuggestedTakeProfit.HasValue && update.SuggestedTakeProfit > 0)
trade.TakeProfit = update.SuggestedTakeProfit.Value;
if (string.Equals(update.Recommendation, "Close", StringComparison.OrdinalIgnoreCase))
{
if (trade.IsGlobalProposal || trade.Status == TradeStatus.Proposed)
{
trade.Status = TradeStatus.Invalidated;
trade.CloseReason = "ProposalInvalidated";
trade.ClosedAt = DateTime.UtcNow;
}
else
{
await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Active trade {TradeId} (UserId: {UserId}) received Close recommendation ({Reasoning}). Trade kept Active for user action.",
trade.TradeId, trade.UserId, update.Reasoning);
}
} }
} }
await _dbContext.SaveChangesAsync(cancellationToken); await _dbContext.SaveChangesAsync(cancellationToken);
await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Added hourly update for Trade {TradeId}. Recommendation: {Rec}, Price: {Price}", await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Added hourly update across {Count} matched trades for identifier {TradeId}. Rec: {Rec}, Price: {Price}",
update.TradeId, update.Recommendation, update.CurrentPrice); matchedTrades.Count, update.TradeId, update.Recommendation, update.CurrentPrice);
} }
public async Task<List<TradeEntity>> GetActiveTradesAsync(string? userId = null, CancellationToken cancellationToken = default) public async Task<List<TradeEntity>> GetActiveTradesAsync(string? userId = null, CancellationToken cancellationToken = default)