diff --git a/FinlyticApp/lib/features/asset_detail/models/fundamental_data_model.dart b/FinlyticApp/lib/features/asset_detail/models/fundamental_data_model.dart index f845e7c..2209180 100644 --- a/FinlyticApp/lib/features/asset_detail/models/fundamental_data_model.dart +++ b/FinlyticApp/lib/features/asset_detail/models/fundamental_data_model.dart @@ -191,13 +191,17 @@ class FundamentalDataModel extends Equatable { } final totalRev = parseNullableDouble(fundMap?['totalRevenue'] ?? json['totalRevenue']); - final grossProf = parseNullableDouble(fundMap?['grossProfit'] ?? json['grossProfit']); - double? grossMarginVal = parseNullableDouble(fundMap?['grossMargin'] ?? json['grossMargin']); - if (grossMarginVal == null && grossProf != null) { - if (grossProf <= 1.0 && grossProf >= 0.0) { - grossMarginVal = grossProf; + final rawGrossProfit = parseNullableDouble(fundMap?['grossProfit'] ?? json['grossProfit']); + double? grossMarginVal = parseNullableDouble(fundMap?['grossMargins'] ?? fundMap?['grossMargin'] ?? json['grossMargin']); + double? grossProfVal = rawGrossProfit; + if (rawGrossProfit != null) { + if (rawGrossProfit <= 1.0 && rawGrossProfit >= 0.0) { + grossMarginVal ??= rawGrossProfit; + if (totalRev != null && totalRev > 0) { + grossProfVal = rawGrossProfit * totalRev; + } } else if (totalRev != null && totalRev > 0) { - grossMarginVal = grossProf / totalRev; + grossMarginVal ??= rawGrossProfit / totalRev; } } @@ -207,6 +211,50 @@ class FundamentalDataModel extends Equatable { evToRevVal = evVal / totalRev; } + String? exDivDateStr = fundMap?['exDividendDate']?.toString() ?? json['exDividendDate']?.toString(); + String? nextEarningsDateStr = fundMap?['nextEarningsDate']?.toString() ?? json['nextEarningsDate']?.toString(); + + final rawEvents = json['events']; + if (rawEvents is List) { + final now = DateTime.now(); + + final divEvents = rawEvents.whereType>().where((e) { + final t = e['type']?.toString().toUpperCase() ?? ''; + return t == 'DIVIDEND' || t == 'EX_DIVIDEND'; + }).toList(); + + if (exDivDateStr == null && divEvents.isNotEmpty) { + divEvents.sort((a, b) { + final da = DateTime.tryParse(a['date']?.toString() ?? '') ?? DateTime(1970); + final db = DateTime.tryParse(b['date']?.toString() ?? '') ?? DateTime(1970); + return da.compareTo(db); + }); + final upcoming = divEvents.firstWhere((e) { + final d = DateTime.tryParse(e['date']?.toString() ?? ''); + return d != null && d.isAfter(now.subtract(const Duration(days: 7))); + }, orElse: () => divEvents.last); + exDivDateStr = upcoming['date']?.toString(); + } + + final earningsEvents = rawEvents.whereType>().where((e) { + final t = e['type']?.toString().toUpperCase() ?? ''; + return t.contains('EARNINGS'); + }).toList(); + + if (nextEarningsDateStr == null && earningsEvents.isNotEmpty) { + earningsEvents.sort((a, b) { + final da = DateTime.tryParse(a['date']?.toString() ?? '') ?? DateTime(1970); + final db = DateTime.tryParse(b['date']?.toString() ?? '') ?? DateTime(1970); + return da.compareTo(db); + }); + final upcoming = earningsEvents.firstWhere((e) { + final d = DateTime.tryParse(e['date']?.toString() ?? ''); + return d != null && d.isAfter(now.subtract(const Duration(days: 1))); + }, orElse: () => earningsEvents.last); + nextEarningsDateStr = upcoming['date']?.toString(); + } + } + return FundamentalDataModel( isin: isinVal, primaryTicker: primaryTickerVal, @@ -228,25 +276,25 @@ class FundamentalDataModel extends Equatable { fiftyTwoWeekLow: parseNullableDouble(fundMap?['fiftyTwoWeekLow'] ?? json['fiftyTwoWeekLow']), marketCapitalization: parseNullableDouble(fundMap?['marketCap'] ?? fundMap?['marketCapitalization'] ?? json['marketCapitalization']), enterpriseValue: evVal, - peRatioTrailing: parseNullableDouble(fundMap?['trailingPE'] ?? fundMap?['peRatioTrailing'] ?? json['peRatioTrailing']), - peRatioForward: parseNullableDouble(fundMap?['forwardPE'] ?? fundMap?['peRatioForward'] ?? json['peRatioForward']), + peRatioTrailing: parseNullableDouble(fundMap?['trailingPe'] ?? fundMap?['trailingPE'] ?? fundMap?['peRatioTrailing'] ?? json['peRatioTrailing'] ?? json['trailingPe']), + peRatioForward: parseNullableDouble(fundMap?['forwardPe'] ?? fundMap?['forwardPE'] ?? fundMap?['peRatioForward'] ?? json['peRatioForward'] ?? json['forwardPe']), pegRatio: parseNullableDouble(fundMap?['pegRatio'] ?? json['pegRatio']), pbRatio: parseNullableDouble(fundMap?['priceToBook'] ?? fundMap?['pbRatio'] ?? json['pbRatio']), - psRatio: parseNullableDouble(fundMap?['priceToSalesTrailing12Months'] ?? fundMap?['psRatio'] ?? json['psRatio']), - evToEbitda: parseNullableDouble(fundMap?['enterpriseToEbitda'] ?? fundMap?['evToEbitda'] ?? json['evToEbitda']), + psRatio: parseNullableDouble(fundMap?['priceToSales'] ?? fundMap?['priceToSalesTrailing12Months'] ?? fundMap?['psRatio'] ?? json['psRatio']), + evToEbitda: parseNullableDouble(fundMap?['evToEbitda'] ?? fundMap?['enterpriseToEbitda'] ?? json['evToEbitda']), evToRevenue: evToRevVal, totalRevenue: totalRev, - revenueGrowthYoY: parseNullableDouble(fundMap?['revenueGrowth'] ?? fundMap?['revenueGrowthYoY'] ?? json['revenueGrowthYoY']), - grossProfit: grossProf, + revenueGrowthYoY: parseNullableDouble(fundMap?['revenueGrowthYoY'] ?? fundMap?['revenueGrowth'] ?? json['revenueGrowthYoY']), + grossProfit: grossProfVal, ebitda: parseNullableDouble(fundMap?['ebitda'] ?? json['ebitda']), - dilutedEps: parseNullableDouble(fundMap?['trailingEps'] ?? fundMap?['dilutedEps'] ?? json['dilutedEps']), + dilutedEps: parseNullableDouble(fundMap?['dilutedEps'] ?? fundMap?['trailingEps'] ?? json['dilutedEps']), totalCash: parseNullableDouble(fundMap?['totalCash'] ?? json['totalCash']), totalDebt: parseNullableDouble(fundMap?['totalDebt'] ?? json['totalDebt']), - operatingCashFlow: parseNullableDouble(fundMap?['operatingCashflow'] ?? fundMap?['operatingCashFlow'] ?? json['operatingCashFlow']), - freeCashFlow: parseNullableDouble(fundMap?['freeCashflow'] ?? fundMap?['freeCashFlow'] ?? json['freeCashFlow']), + operatingCashFlow: parseNullableDouble(fundMap?['operatingCashFlow'] ?? fundMap?['operatingCashflow'] ?? json['operatingCashFlow']), + freeCashFlow: parseNullableDouble(fundMap?['freeCashFlow'] ?? fundMap?['freeCashflow'] ?? json['freeCashFlow']), grossMargin: grossMarginVal, - operatingMargin: parseNullableDouble(fundMap?['operatingMargins'] ?? fundMap?['operatingMargin'] ?? json['operatingMargin']), - netProfitMargin: parseNullableDouble(fundMap?['profitMargins'] ?? fundMap?['netProfitMargin'] ?? json['netProfitMargin']), + operatingMargin: parseNullableDouble(fundMap?['operatingIncome'] ?? fundMap?['operatingMargins'] ?? fundMap?['operatingMargin'] ?? json['operatingMargin']), + netProfitMargin: parseNullableDouble(fundMap?['netIncome'] ?? fundMap?['profitMargins'] ?? fundMap?['netProfitMargin'] ?? json['netProfitMargin']), returnOnEquity: parseNullableDouble(fundMap?['returnOnEquity'] ?? json['returnOnEquity']), returnOnAssets: parseNullableDouble(fundMap?['returnOnAssets'] ?? json['returnOnAssets']), returnOnInvestedCapital: parseNullableDouble(fundMap?['returnOnInvestedCapital'] ?? json['returnOnInvestedCapital']), @@ -254,10 +302,10 @@ class FundamentalDataModel extends Equatable { currentRatio: parseNullableDouble(fundMap?['currentRatio'] ?? json['currentRatio']), quickRatio: parseNullableDouble(fundMap?['quickRatio'] ?? json['quickRatio']), interestCoverage: parseNullableDouble(fundMap?['interestCoverage'] ?? json['interestCoverage']), - dividendYield: parseNullableDouble(fundMap?['dividendYield'] ?? json['dividendYield']), + dividendYield: parseNullableDouble(fundMap?['forwardDividendYield'] ?? fundMap?['dividendYield'] ?? json['dividendYield']), payoutRatio: parseNullableDouble(fundMap?['payoutRatio'] ?? json['payoutRatio']), - exDividendDate: fundMap?['exDividendDate']?.toString() ?? json['exDividendDate']?.toString(), - nextEarningsDate: fundMap?['nextEarningsDate']?.toString() ?? json['nextEarningsDate']?.toString(), + exDividendDate: exDivDateStr, + nextEarningsDate: nextEarningsDateStr, percentHeldByInstitutions: parseNullableDouble(fundMap?['percentHeldByInstitutions'] ?? json['percentHeldByInstitutions']), percentHeldByInsiders: parseNullableDouble(fundMap?['percentHeldByInsiders'] ?? json['percentHeldByInsiders']), shortRatio: parseNullableDouble(fundMap?['shortRatio'] ?? json['shortRatio']), diff --git a/FinlyticApp/lib/features/asset_detail/widgets/fundamentals/fundamental_category_panels.dart b/FinlyticApp/lib/features/asset_detail/widgets/fundamentals/fundamental_category_panels.dart index e382868..95b738a 100644 --- a/FinlyticApp/lib/features/asset_detail/widgets/fundamentals/fundamental_category_panels.dart +++ b/FinlyticApp/lib/features/asset_detail/widgets/fundamentals/fundamental_category_panels.dart @@ -45,7 +45,7 @@ class FundamentalCategoryPanels extends StatelessWidget { String _fmtPercent(double? val) { if (val == null) return 'N/A'; - final p = (val.abs() <= 1.0 && val != 0.0) ? val * 100.0 : val; + final p = (val.abs() <= 5.0 && val != 0.0) ? val * 100.0 : val; return '${p.toStringAsFixed(2)}%'; } diff --git a/FinlyticCore/Clients/YahooFinanceClient.cs b/FinlyticCore/Clients/YahooFinanceClient.cs index a1c4abc..2e812e9 100644 --- a/FinlyticCore/Clients/YahooFinanceClient.cs +++ b/FinlyticCore/Clients/YahooFinanceClient.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using FinlyticCore.Dtos.Yahoo; using FinlyticCore.Models.Settings; using FinlyticCore.Services; +using FinlyticCore.Services.PlaywrightScrapper; +using Microsoft.Playwright; namespace FinlyticCore.Services.Yahoo; @@ -18,9 +20,11 @@ public class YahooFinanceClient private readonly CookieContainer _cookieContainer; private readonly IFinlyticLogger? _finlyticLogger; private readonly ISettingsService? _settingsService; + private readonly IPlaywrightExecutionService? _playwrightService; private readonly SemaphoreSlim _authLock = new(1, 1); private string? _crumb; + private string? _rawCookieHeader; private DateTime _lastAuthTime = DateTime.MinValue; /// @@ -44,10 +48,12 @@ public class YahooFinanceClient public YahooFinanceClient( IFinlyticLogger? finlyticLogger = null, ISettingsService? settingsService = null, + IPlaywrightExecutionService? playwrightService = null, HttpClient? httpClient = null) { _finlyticLogger = finlyticLogger; _settingsService = settingsService; + _playwrightService = playwrightService; _cookieContainer = new CookieContainer(); if (httpClient != null) @@ -97,6 +103,7 @@ public class YahooFinanceClient { RestoreCookies(cachedCookies); _crumb = cachedCrumb; + _rawCookieHeader = cachedCookies; _lastAuthTime = DateTime.UtcNow; if (_finlyticLogger != null) @@ -115,70 +122,40 @@ public class YahooFinanceClient await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)..."); // 3. Send GET request to fc.yahoo.com to obtain session cookie A3 - using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com")) + try { + using var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"); initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"); using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken); } + catch { } // 4. Send GET request to getcrumb to obtain dynamic crumb token - string[] crumbUrls = new[] + var crumb = await FetchCrumbWithHttpClientAsync(cancellationToken); + if (!string.IsNullOrWhiteSpace(crumb)) { - "https://query1.finance.yahoo.com/v1/test/getcrumb", - "https://query2.finance.yahoo.com/v1/test/getcrumb" - }; + _crumb = crumb; + _lastAuthTime = DateTime.UtcNow; + await PersistSessionAsync(_crumb, cancellationToken); + return _crumb; + } - foreach (var url in crumbUrls) + // 5. FALLBACK: Playwright Browser Authentication (Bypasses EU Consent Wall and 429) + if (_playwrightService != null) { - try + var (browserCrumb, browserCookies) = await AuthenticateViaPlaywrightAsync(cancellationToken); + if (!string.IsNullOrWhiteSpace(browserCrumb)) { - using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url); - crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); - crumbRequest.Headers.Add("Accept", "*/*"); - - using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken); - if (crumbResponse.IsSuccessStatusCode) + _crumb = browserCrumb; + _lastAuthTime = DateTime.UtcNow; + if (!string.IsNullOrWhiteSpace(browserCookies)) { - var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken); - if (!string.IsNullOrWhiteSpace(crumbText)) - { - _crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n'); - _lastAuthTime = DateTime.UtcNow; - - // Persist to DB cache via SettingsService - if (_settingsService != null) - { - try - { - var serializedCookies = SerializeCookies(); - await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken); - if (!string.IsNullOrWhiteSpace(serializedCookies)) - { - await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken); - } - } - catch (Exception persistEx) - { - if (_finlyticLogger != null) - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database."); - } - } - - if (_finlyticLogger != null) - await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb); - return _crumb; - } + RestoreCookies(browserCookies); + _rawCookieHeader = browserCookies; } - else - { - if (_finlyticLogger != null) - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode); - } - } - catch - { - // Fallthrough to next endpoint + await PersistSessionAsync(_crumb, cancellationToken); + return _crumb; } } @@ -198,13 +175,204 @@ public class YahooFinanceClient } } + private async Task FetchCrumbWithHttpClientAsync(CancellationToken cancellationToken) + { + string[] crumbUrls = new[] + { + "https://query1.finance.yahoo.com/v1/test/getcrumb", + "https://query2.finance.yahoo.com/v1/test/getcrumb" + }; + + foreach (var url in crumbUrls) + { + try + { + using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url); + crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); + crumbRequest.Headers.Add("Accept", "*/*"); + crumbRequest.Headers.Add("Origin", "https://finance.yahoo.com"); + crumbRequest.Headers.Add("Referer", "https://finance.yahoo.com/"); + + if (!string.IsNullOrWhiteSpace(_rawCookieHeader)) + { + crumbRequest.Headers.Add("Cookie", _rawCookieHeader); + } + + using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken); + var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken); + + if (crumbResponse.IsSuccessStatusCode && !string.IsNullOrWhiteSpace(crumbText)) + { + var cleanCrumb = crumbText.Trim('"', ' ', '\t', '\r', '\n'); + if (!cleanCrumb.Contains(" AuthenticateViaPlaywrightAsync(CancellationToken cancellationToken) + { + if (_playwrightService == null) return (null, null); + + try + { + if (_finlyticLogger != null) + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Launching Playwright browser to acquire valid EU Yahoo session and Crumb..."); + + return await _playwrightService.ExecuteInContextAsync(async context => + { + var page = await context.NewPageAsync(); + try + { + await page.GotoAsync("https://finance.yahoo.com/quote/AAPL/", new PageGotoOptions + { + WaitUntil = WaitUntilState.DOMContentLoaded, + Timeout = 30_000 + }); + + // Handle EU Consent if redirected + var url = page.Url; + if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) || + url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase)) + { + var selectors = new[] + { + "button[name='agree']", + "button[value='agree']", + "button.accept-all", + "button.btn.primary", + "button.btn.secondary.accept-all", + "form[action*='consent'] button[type='submit']", + "button:has-text('Alle akzeptieren')", + "button:has-text('Accept all')", + "button:has-text('Akzeptieren')", + "button:has-text('Agree')" + }; + + foreach (var sel in selectors) + { + var btn = page.Locator(sel); + if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync()) + { + await btn.First.ClickAsync(); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15_000 }); + break; + } + } + } + + // 1. Extract cookies from BrowserContext + var cookies = await context.CookiesAsync(); + var pairs = new List(); + foreach (var c in cookies) + { + if (c.Domain.Contains("yahoo.com", StringComparison.OrdinalIgnoreCase)) + { + pairs.Add($"{c.Name}={c.Value}"); + } + } + var extractedCookies = string.Join(";", pairs); + + // 2. Fetch Crumb from inside the authenticated page + string? extractedCrumb = null; + try + { + extractedCrumb = await page.EvaluateAsync(@"async () => { + try { + const res = await fetch('/v1/test/getcrumb'); + if (res.ok) { + return await res.text(); + } + } catch {} + return null; + }"); + } + catch { } + + // 3. If in-page fetch was empty, use the extracted cookies with HttpClient + if (string.IsNullOrWhiteSpace(extractedCrumb) && !string.IsNullOrWhiteSpace(extractedCookies)) + { + RestoreCookies(extractedCookies); + _rawCookieHeader = extractedCookies; + extractedCrumb = await FetchCrumbWithHttpClientAsync(cancellationToken); + } + + if (!string.IsNullOrWhiteSpace(extractedCrumb)) + { + extractedCrumb = extractedCrumb.Trim('"', ' ', '\t', '\r', '\n'); + if (_finlyticLogger != null) + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Playwright successfully acquired Yahoo Crumb token: {Crumb}", extractedCrumb); + } + + return (extractedCrumb, extractedCookies); + } + finally + { + await page.CloseAsync(); + } + }, cancellationToken: cancellationToken); + } + catch (Exception ex) + { + if (_finlyticLogger != null) + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Playwright-based authentication failed."); + return (null, null); + } + } + + private async Task PersistSessionAsync(string crumb, CancellationToken cancellationToken) + { + if (_settingsService == null || string.IsNullOrWhiteSpace(crumb)) return; + + try + { + var serializedCookies = SerializeCookies(); + await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, crumb, cancellationToken); + if (!string.IsNullOrWhiteSpace(serializedCookies)) + { + _rawCookieHeader = serializedCookies; + await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken); + } + + if (_finlyticLogger != null) + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Persisted valid Yahoo session & crumb to database."); + } + catch (Exception ex) + { + if (_finlyticLogger != null) + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Failed to persist Yahoo session to database."); + } + } + private string SerializeCookies() { + if (!string.IsNullOrWhiteSpace(_rawCookieHeader)) + { + return _rawCookieHeader; + } + try { var cookies = _cookieContainer.GetAllCookies(); var pairs = new List(); - foreach (Cookie cookie in cookies) + foreach (System.Net.Cookie cookie in cookies) { pairs.Add($"{cookie.Name}={cookie.Value}"); } @@ -220,9 +388,20 @@ public class YahooFinanceClient { if (string.IsNullOrWhiteSpace(serializedCookies)) return; + _rawCookieHeader = serializedCookies; + try { var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var uris = new[] + { + new Uri("https://yahoo.com"), + new Uri("https://finance.yahoo.com"), + new Uri("https://query1.finance.yahoo.com"), + new Uri("https://query2.finance.yahoo.com"), + new Uri("https://fc.yahoo.com") + }; + foreach (var part in parts) { var eqIdx = part.IndexOf('='); @@ -230,7 +409,14 @@ public class YahooFinanceClient { var name = part.Substring(0, eqIdx).Trim(); var val = part.Substring(eqIdx + 1).Trim(); - _cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com")); + foreach (var uri in uris) + { + try + { + _cookieContainer.Add(uri, new System.Net.Cookie(name, val)); + } + catch { } + } } } } @@ -242,8 +428,6 @@ public class YahooFinanceClient /// /// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API. - /// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&quotesCount={quotesCount}&newsCount={newsCount} - /// Note: Does not require Cookie/Crumb authentication. /// public async Task SearchAsync( string query, @@ -253,33 +437,55 @@ public class YahooFinanceClient { if (string.IsNullOrWhiteSpace(query)) return null; - try + string[] endpoints = new[] { - var url = - $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}"; - using var response = await _httpClient.GetAsync(url, cancellationToken); + $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}", + $"https://query1.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}" + }; - if (!response.IsSuccessStatusCode) + foreach (var url in endpoints) + { + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); + request.Headers.Add("Accept", "*/*"); + if (!string.IsNullOrWhiteSpace(_rawCookieHeader)) + { + request.Headers.Add("Cookie", _rawCookieHeader); + } + + using var response = await _httpClient.SendAsync(request, cancellationToken); + var json = await response.Content.ReadAsStringAsync(cancellationToken); + + if (response.IsSuccessStatusCode) + { + var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); + if (dto != null) + { + return dto; + } + } + else + { + if (_finlyticLogger != null) + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, + "[YahooFinanceClient] Search endpoint '{Url}' for '{Query}' returned status {Status}. Response Body: {Body}", + url, query, response.StatusCode, json); + } + } + catch (Exception ex) { if (_finlyticLogger != null) - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode); - return null; + await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search endpoint '{Url}'", url); } + } - var json = await response.Content.ReadAsStringAsync(cancellationToken); - return JsonSerializer.Deserialize(json, GetJsonOptions()); - } - catch (Exception ex) - { - if (_finlyticLogger != null) - await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query); - return null; - } + return null; } /// /// Retrieves fundamentals and company metadata using the quoteSummary endpoint. - /// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&modules={modules} /// public async Task GetQuoteSummaryAsync( string symbol, @@ -291,22 +497,49 @@ public class YahooFinanceClient var moduleList = string.Join(",", modules); return await ExecuteWithRetryAsync(async (crumb) => { - var url = - $"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}"; - using var response = await _httpClient.GetAsync(url, cancellationToken); - - if (!response.IsSuccessStatusCode) + string[] baseUrls = new[] { + "https://query2.finance.yahoo.com/v10/finance/quoteSummary", + "https://query1.finance.yahoo.com/v10/finance/quoteSummary" + }; + + foreach (var baseUrl in baseUrls) + { + var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}"; + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); + request.Headers.Add("Accept", "*/*"); + request.Headers.Add("Origin", "https://finance.yahoo.com"); + request.Headers.Add("Referer", "https://finance.yahoo.com/"); + + if (!string.IsNullOrWhiteSpace(_rawCookieHeader)) + { + request.Headers.Add("Cookie", _rawCookieHeader); + } + + using var response = await _httpClient.SendAsync(request, cancellationToken); + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken); + + if (response.IsSuccessStatusCode) + { + var dto = JsonSerializer.Deserialize(responseContent, GetJsonOptions()); + return (false, dto); + } + if (_finlyticLogger != null) - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode); - return ( - response.StatusCode == HttpStatusCode.Unauthorized || - response.StatusCode == HttpStatusCode.Forbidden, null); + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, + "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}", + symbol, response.StatusCode, url, responseContent); + + if (response.StatusCode == HttpStatusCode.Unauthorized || + response.StatusCode == HttpStatusCode.Forbidden || + response.StatusCode == HttpStatusCode.TooManyRequests) + { + return (true, null); + } } - var json = await response.Content.ReadAsStringAsync(cancellationToken); - var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); - return (false, dto); + return (false, null); }, cancellationToken); } @@ -321,7 +554,6 @@ public class YahooFinanceClient /// /// Retrieves historical OHLCV chart data for a given symbol. - /// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&interval={interval}&crumb={crumb} /// public async Task GetChartAsync( string symbol, @@ -333,28 +565,54 @@ public class YahooFinanceClient return await ExecuteWithRetryAsync(async (crumb) => { - var url = - $"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}"; - using var response = await _httpClient.GetAsync(url, cancellationToken); - - if (!response.IsSuccessStatusCode) + string[] baseUrls = new[] { + "https://query1.finance.yahoo.com/v8/finance/chart", + "https://query2.finance.yahoo.com/v8/finance/chart" + }; + + foreach (var baseUrl in baseUrls) + { + var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}"; + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); + request.Headers.Add("Accept", "*/*"); + request.Headers.Add("Origin", "https://finance.yahoo.com"); + request.Headers.Add("Referer", "https://finance.yahoo.com/"); + + if (!string.IsNullOrWhiteSpace(_rawCookieHeader)) + { + request.Headers.Add("Cookie", _rawCookieHeader); + } + + using var response = await _httpClient.SendAsync(request, cancellationToken); + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken); + + if (response.IsSuccessStatusCode) + { + var dto = JsonSerializer.Deserialize(responseContent, GetJsonOptions()); + return (false, dto); + } + if (_finlyticLogger != null) - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode); - return ( - response.StatusCode == HttpStatusCode.Unauthorized || - response.StatusCode == HttpStatusCode.Forbidden, null); + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, + "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}", + symbol, response.StatusCode, url, responseContent); + + if (response.StatusCode == HttpStatusCode.Unauthorized || + response.StatusCode == HttpStatusCode.Forbidden || + response.StatusCode == HttpStatusCode.TooManyRequests) + { + return (true, null); + } } - var json = await response.Content.ReadAsStringAsync(cancellationToken); - var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); - return (false, dto); + return (false, null); }, cancellationToken); } /// /// Retrieves quick real-time price quotes for one or more symbols. - /// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&crumb={crumb} /// public async Task GetQuotesAsync( IEnumerable symbols, @@ -366,22 +624,49 @@ public class YahooFinanceClient var symbolsParam = string.Join(",", symbolList); return await ExecuteWithRetryAsync(async (crumb) => { - var url = - $"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}"; - using var response = await _httpClient.GetAsync(url, cancellationToken); - - if (!response.IsSuccessStatusCode) + string[] baseUrls = new[] { + "https://query1.finance.yahoo.com/v7/finance/quote", + "https://query2.finance.yahoo.com/v7/finance/quote" + }; + + foreach (var baseUrl in baseUrls) + { + var url = $"{baseUrl}?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}"; + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); + request.Headers.Add("Accept", "*/*"); + request.Headers.Add("Origin", "https://finance.yahoo.com"); + request.Headers.Add("Referer", "https://finance.yahoo.com/"); + + if (!string.IsNullOrWhiteSpace(_rawCookieHeader)) + { + request.Headers.Add("Cookie", _rawCookieHeader); + } + + using var response = await _httpClient.SendAsync(request, cancellationToken); + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken); + + if (response.IsSuccessStatusCode) + { + var dto = JsonSerializer.Deserialize(responseContent, GetJsonOptions()); + return (false, dto); + } + if (_finlyticLogger != null) - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode); - return ( - response.StatusCode == HttpStatusCode.Unauthorized || - response.StatusCode == HttpStatusCode.Forbidden, null); + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, + "[YahooFinanceClient] GetQuotes failed with status {Status}. URL: {Url}. Response Body: {Body}", + response.StatusCode, url, responseContent); + + if (response.StatusCode == HttpStatusCode.Unauthorized || + response.StatusCode == HttpStatusCode.Forbidden || + response.StatusCode == HttpStatusCode.TooManyRequests) + { + return (true, null); + } } - var json = await response.Content.ReadAsStringAsync(cancellationToken); - var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); - return (false, dto); + return (false, null); }, cancellationToken); } @@ -408,26 +693,29 @@ public class YahooFinanceClient CancellationToken cancellationToken) where T : class { var crumb = await EnsureAuthenticatedAsync(false, cancellationToken); + if (!string.IsNullOrEmpty(crumb)) + { + var (isAuthError, result) = await action(crumb); + if (!isAuthError && result != null) + { + return result; + } + + if (!isAuthError) + { + return result; + } + } + + // Re-authenticate when auth error (401/403/429) or empty crumb occurs + if (_finlyticLogger != null) + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error or invalid crumb encountered (401/403/429). Force re-authenticating..."); + + crumb = await EnsureAuthenticatedAsync(true, cancellationToken); if (string.IsNullOrEmpty(crumb)) return null; - var (isAuthError, result) = await action(crumb); - if (!isAuthError && result != null) - { - return result; - } - - if (isAuthError) - { - if (_finlyticLogger != null) - await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating..."); - crumb = await EnsureAuthenticatedAsync(true, cancellationToken); - if (string.IsNullOrEmpty(crumb)) return null; - - var (_, retryResult) = await action(crumb); - return retryResult; - } - - return result; + var (_, retryResult) = await action(crumb); + return retryResult; } private static JsonSerializerOptions GetJsonOptions() diff --git a/FinlyticCore/Clients/YahooFinanceHtmlClient.cs b/FinlyticCore/Clients/YahooFinanceHtmlClient.cs index f35793b..01d4232 100644 --- a/FinlyticCore/Clients/YahooFinanceHtmlClient.cs +++ b/FinlyticCore/Clients/YahooFinanceHtmlClient.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -17,14 +18,15 @@ public interface IYahooFinanceHtmlClient { Task ScrapeQuoteSummaryModulesAsync( string isinOrSymbol, + bool includeProfile = true, CancellationToken cancellationToken = default); } public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient { - private const string _serviceName = nameof(YahooFinanceHtmlClient); private readonly IPlaywrightExecutionService _playwrightService; private readonly IFinlyticLogger _finlyticLogger; + private const string _serviceName = "YahooFinanceHtmlClient"; public YahooFinanceHtmlClient( IPlaywrightExecutionService playwrightService, @@ -36,55 +38,93 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient public async Task ScrapeQuoteSummaryModulesAsync( string isinOrSymbol, + bool includeProfile = true, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null; var symbol = isinOrSymbol.Trim().ToUpperInvariant(); - await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'..."); + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Starting parallel fast Playwright HTML DOM scrape for symbol '{symbol}' (IncludeProfile: {includeProfile})..."); return await _playwrightService.ExecuteInContextAsync(async context => { - var keyStatsData = new Dictionary(StringComparer.OrdinalIgnoreCase); - var financialsData = new Dictionary(StringComparer.OrdinalIgnoreCase); - var analysisData = new Dictionary(StringComparer.OrdinalIgnoreCase); - - ProfileExtractionResult? profileResult = null; - + var summaryUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/"; var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/"; - var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/"; var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/"; - var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/"; + var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/"; - var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken); - var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken); - var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken); - var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken); + // 1. Initial Page: Authenticate session and pass Cookie Consent once for the entire context + var initialPage = await context.NewPageAsync(); + var summaryData = new Dictionary(StringComparer.OrdinalIgnoreCase); try { - await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask); + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [1/2] Loading Summary & passing Consent: {summaryUrl}"); + await initialPage.GotoAsync(summaryUrl, new PageGotoOptions + { + WaitUntil = WaitUntilState.DOMContentLoaded, + Timeout = 15_000 + }); - keyStatsData = await statsTask; - profileResult = await profileTask; - financialsData = await financialsTask; - analysisData = await analysisTask; + await HandleConsentAsync(initialPage); + await WaitForContentAsync(initialPage); + summaryData = await ExtractKeyValuePairsFromPageAsync(initialPage, summaryUrl); } catch (Exception ex) { - await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}."); + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping initial Summary page for {symbol}."); + } + finally + { + await initialPage.CloseAsync(); + } + + // 2. Parallel Sub-Pages (Stats, Financials, and conditionally Profile) + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [2/2] Fetching Sub-Pages in parallel (IncludeProfile: {includeProfile})..."); + + var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken); + var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken); + Task profileTask = includeProfile + ? ScrapeProfilePageAsync(context, profileUrl, cancellationToken)! + : Task.FromResult(null); + + var keyStatsData = new Dictionary(StringComparer.OrdinalIgnoreCase); + var financialsData = new Dictionary(StringComparer.OrdinalIgnoreCase); + ProfileExtractionResult? profileResult = null; + + try + { + await Task.WhenAll(statsTask, financialsTask, profileTask); + + keyStatsData = await statsTask; + financialsData = await financialsTask; + profileResult = await profileTask; + } + catch (Exception ex) + { + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error during parallel sub-page scrape for {symbol}."); + } + + // Merge summary data into key stats + foreach (var (k, v) in summaryData) + { + if (!keyStatsData.ContainsKey(k)) + { + keyStatsData[k] = v; + } } var profileDict = profileResult?.ProfileDict ?? new Dictionary(StringComparer.OrdinalIgnoreCase); var officers = profileResult?.Officers ?? new List(); - await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}"); + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, + $"[{_serviceName}] Fast HTML DOM scrape complete for '{symbol}'. Summary: {summaryData.Count}, Stats: {keyStatsData.Count}, Financials: {financialsData.Count}, Profile: {profileDict.Count}, Officers: {officers.Count}"); return BuildModulesDto( keyStatsData, profileDict, financialsData, - analysisData, + new Dictionary(), officers, profileResult?.Sector, profileResult?.Industry, @@ -107,47 +147,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient await page.GotoAsync(url, new PageGotoOptions { WaitUntil = WaitUntilState.DOMContentLoaded, - Timeout = 20_000 + Timeout = 10_000 }); - await HandleConsentAsync(page); - - var extracted = await page.EvaluateAsync>(@"() => { - const results = {}; - - const cleanKey = (str) => { - return str.toLowerCase() - .replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '') - .replace(/\s*\d+\s*$/, '') - .replace(/\s+/g, ' ') - .trim(); - }; - - document.querySelectorAll('table tr').forEach(tr => { - const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim()); - if (cells.length >= 2 && cells[0] && cells[1]) { - const key = cleanKey(cells[0]); - const val = cells[1].replace(/\s+/g, ' ').trim(); - if (key && val && val !== 'N/A' && val !== '--' && val !== '-') { - results[key] = val; - } - } - }); - - return results; - }"); - - if (extracted != null) - { - foreach (var (k, v) in extracted) - { - targetDict[k] = v; - } - } + await WaitForContentAsync(page); + targetDict = await ExtractKeyValuePairsFromPageAsync(page, url); } catch (Exception ex) { - await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}"); + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping URL {url}"); } finally { @@ -157,7 +165,7 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient return targetDict; } - private async Task ScrapeProfilePageAsync( + private async Task ScrapeProfilePageAsync( IBrowserContext context, string url, CancellationToken cancellationToken) @@ -176,15 +184,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient await page.GotoAsync(url, new PageGotoOptions { WaitUntil = WaitUntilState.DOMContentLoaded, - Timeout = 20_000 + Timeout = 10_000 }); - await HandleConsentAsync(page); + await WaitForContentAsync(page); - var metaInfo = await page.EvaluateAsync(@"() => { + var jsonStr = await page.EvaluateAsync(@"() => { let sector = null, industry = null, employees = null, description = null; - const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary'); + const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary, section[data-testid=""asset-profile""] p'); if (descEl) description = descEl.innerText.trim(); const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main'); @@ -219,40 +227,51 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient } }); - return { sector, industry, employees, description, officers }; + return JSON.stringify({ sector, industry, employees, description, officers }); }"); - if (metaInfo != null) + if (!string.IsNullOrWhiteSpace(jsonStr)) { - sector = metaInfo.Sector; - industry = metaInfo.Industry; - fullTimeEmployees = metaInfo.Employees; - description = metaInfo.Description; + var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; + var metaInfo = JsonSerializer.Deserialize(jsonStr, options); - if (metaInfo.Officers != null) + if (metaInfo != null) { - foreach (var off in metaInfo.Officers) + sector = metaInfo.Sector; + industry = metaInfo.Industry; + fullTimeEmployees = metaInfo.Employees; + description = metaInfo.Description; + + if (metaInfo.Officers != null) { - if (!string.IsNullOrWhiteSpace(off.Name)) + foreach (var off in metaInfo.Officers) { - companyOfficers.Add(new YahooCompanyOfficerDto( - Name: off.Name, - Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null, - Title: off.Title, - YearBorn: off.YearBorn, - FiscalYear: null, - TotalPay: ParseYahooValue(off.Pay), - ExercisedValue: ParseYahooValue(off.Exercised), - UnexercisedValue: null - )); + if (!string.IsNullOrWhiteSpace(off.Name)) + { + companyOfficers.Add(new YahooCompanyOfficerDto( + Name: off.Name, + Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null, + Title: off.Title, + YearBorn: off.YearBorn, + FiscalYear: null, + TotalPay: ParseYahooValue(off.Pay), + ExercisedValue: ParseYahooValue(off.Exercised), + UnexercisedValue: null + )); + } } } + + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, + $"[{_serviceName}] HTML Profile extracted -> Sector: '{sector}', Industry: '{industry}', Employees: {fullTimeEmployees}, Officers: {companyOfficers.Count}, Desc length: {description?.Length ?? 0}"); } } + + profileDict = await ExtractKeyValuePairsFromPageAsync(page, url); } catch (Exception ex) { - await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}"); + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping Profile data from {url}"); } finally { @@ -262,21 +281,151 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description); } - private static async Task HandleConsentAsync(IPage page) + private static async Task WaitForContentAsync(IPage page) { try { - if (page.Url.Contains("consent.yahoo.com")) + await page.WaitForSelectorAsync("table tr, ul li, div[data-testid], section, main", new PageWaitForSelectorOptions { - var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button"); - if (await consentBtn.CountAsync() > 0) + State = WaitForSelectorState.Attached, + Timeout = 2_500 + }); + } + catch { } + } + + private async Task> ExtractKeyValuePairsFromPageAsync(IPage page, string pageUrl) + { + var targetDict = new Dictionary(StringComparer.OrdinalIgnoreCase); + + try + { + var jsonStr = await page.EvaluateAsync(@"() => { + const results = {}; + + const cleanKey = (str) => { + return str.toLowerCase() + .replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '') + .replace(/\s*\d+\s*$/, '') + .replace(/\s+/g, ' ') + .trim(); + }; + + // 1. Standard HTML Table Rows (Financials, Balance Sheet, Key Stats) + document.querySelectorAll('table tr').forEach(tr => { + const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim()); + if (cells.length >= 2 && cells[0] && cells[1]) { + const key = cleanKey(cells[0]); + const val = cells[1].replace(/\s+/g, ' ').trim(); + if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) { + results[key] = val; + } + } + }); + + // 2. Modern Yahoo Finance List / Div pairs (Summary Table, Quote Statistics, Flex containers) + document.querySelectorAll('li, div[class*=""container""], div[class*=""row""], section div, div[data-testid]').forEach(el => { + const children = Array.from(el.querySelectorAll(':scope > span, :scope > div, :scope > p')).map(s => s.innerText.trim()).filter(Boolean); + if (children.length === 2) { + const key = cleanKey(children[0]); + const val = children[1].replace(/\s+/g, ' ').trim(); + if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) { + if (!results[key]) { + results[key] = val; + } + } + } + }); + + return JSON.stringify(results); + }"); + + if (!string.IsNullOrWhiteSpace(jsonStr)) + { + var parsed = JsonSerializer.Deserialize>(jsonStr); + if (parsed != null) { - await consentBtn.First.ClickAsync(); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 }); + foreach (var (k, v) in parsed) + { + targetDict[k] = v; + } + } + } + + var sampleKeys = targetDict.Keys.Take(6).ToList(); + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, + $"[{_serviceName}] Extracted {targetDict.Count} items from HTML of '{pageUrl}'. Sample keys: [{string.Join(", ", sampleKeys)}]"); + } + catch (Exception ex) + { + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error extracting key-value pairs from HTML of '{pageUrl}'"); + } + + return targetDict; + } + + private async Task HandleConsentAsync(IPage page) + { + try + { + var url = page.Url; + if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) || + url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase)) + { + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, + $"[{_serviceName}] Detected EU Cookie Consent redirect: '{url}'. Searching for Accept/Reject buttons..."); + + var selectors = new[] + { + "button[name='agree']", + "button[value='agree']", + "button.accept-all", + "button.btn.primary", + "button.btn.secondary.accept-all", + "button[name='reject']", + "button[value='reject']", + "button.reject-all", + "form[action*='consent'] button[type='submit']", + "button:has-text('Alle akzeptieren')", + "button:has-text('Accept all')", + "button:has-text('Alle ablehnen')", + "button:has-text('Reject all')", + "button:has-text('Akzeptieren')", + "button:has-text('Ablehnen')", + "button:has-text('Agree')", + "button:has-text('I agree')" + }; + + foreach (var sel in selectors) + { + var btn = page.Locator(sel); + if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync()) + { + var btnText = await btn.First.InnerTextAsync(); + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, + $"[{_serviceName}] Found consent button '{btnText.Trim()}' with selector '{sel}'. Clicking..."); + + // Fast click and wait for DOMContentLoaded on finance.yahoo.com + await btn.First.ClickAsync(); + try + { + await page.WaitForURLAsync(u => u.Contains("finance.yahoo.com", StringComparison.OrdinalIgnoreCase), + new PageWaitForURLOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 5_000 }); + } + catch { } + + await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, + $"[{_serviceName}] Successfully passed consent wall. Current URL: '{page.Url}'"); + break; + } } } } - catch { /* Fallback */ } + catch (Exception ex) + { + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, + $"[{_serviceName}] Exception in HandleConsentAsync."); + } } private YahooQuoteSummaryModulesDto BuildModulesDto( @@ -297,10 +446,8 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient var assetProfile = new YahooAssetProfileDto( Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null, - Industry: industry ?? GetString(allStats, "industry"), - IndustryKey: null, IndustryDisp: null, - Sector: sector ?? GetString(allStats, "sector"), - SectorKey: null, SectorDisp: null, + Industry: industry, IndustryKey: null, IndustryDisp: null, + Sector: sector, SectorKey: null, SectorDisp: null, LongBusinessSummary: description, FullTimeEmployees: fullTimeEmployees, CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null, @@ -308,100 +455,104 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient GovernanceEpochDate: null, CompensationAsOfEpochDate: null ); - var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto( - PriceToBook: GetVal(allStats, "price/book", "price / book"), - EnterpriseValue: GetVal(allStats, "enterprise value"), - ForwardPE: GetVal(allStats, "forward p/e"), - ProfitMargins: GetVal(allStats, "profit margin"), - FloatShares: GetVal(allStats, "float"), - SharesOutstanding: GetVal(allStats, "shares outstanding"), - SharesShort: GetVal(allStats, "shares short"), - SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"), - SharesShortPreviousMonthDate: null, DateShortInterest: null, - SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"), - HeldPercentInsiders: GetVal(allStats, "% held by insiders"), - HeldPercentInstitutions: GetVal(allStats, "% held by institutions"), - ShortRatio: GetVal(allStats, "short ratio"), - ShortPercentOfFloat: GetVal(allStats, "short % of float"), - Beta: GetVal(allStats, "beta (5y monthly)", "beta"), - Category: null, - BookValue: GetVal(allStats, "book value per share", "book value"), - PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"), - LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"), - NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"), - MostRecentQuarter: GetVal(allStats, "most recent quarter"), - EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"), - NetIncomeToCommon: GetVal(allStats, "net income avi to common"), - TrailingEps: GetVal(allStats, "diluted eps"), - ForwardEps: GetVal(allStats, "forward eps"), - PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"), - EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"), - EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"), - FiftyTwoWeekChange: GetVal(allStats, "52-week change"), - SandP52WeekChange: GetVal(allStats, "s&p500 52-week change") + var financialData = new YahooFinancialDataDto( + CurrentPrice: GetVal(allStats, "previous close", "current price", "price", "regular market price"), + TargetHighPrice: GetVal(allStats, "target high price", "target high"), + TargetLowPrice: GetVal(allStats, "target low price", "target low"), + TargetMeanPrice: GetVal(allStats, "1y target est", "target mean price", "target est"), + TargetMedianPrice: GetVal(allStats, "target median price"), + RecommendationMean: GetVal(allStats, "recommendation mean"), + RecommendationKey: allStats.GetValueOrDefault("recommendation") ?? allStats.GetValueOrDefault("recommendation key"), + NumberOfAnalystOpinions: GetVal(allStats, "number of analyst opinions", "analyst opinions"), + TotalCash: GetVal(allStats, "total cash", "total cash (mrq)"), + TotalCashPerShare: GetVal(allStats, "total cash per share", "total cash per share (mrq)"), + Ebitda: GetVal(allStats, "ebitda"), + TotalDebt: GetVal(allStats, "total debt", "total debt (mrq)"), + QuickRatio: GetVal(allStats, "quick ratio"), + CurrentRatio: GetVal(allStats, "current ratio", "current ratio (mrq)"), + TotalRevenue: GetVal(allStats, "total revenue", "revenue", "revenue (ttm)"), + DebtToEquity: GetVal(allStats, "total debt/equity", "total debt/equity (mrq)", "debt to equity"), + RevenuePerShare: GetVal(allStats, "revenue per share", "revenue per share (ttm)"), + ReturnOnAssets: GetVal(allStats, "return on assets", "return on assets (ttm)"), + ReturnOnEquity: GetVal(allStats, "return on equity", "return on equity (ttm)"), + GrossProfits: GetVal(allStats, "gross profit", "gross profit (ttm)", "gross profits"), + FreeCashflow: GetVal(allStats, "levered free cash flow", "levered free cash flow (ttm)", "free cash flow"), + OperatingCashflow: GetVal(allStats, "operating cash flow", "operating cash flow (ttm)"), + RevenueGrowth: GetVal(allStats, "quarterly revenue growth", "quarterly revenue growth (yoy)", "revenue growth"), + GrossMargins: GetVal(allStats, "gross margin", "gross margins"), + EbitdaMargins: GetVal(allStats, "ebitda margin", "ebitda margins"), + OperatingMargins: GetVal(allStats, "operating margin", "operating margin (ttm)", "operating margins"), + ProfitMargins: GetVal(allStats, "profit margin", "profit margins"), + FinancialCurrency: null ); - var financialData = new YahooFinancialDataDto( - CurrentPrice: GetVal(allStats, "current price", "price"), - TargetHighPrice: GetVal(allStats, "target high", "high target"), - TargetLowPrice: GetVal(allStats, "target low", "low target"), - TargetMeanPrice: GetVal(allStats, "target mean", "target est"), - TargetMedianPrice: GetVal(allStats, "target median"), - RecommendationMean: GetVal(allStats, "recommendation mean"), - RecommendationKey: GetString(allStats, "recommendation key"), - NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"), - TotalCash: GetVal(allStats, "total cash"), - TotalCashPerShare: GetVal(allStats, "total cash per share"), - Ebitda: GetVal(allStats, "ebitda"), - TotalDebt: GetVal(allStats, "total debt"), - QuickRatio: GetVal(allStats, "quick ratio"), - CurrentRatio: GetVal(allStats, "current ratio"), - TotalRevenue: GetVal(allStats, "revenue", "total revenue"), - DebtToEquity: GetVal(allStats, "total debt/equity"), - RevenuePerShare: GetVal(allStats, "revenue per share"), - ReturnOnAssets: GetVal(allStats, "return on assets"), - ReturnOnEquity: GetVal(allStats, "return on equity"), - GrossProfits: GetVal(allStats, "gross profit"), - FreeCashflow: GetVal(allStats, "levered free cash flow"), - OperatingCashflow: GetVal(allStats, "operating cash flow"), - RevenueGrowth: GetVal(allStats, "quarterly revenue growth"), - GrossMargins: GetVal(allStats, "gross margin"), - EbitdaMargins: GetVal(allStats, "ebitda margin"), - OperatingMargins: GetVal(allStats, "operating margin"), - ProfitMargins: GetVal(allStats, "profit margin"), - FinancialCurrency: "USD" + var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto( + PriceToBook: GetVal(allStats, "price/book", "price to book", "kbv", "kurs-buchwert-verhältnis"), + EnterpriseValue: GetVal(allStats, "enterprise value", "unternehmenswert"), + ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"), + ProfitMargins: GetVal(allStats, "profit margin", "gewinnmarge"), + FloatShares: GetVal(allStats, "float", "streubesitz"), + SharesOutstanding: GetVal(allStats, "shares outstanding", "ausstehende aktien"), + SharesShort: GetVal(allStats, "shares short", "leerverkaufte aktien"), + SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"), + SharesShortPreviousMonthDate: null, + DateShortInterest: null, + SharesPercentSharesOut: GetVal(allStats, "shares % of shares outstanding", "short % of shares outstanding"), + HeldPercentInsiders: GetVal(allStats, "% held by insiders", "insider anteil"), + HeldPercentInstitutions: GetVal(allStats, "% held by institutions", "institutioneller anteil"), + ShortRatio: GetVal(allStats, "short ratio"), + ShortPercentOfFloat: GetVal(allStats, "short % of float", "short percent of float"), + Beta: GetVal(allStats, "beta", "beta (5y monthly)"), + Category: null, + BookValue: GetVal(allStats, "book value per share", "book value per share (mrq)", "book value", "buchwert"), + PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price to sales", "kuv"), + LastFiscalYearEnd: null, + NextFiscalYearEnd: null, + MostRecentQuarter: null, + EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth", "quarterly earnings growth (yoy)", "earnings growth"), + NetIncomeToCommon: GetVal(allStats, "net income avi to common", "net income avi to common (ttm)", "net income avail. to common", "net income"), + TrailingEps: GetVal(allStats, "diluted eps", "diluted eps (ttm)", "trailing eps", "eps (ttm)", "gewinn je aktie"), + ForwardEps: GetVal(allStats, "forward eps"), + PegRatio: GetVal(allStats, "peg ratio (5 yr expected)", "peg ratio (5yr expected)", "peg ratio", "peg-verhältnis"), + EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue", "ev/revenue"), + EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda", "ev/ebitda"), + FiftyTwoWeekChange: GetVal(allStats, "52 week change", "52-week change", "52-wochen-änderung"), + SandP52WeekChange: GetVal(allStats, "s&p 500 52-week change", "s&p500 52-week change", "s&p 500 52 week change") ); var summaryDetail = new YahooSummaryDetailDto( - MaxAge: 86400, PriceHint: null, - PreviousClose: GetVal(allStats, "previous close"), - Open: GetVal(allStats, "open"), - DayLow: GetVal(allStats, "day low"), - DayHigh: GetVal(allStats, "day high"), - RegularMarketPreviousClose: GetVal(allStats, "previous close"), - RegularMarketOpen: GetVal(allStats, "open"), - RegularMarketDayLow: GetVal(allStats, "day low"), - RegularMarketDayHigh: GetVal(allStats, "day high"), - DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"), - DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"), - ExDividendDate: GetVal(allStats, "ex-dividend date"), - PayoutRatio: GetVal(allStats, "payout ratio"), - FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"), - Beta: GetVal(allStats, "beta"), - TrailingPE: GetVal(allStats, "trailing p/e"), - ForwardPE: GetVal(allStats, "forward p/e"), - Volume: GetVal(allStats, "volume"), - RegularMarketVolume: GetVal(allStats, "volume"), - AverageVolume: GetVal(allStats, "avg. volume", "average volume"), - AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"), - AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"), - Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"), - BidSize: null, AskSize: null, - MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"), - FiftyTwoWeekLow: GetVal(allStats, "52 week low"), - FiftyTwoWeekHigh: GetVal(allStats, "52 week high"), - PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"), - Currency: "USD" + MaxAge: null, + PriceHint: null, + PreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"), + Open: GetVal(allStats, "open", "eröffnung"), + DayLow: null, + DayHigh: null, + RegularMarketPreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"), + RegularMarketOpen: GetVal(allStats, "open", "eröffnung"), + RegularMarketDayLow: null, + RegularMarketDayHigh: null, + DividendRate: GetVal(allStats, "forward annual dividend rate", "trailing annual dividend rate", "forward dividend & yield", "dividend rate", "dividende"), + DividendYield: GetVal(allStats, "forward annual dividend yield", "trailing annual dividend yield", "dividend yield", "dividendenrendite"), + ExDividendDate: null, + PayoutRatio: GetVal(allStats, "payout ratio", "ausschüttungsquote"), + FiveYearAvgDividendYield: GetVal(allStats, "5 year average dividend yield"), + Beta: GetVal(allStats, "beta", "beta (5y monthly)"), + TrailingPE: GetVal(allStats, "pe ratio (ttm)", "trailing p/e", "p/e ratio", "pe", "trailing pe", "kgv (ttm)", "kgv"), + ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"), + Volume: GetVal(allStats, "volume", "volumen"), + RegularMarketVolume: GetVal(allStats, "volume", "volumen"), + AverageVolume: GetVal(allStats, "avg vol (3 month)", "avg. volume", "average volume", "durchschnittsvolumen"), + AverageVolume10days: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"), + AverageDailyVolume10Day: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"), + Bid: GetVal(allStats, "bid", "geld"), + Ask: GetVal(allStats, "ask", "brief"), + BidSize: null, + AskSize: null, + MarketCap: GetVal(allStats, "market cap", "market capitalization", "market cap (intraday)", "marktkapitalisierung"), + FiftyTwoWeekLow: GetVal(allStats, "52 week low", "52-week low", "52 wochen tief"), + FiftyTwoWeekHigh: GetVal(allStats, "52 week high", "52-week high", "52 wochen hoch"), + PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "kuv"), + Currency: null ); return new YahooQuoteSummaryModulesDto( @@ -420,66 +571,59 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient ); } - private static YahooValueDto? GetVal(Dictionary dict, params string[] keys) + private static YahooValueDto? GetVal(Dictionary dict, params string[] candidateKeys) { - foreach (var k in keys) + foreach (var candidate in candidateKeys) { - if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val)) - return ParseYahooValue(val); - - var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase)); - if (!string.IsNullOrWhiteSpace(match.Value)) - return ParseYahooValue(match.Value); + if (dict.TryGetValue(candidate, out var valStr) && !string.IsNullOrWhiteSpace(valStr)) + { + var parsed = ParseYahooValue(valStr); + if (parsed != null) return parsed; + } } return null; } - private static string? GetString(Dictionary dict, params string[] keys) + private static YahooValueDto? ParseYahooValue(string? raw) { - foreach (var k in keys) - { - if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val)) - return val.Trim(); + if (string.IsNullOrWhiteSpace(raw) || raw == "N/A" || raw == "--" || raw == "-") return null; - var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase)); - if (!string.IsNullOrWhiteSpace(match.Value)) - return match.Value.Trim(); - } - return null; - } - - /// - /// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln. - /// - public static YahooValueDto? ParseYahooValue(string? text) - { - if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-") - return null; - - var trimmed = text.Trim(); - bool isPercent = trimmed.EndsWith("%"); + var clean = raw.Trim().Replace(" ", "").Replace("$", "").Replace("€", "").Replace("£", ""); + var isPercent = clean.EndsWith("%"); + if (isPercent) clean = clean.TrimEnd('%'); double multiplier = 1.0; - if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0; - else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0; - else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0; - else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0; + if (clean.EndsWith("T", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000_000.0; clean = clean[..^1]; } + else if (clean.EndsWith("B", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000.0; clean = clean[..^1]; } + else if (clean.EndsWith("M", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000.0; clean = clean[..^1]; } + else if (clean.EndsWith("K", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000.0; clean = clean[..^1]; } - // Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt - var numPart = Regex.Replace(trimmed, @"[^\d.-]", ""); - - if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal)) + if (double.TryParse(clean, NumberStyles.Any, CultureInfo.InvariantCulture, out var num)) { - double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier); - return new YahooValueDto - { - Raw = finalVal, - Fmt = trimmed, - LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture) - }; + var finalRaw = num * multiplier; + if (isPercent) finalRaw /= 100.0; + return new YahooValueDto { Raw = finalRaw, Fmt = raw }; } - return null; + return new YahooValueDto { Raw = null, Fmt = raw }; + } + + private class ProfileMetaJsResult + { + public string? Sector { get; set; } + public string? Industry { get; set; } + public int? Employees { get; set; } + public string? Description { get; set; } + public List? Officers { get; set; } + } + + private class JsOfficer + { + public string? Name { get; set; } + public string? Title { get; set; } + public string? Pay { get; set; } + public string? Exercised { get; set; } + public int? YearBorn { get; set; } } private record ProfileExtractionResult( @@ -488,24 +632,5 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient string? Sector, string? Industry, int? Employees, - string? Description - ); - - private class ProfileMetaJsResult - { - public string? Sector { get; set; } - public string? Industry { get; set; } - public int? Employees { get; set; } - public string? Description { get; set; } - public List? Officers { get; set; } - } - - private class OfficerJsResult - { - public string? Name { get; set; } - public string? Title { get; set; } - public string? Pay { get; set; } - public string? Exercised { get; set; } - public int? YearBorn { get; set; } - } + string? Description); } \ No newline at end of file diff --git a/FinlyticCore/Dtos/Fundamentals/AssetHeaderDto.cs b/FinlyticCore/Dtos/Fundamentals/AssetHeaderDto.cs index 048f096..5167eff 100644 --- a/FinlyticCore/Dtos/Fundamentals/AssetHeaderDto.cs +++ b/FinlyticCore/Dtos/Fundamentals/AssetHeaderDto.cs @@ -1,4 +1,4 @@ -using System.Text.Json.Serialization; +using System.Text.Json.Serialization; namespace FinlyticCore.Dtos.Fundamentals; @@ -18,7 +18,7 @@ public record AssetHeaderDto public string Description { get; init; } = string.Empty; [JsonPropertyName("primaryTicker")] - public TickerInfoDto PrimaryTicker { get; init; } + public TickerInfoDto PrimaryTicker { get; init; } = new(); [JsonPropertyName("availableTickers")] public List AvailableTickers { get; init; } = []; diff --git a/FinlyticCore/Dtos/Fundamentals/CorporateEventDto.cs b/FinlyticCore/Dtos/Fundamentals/CorporateEventDto.cs index 94b8c7d..96dc439 100644 --- a/FinlyticCore/Dtos/Fundamentals/CorporateEventDto.cs +++ b/FinlyticCore/Dtos/Fundamentals/CorporateEventDto.cs @@ -16,7 +16,7 @@ public record CorporateEventDto public string? Isin { get; init; } [JsonPropertyName("ticker")] - public TickerInfoDto Ticker { get; init; } + public TickerInfoDto Ticker { get; init; } = new(); [JsonPropertyName("companyName")] public string? CompanyName { get; init; } diff --git a/FinlyticCore/Dtos/Fundamentals/FundamentalDataDto.cs b/FinlyticCore/Dtos/Fundamentals/FundamentalDataDto.cs index 6c92589..3db157a 100644 --- a/FinlyticCore/Dtos/Fundamentals/FundamentalDataDto.cs +++ b/FinlyticCore/Dtos/Fundamentals/FundamentalDataDto.cs @@ -9,7 +9,7 @@ namespace FinlyticCore.Dtos.Fundamentals; public record FundamentalDataDto { [JsonPropertyName("ticker")] - public TickerInfoDto Ticker { get; init; } + public TickerInfoDto Ticker { get; init; } = new(); // --- Valuation & Multiples --- [JsonPropertyName("marketCap")] diff --git a/FinlyticCore/Services/Yahoo/YahooFinanceScraper.cs b/FinlyticCore/Services/Yahoo/YahooFinanceScraper.cs index ae80248..fa5c527 100644 --- a/FinlyticCore/Services/Yahoo/YahooFinanceScraper.cs +++ b/FinlyticCore/Services/Yahoo/YahooFinanceScraper.cs @@ -32,6 +32,7 @@ public interface IYahooFinanceScraper Task GetQuoteSummaryModulesAsync( string symbolOrIsin, bool forceHtmlScrape = false, + bool includeProfile = true, CancellationToken cancellationToken = default); } @@ -98,7 +99,11 @@ public class YahooFinanceScraper : IYahooFinanceScraper } } } - catch { } + catch (Exception ex) + { + await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, + $"[{_serviceName}] Crypto search failed for {cryptoSubtitle}"); + } await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel, $"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}"); @@ -131,7 +136,7 @@ public class YahooFinanceScraper : IYahooFinanceScraper } } - // 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind + // 2. Falls Ticker gefunden, mit Unternehmensname noch mehr internationale Exchangeticker suchen (z.B. APC.DE) if (validQuotes.Count > 0) { var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName; @@ -155,16 +160,24 @@ public class YahooFinanceScraper : IYahooFinanceScraper $"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'"); } - return symbols + var result = symbols .OrderBy(s => s.priority) - .Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange }) + .Select(s => new TickerInfoDto + { + Ticker = s.symbol, + Exchange = !string.IsNullOrWhiteSpace(s.exchange) ? s.exchange : "Unknown" + }) + .DistinctBy(s => s.Ticker, StringComparer.OrdinalIgnoreCase) .ToList(); + + return result; } /// public async Task GetQuoteSummaryModulesAsync( string symbolOrIsin, bool forceHtmlScrape = false, + bool includeProfile = true, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null; @@ -220,9 +233,9 @@ public class YahooFinanceScraper : IYahooFinanceScraper try { await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, - $"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}'..."); + $"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}' (IncludeProfile: {includeProfile})..."); - htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken); + htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, includeProfile, cancellationToken); } catch (Exception ex) { @@ -246,16 +259,20 @@ public class YahooFinanceScraper : IYahooFinanceScraper modules.DefaultKeyStatistics != null; } - private static YahooQuoteSummaryModulesDto MergeModules( - YahooQuoteSummaryModulesDto primary, - YahooQuoteSummaryModulesDto secondary) + public static YahooQuoteSummaryModulesDto? MergeModules( + YahooQuoteSummaryModulesDto? primary, + YahooQuoteSummaryModulesDto? secondary) { + if (primary == null && secondary == null) return null; + if (primary == null) return secondary; + if (secondary == null) return primary; + return new YahooQuoteSummaryModulesDto( QuoteType: primary.QuoteType ?? secondary.QuoteType, AssetProfile: primary.AssetProfile ?? secondary.AssetProfile, - FinancialData: primary.FinancialData ?? secondary.FinancialData, - DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics, - SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail, + FinancialData: MergeFinancialData(primary.FinancialData, secondary.FinancialData), + DefaultKeyStatistics: MergeDefaultKeyStatistics(primary.DefaultKeyStatistics, secondary.DefaultKeyStatistics), + SummaryDetail: MergeSummaryDetail(primary.SummaryDetail, secondary.SummaryDetail), IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory, IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly, BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory, @@ -266,6 +283,124 @@ public class YahooFinanceScraper : IYahooFinanceScraper ); } + private static YahooFinancialDataDto? MergeFinancialData(YahooFinancialDataDto? a, YahooFinancialDataDto? b) + { + if (a == null) return b; + if (b == null) return a; + + return new YahooFinancialDataDto( + CurrentPrice: a.CurrentPrice ?? b.CurrentPrice, + TargetHighPrice: a.TargetHighPrice ?? b.TargetHighPrice, + TargetLowPrice: a.TargetLowPrice ?? b.TargetLowPrice, + TargetMeanPrice: a.TargetMeanPrice ?? b.TargetMeanPrice, + TargetMedianPrice: a.TargetMedianPrice ?? b.TargetMedianPrice, + RecommendationMean: a.RecommendationMean ?? b.RecommendationMean, + RecommendationKey: !string.IsNullOrWhiteSpace(a.RecommendationKey) && a.RecommendationKey != "none" ? a.RecommendationKey : b.RecommendationKey, + NumberOfAnalystOpinions: a.NumberOfAnalystOpinions ?? b.NumberOfAnalystOpinions, + TotalCash: a.TotalCash ?? b.TotalCash, + TotalCashPerShare: a.TotalCashPerShare ?? b.TotalCashPerShare, + Ebitda: a.Ebitda ?? b.Ebitda, + TotalDebt: a.TotalDebt ?? b.TotalDebt, + QuickRatio: a.QuickRatio ?? b.QuickRatio, + CurrentRatio: a.CurrentRatio ?? b.CurrentRatio, + TotalRevenue: a.TotalRevenue ?? b.TotalRevenue, + DebtToEquity: a.DebtToEquity ?? b.DebtToEquity, + RevenuePerShare: a.RevenuePerShare ?? b.RevenuePerShare, + ReturnOnAssets: a.ReturnOnAssets ?? b.ReturnOnAssets, + ReturnOnEquity: a.ReturnOnEquity ?? b.ReturnOnEquity, + GrossProfits: a.GrossProfits ?? b.GrossProfits, + FreeCashflow: a.FreeCashflow ?? b.FreeCashflow, + OperatingCashflow: a.OperatingCashflow ?? b.OperatingCashflow, + RevenueGrowth: a.RevenueGrowth ?? b.RevenueGrowth, + GrossMargins: a.GrossMargins ?? b.GrossMargins, + EbitdaMargins: a.EbitdaMargins ?? b.EbitdaMargins, + OperatingMargins: a.OperatingMargins ?? b.OperatingMargins, + ProfitMargins: a.ProfitMargins ?? b.ProfitMargins, + FinancialCurrency: a.FinancialCurrency ?? b.FinancialCurrency + ); + } + + private static YahooDefaultKeyStatisticsDto? MergeDefaultKeyStatistics(YahooDefaultKeyStatisticsDto? a, YahooDefaultKeyStatisticsDto? b) + { + if (a == null) return b; + if (b == null) return a; + + return new YahooDefaultKeyStatisticsDto( + PriceToBook: a.PriceToBook ?? b.PriceToBook, + EnterpriseValue: a.EnterpriseValue ?? b.EnterpriseValue, + ForwardPE: a.ForwardPE ?? b.ForwardPE, + ProfitMargins: a.ProfitMargins ?? b.ProfitMargins, + FloatShares: a.FloatShares ?? b.FloatShares, + SharesOutstanding: a.SharesOutstanding ?? b.SharesOutstanding, + SharesShort: a.SharesShort ?? b.SharesShort, + SharesShortPriorMonth: a.SharesShortPriorMonth ?? b.SharesShortPriorMonth, + SharesShortPreviousMonthDate: a.SharesShortPreviousMonthDate ?? b.SharesShortPreviousMonthDate, + DateShortInterest: a.DateShortInterest ?? b.DateShortInterest, + SharesPercentSharesOut: a.SharesPercentSharesOut ?? b.SharesPercentSharesOut, + HeldPercentInsiders: a.HeldPercentInsiders ?? b.HeldPercentInsiders, + HeldPercentInstitutions: a.HeldPercentInstitutions ?? b.HeldPercentInstitutions, + ShortRatio: a.ShortRatio ?? b.ShortRatio, + ShortPercentOfFloat: a.ShortPercentOfFloat ?? b.ShortPercentOfFloat, + Beta: a.Beta ?? b.Beta, + Category: a.Category ?? b.Category, + BookValue: a.BookValue ?? b.BookValue, + PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months, + LastFiscalYearEnd: a.LastFiscalYearEnd ?? b.LastFiscalYearEnd, + NextFiscalYearEnd: a.NextFiscalYearEnd ?? b.NextFiscalYearEnd, + MostRecentQuarter: a.MostRecentQuarter ?? b.MostRecentQuarter, + EarningsQuarterlyGrowth: a.EarningsQuarterlyGrowth ?? b.EarningsQuarterlyGrowth, + NetIncomeToCommon: a.NetIncomeToCommon ?? b.NetIncomeToCommon, + TrailingEps: a.TrailingEps ?? b.TrailingEps, + ForwardEps: a.ForwardEps ?? b.ForwardEps, + PegRatio: a.PegRatio ?? b.PegRatio, + EnterpriseToRevenue: a.EnterpriseToRevenue ?? b.EnterpriseToRevenue, + EnterpriseToEbitda: a.EnterpriseToEbitda ?? b.EnterpriseToEbitda, + FiftyTwoWeekChange: a.FiftyTwoWeekChange ?? b.FiftyTwoWeekChange, + SandP52WeekChange: a.SandP52WeekChange ?? b.SandP52WeekChange + ); + } + + private static YahooSummaryDetailDto? MergeSummaryDetail(YahooSummaryDetailDto? a, YahooSummaryDetailDto? b) + { + if (a == null) return b; + if (b == null) return a; + + return new YahooSummaryDetailDto( + MaxAge: a.MaxAge ?? b.MaxAge, + PriceHint: a.PriceHint ?? b.PriceHint, + PreviousClose: a.PreviousClose ?? b.PreviousClose, + Open: a.Open ?? b.Open, + DayLow: a.DayLow ?? b.DayLow, + DayHigh: a.DayHigh ?? b.DayHigh, + RegularMarketPreviousClose: a.RegularMarketPreviousClose ?? b.RegularMarketPreviousClose, + RegularMarketOpen: a.RegularMarketOpen ?? b.RegularMarketOpen, + RegularMarketDayLow: a.RegularMarketDayLow ?? b.RegularMarketDayLow, + RegularMarketDayHigh: a.RegularMarketDayHigh ?? b.RegularMarketDayHigh, + DividendRate: a.DividendRate ?? b.DividendRate, + DividendYield: a.DividendYield ?? b.DividendYield, + ExDividendDate: a.ExDividendDate ?? b.ExDividendDate, + PayoutRatio: a.PayoutRatio ?? b.PayoutRatio, + FiveYearAvgDividendYield: a.FiveYearAvgDividendYield ?? b.FiveYearAvgDividendYield, + Beta: a.Beta ?? b.Beta, + TrailingPE: a.TrailingPE ?? b.TrailingPE, + ForwardPE: a.ForwardPE ?? b.ForwardPE, + Volume: a.Volume ?? b.Volume, + RegularMarketVolume: a.RegularMarketVolume ?? b.RegularMarketVolume, + AverageVolume: a.AverageVolume ?? b.AverageVolume, + AverageVolume10days: a.AverageVolume10days ?? b.AverageVolume10days, + AverageDailyVolume10Day: a.AverageDailyVolume10Day ?? b.AverageDailyVolume10Day, + Bid: a.Bid ?? b.Bid, + Ask: a.Ask ?? b.Ask, + BidSize: a.BidSize ?? b.BidSize, + AskSize: a.AskSize ?? b.AskSize, + MarketCap: a.MarketCap ?? b.MarketCap, + FiftyTwoWeekLow: a.FiftyTwoWeekLow ?? b.FiftyTwoWeekLow, + FiftyTwoWeekHigh: a.FiftyTwoWeekHigh ?? b.FiftyTwoWeekHigh, + PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months, + Currency: a.Currency ?? b.Currency + ); + } + private static bool IsIsin(string value) { return value.Length == 12 && diff --git a/FinlyticFundamentals/Database/FundamentalsDbContext.cs b/FinlyticFundamentals/Database/FundamentalsDbContext.cs index 710439b..b5b3477 100644 --- a/FinlyticFundamentals/Database/FundamentalsDbContext.cs +++ b/FinlyticFundamentals/Database/FundamentalsDbContext.cs @@ -67,13 +67,16 @@ public class FundamentalsDbContext : DbContext, ISettingsDbContext modelBuilder.Entity(entity => { - entity.HasKey(e => e.Isin); + entity.HasKey(e => e.Id); entity.OwnsOne(e => e.Ticker, t => { t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty); t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty); + t.HasIndex(p => p.Ticker); }); + + entity.HasIndex(e => new { e.AssetDataIsin }); }); modelBuilder.Entity(entity => diff --git a/FinlyticFundamentals/Entities/FundamentalDataEntity.cs b/FinlyticFundamentals/Entities/FundamentalDataEntity.cs index 4c2dab9..b8a2a88 100644 --- a/FinlyticFundamentals/Entities/FundamentalDataEntity.cs +++ b/FinlyticFundamentals/Entities/FundamentalDataEntity.cs @@ -6,7 +6,7 @@ namespace FinlyticFundamentals.Entities; public class FundamentalDataEntity { [Key] - public string Isin { get; set; } = string.Empty; + public Guid Id { get; set; } = Guid.NewGuid(); public TickerEntity Ticker { get; set; } = new(); diff --git a/FinlyticFundamentals/Migrations/20260816103109_MakeFundamentalDataPerTicker.Designer.cs b/FinlyticFundamentals/Migrations/20260816103109_MakeFundamentalDataPerTicker.Designer.cs new file mode 100644 index 0000000..acaf995 --- /dev/null +++ b/FinlyticFundamentals/Migrations/20260816103109_MakeFundamentalDataPerTicker.Designer.cs @@ -0,0 +1,433 @@ +// +using System; +using FinlyticFundamentals.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + [DbContext(typeof(FundamentalsDbContext))] + [Migration("20260816103109_MakeFundamentalDataPerTicker")] + partial class MakeFundamentalDataPerTicker + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("ServiceIdentifier") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ValueJson") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Key") + .IsUnique(); + + b.ToTable("DynamicSettings"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Isin"); + + b.ToTable("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Date") + .HasColumnType("timestamp with time zone"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("AssetEvents"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConsensusRating") + .HasColumnType("text"); + + b.Property("CurrentRatio") + .HasColumnType("numeric"); + + b.Property("DebtToEquity") + .HasColumnType("numeric"); + + b.Property("DilutedEps") + .HasColumnType("numeric"); + + b.Property("Ebitda") + .HasColumnType("numeric"); + + b.Property("EnterpriseValue") + .HasColumnType("numeric"); + + b.Property("EvToEbitda") + .HasColumnType("numeric"); + + b.Property("FiftyTwoWeekHigh") + .HasColumnType("numeric"); + + b.Property("FiftyTwoWeekLow") + .HasColumnType("numeric"); + + b.Property("ForwardDividendYield") + .HasColumnType("numeric"); + + b.Property("ForwardPe") + .HasColumnType("numeric"); + + b.Property("FreeCashFlow") + .HasColumnType("numeric"); + + b.Property("GrossProfit") + .HasColumnType("numeric"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("MarketCap") + .HasColumnType("numeric"); + + b.Property("NetIncome") + .HasColumnType("numeric"); + + b.Property("OperatingCashFlow") + .HasColumnType("numeric"); + + b.Property("OperatingIncome") + .HasColumnType("numeric"); + + b.Property("PayoutRatio") + .HasColumnType("numeric"); + + b.Property("PegRatio") + .HasColumnType("numeric"); + + b.Property("PercentHeldByInsiders") + .HasColumnType("numeric"); + + b.Property("PercentHeldByInstitutions") + .HasColumnType("numeric"); + + b.Property("PriceTargetHigh") + .HasColumnType("numeric"); + + b.Property("PriceTargetLow") + .HasColumnType("numeric"); + + b.Property("PriceTargetMean") + .HasColumnType("numeric"); + + b.Property("PriceToBook") + .HasColumnType("numeric"); + + b.Property("PriceToSales") + .HasColumnType("numeric"); + + b.Property("ReturnOnAssets") + .HasColumnType("numeric"); + + b.Property("ReturnOnEquity") + .HasColumnType("numeric"); + + b.Property("RevenueGrowthYoY") + .HasColumnType("numeric"); + + b.Property("ShortPercentOfFloat") + .HasColumnType("numeric"); + + b.Property("ShortRatio") + .HasColumnType("numeric"); + + b.Property("TotalCash") + .HasColumnType("numeric"); + + b.Property("TotalDebt") + .HasColumnType("numeric"); + + b.Property("TotalRevenue") + .HasColumnType("numeric"); + + b.Property("TrailingPe") + .HasColumnType("numeric"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("FundamentalData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Payment") + .IsRequired() + .HasColumnType("text"); + + b.Property("SortOrder") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin", "SortOrder"); + + b.ToTable("KeyExecutives"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 => + { + b1.Property("AssetDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTicker"); + + b1.HasKey("AssetDataEntityIsin"); + + b1.ToTable("AssetData"); + + b1.WithOwner() + .HasForeignKey("AssetDataEntityIsin"); + }); + + b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 => + { + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b1.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Exchange"); + + b1.Property("Ticker") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Ticker"); + + b1.HasKey("Id"); + + b1.HasIndex("AssetDataIsin"); + + b1.HasIndex("Ticker"); + + b1.ToTable("Tickers", (string)null); + + b1.WithOwner() + .HasForeignKey("AssetDataIsin"); + }); + + b.Navigation("AvailableTickers"); + + b.Navigation("PrimaryTicker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("AssetEvents") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("AssetEventEntityId") + .HasColumnType("uuid"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("AssetEventEntityId"); + + b1.ToTable("AssetEvents"); + + b1.WithOwner() + .HasForeignKey("AssetEventEntityId"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("FundamentalData") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("FundamentalDataEntityId") + .HasColumnType("uuid"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("FundamentalDataEntityId"); + + b1.HasIndex("Ticker"); + + b1.ToTable("FundamentalData"); + + b1.WithOwner() + .HasForeignKey("FundamentalDataEntityId"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("KeyExecutives") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Navigation("AssetEvents"); + + b.Navigation("FundamentalData"); + + b.Navigation("KeyExecutives"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticFundamentals/Migrations/20260816103109_MakeFundamentalDataPerTicker.cs b/FinlyticFundamentals/Migrations/20260816103109_MakeFundamentalDataPerTicker.cs new file mode 100644 index 0000000..57331dc --- /dev/null +++ b/FinlyticFundamentals/Migrations/20260816103109_MakeFundamentalDataPerTicker.cs @@ -0,0 +1,68 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + /// + public partial class MakeFundamentalDataPerTicker : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_FundamentalData", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "Isin", + table: "FundamentalData"); + + migrationBuilder.AddColumn( + name: "Id", + table: "FundamentalData", + type: "uuid", + nullable: false, + defaultValueSql: "gen_random_uuid()"); + + migrationBuilder.AddPrimaryKey( + name: "PK_FundamentalData", + table: "FundamentalData", + column: "Id"); + + migrationBuilder.CreateIndex( + name: "IX_FundamentalData_Ticker", + table: "FundamentalData", + column: "Ticker"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_FundamentalData", + table: "FundamentalData"); + + migrationBuilder.DropIndex( + name: "IX_FundamentalData_Ticker", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "Id", + table: "FundamentalData"); + + migrationBuilder.AddColumn( + name: "Isin", + table: "FundamentalData", + type: "text", + nullable: false, + defaultValue: ""); + + migrationBuilder.AddPrimaryKey( + name: "PK_FundamentalData", + table: "FundamentalData", + column: "Isin"); + } + } +} diff --git a/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs b/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs index 0aa9310..815c026 100644 --- a/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs +++ b/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs @@ -97,8 +97,9 @@ namespace FinlyticFundamentals.Migrations modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => { - b.Property("Isin") - .HasColumnType("text"); + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); b.Property("AssetDataIsin") .IsRequired() @@ -212,7 +213,7 @@ namespace FinlyticFundamentals.Migrations b.Property("TrailingPe") .HasColumnType("numeric"); - b.HasKey("Isin"); + b.HasKey("Id"); b.HasIndex("AssetDataIsin"); @@ -371,8 +372,8 @@ namespace FinlyticFundamentals.Migrations b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => { - b1.Property("FundamentalDataEntityIsin") - .HasColumnType("text"); + b1.Property("FundamentalDataEntityId") + .HasColumnType("uuid"); b1.Property("Exchange") .IsRequired() @@ -388,12 +389,14 @@ namespace FinlyticFundamentals.Migrations .HasDefaultValue("") .HasColumnName("Ticker"); - b1.HasKey("FundamentalDataEntityIsin"); + b1.HasKey("FundamentalDataEntityId"); + + b1.HasIndex("Ticker"); b1.ToTable("FundamentalData"); b1.WithOwner() - .HasForeignKey("FundamentalDataEntityIsin"); + .HasForeignKey("FundamentalDataEntityId"); }); b.Navigation("AssetData"); diff --git a/FinlyticFundamentals/Program.cs b/FinlyticFundamentals/Program.cs index 3dbacaa..1e6eff8 100644 --- a/FinlyticFundamentals/Program.cs +++ b/FinlyticFundamentals/Program.cs @@ -31,8 +31,8 @@ builder.Services.AddHttpClient() AllowAutoRedirect = true }); -builder.Services.AddScoped(); -builder.Services.AddScoped(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddTransient(); // Register Application Services diff --git a/FinlyticFundamentals/Services/FundamentalsDbService.cs b/FinlyticFundamentals/Services/FundamentalsDbService.cs index 0feccb4..f695bbe 100644 --- a/FinlyticFundamentals/Services/FundamentalsDbService.cs +++ b/FinlyticFundamentals/Services/FundamentalsDbService.cs @@ -79,44 +79,53 @@ public class FundamentalsDbService : IFundamentalsDbService await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken); bool enableHtmlFallback = await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken); + bool forceHtmlFallback = + await settingsService.GetSettingAsync(SettingKeys.ForceHtmlFallback, cancellationToken); int validityDays = await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken); bool effectiveForceRefresh = forceRefresh && allowForceRefresh; await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, - "[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtmlFallback: {Html}", - cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback); + "[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtml: {Html} | ForceHtml: {ForceHtml}", + cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback, forceHtmlFallback); // 2. Entitäten aus DB laden var assetData = await context.AssetData .Include(a => a.AvailableTickers) .Include(a => a.KeyExecutives) .Include(a => a.AssetEvents) + .Include(a => a.FundamentalData) .FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken); - var fundamentalData = await context.FundamentalData - .FirstOrDefaultAsync(f => f.Isin == cleanIsin, cancellationToken); + string targetTicker = !string.IsNullOrWhiteSpace(requestedTicker) + ? requestedTicker + : (assetData?.PrimaryTicker?.Ticker ?? string.Empty); + + var fundamentalData = assetData?.FundamentalData? + .FirstOrDefault(f => !string.IsNullOrWhiteSpace(targetTicker) && string.Equals(f.Ticker.Ticker, targetTicker, StringComparison.OrdinalIgnoreCase)) + ?? (string.IsNullOrWhiteSpace(requestedTicker) ? assetData?.FundamentalData?.FirstOrDefault() : null); // 3. Prüfen, was aktualisiert werden muss bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name); bool executivesMissing = assetData == null || assetData.KeyExecutives == null || assetData.KeyExecutives.Count == 0; - bool fundamentalsExpired = fundamentalData == null || - (DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays; + bool fundamentalsMissingOrExpired = fundamentalData == null || + (fundamentalData.MarketCap == null && fundamentalData.TrailingPe == null) || + (DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays; + bool tickersCorruptOrMissing = assetData?.AvailableTickers == null || + assetData.AvailableTickers.Count == 0 || + assetData.AvailableTickers.Any(t => t.Ticker != null && t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase)); - // Wenn ein expliziter Ticker übergeben wurde und sich vom gespeicherten unterscheidet, - // müssen Asset-Daten und Fundamentals mit dem neuen Ticker neu abgerufen werden. - bool tickerChanged = !string.IsNullOrWhiteSpace(requestedTicker) - && assetData?.PrimaryTicker != null - && !string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker, - StringComparison.OrdinalIgnoreCase); + bool shouldUpdate = assetDataMissing || executivesMissing || fundamentalsMissingOrExpired || tickersCorruptOrMissing || effectiveForceRefresh || forceHtmlFallback; - bool shouldUpdateAssetData = assetDataMissing || effectiveForceRefresh || tickerChanged; - bool shouldUpdateExecutives = executivesMissing || effectiveForceRefresh; - bool shouldUpdateFundamentals = fundamentalsExpired || effectiveForceRefresh || tickerChanged; - - if (shouldUpdateAssetData || shouldUpdateExecutives || shouldUpdateFundamentals) + if (!shouldUpdate && fundamentalData != null) + { + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[FundamentalsDbService] Returning valid cached fundamental data for ISIN {Isin} (Ticker: {Ticker}, Updated: {UpdatedUtc}). External API fetch skipped.", + cleanIsin, fundamentalData.Ticker.Ticker, fundamentalData.LastUpdatedUtc.ToString("o")); + } + else { // --- STEP 1: Trade Republic Details --- TradeRepublicStockDetailsResponse? trDetails = null; @@ -150,8 +159,10 @@ public class FundamentalsDbService : IFundamentalsDbService TickerInfoDto activeQueryTicker; if (!string.IsNullOrWhiteSpace(requestedTicker)) { - var matchDto = resolvedTickers.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); - var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); + var matchDto = resolvedTickers.FirstOrDefault(t => + string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); + var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(t => + string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); if (matchDto != null) { @@ -197,12 +208,18 @@ public class FundamentalsDbService : IFundamentalsDbService yahooPrimaryTicker.Ticker, activeQueryTicker.Ticker, cleanIsin); // --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper --- + // Profile (Sektor, Industrie, Vorstände) wird gescrapt, wenn weder in DB noch in TR Vorstände/Beschreibungen vorliegen + bool hasProfileInDb = assetData != null && !string.IsNullOrWhiteSpace(assetData.Description) && assetData.KeyExecutives != null && assetData.KeyExecutives.Count > 0; + bool hasCeoInTr = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName); + bool needProfile = !hasProfileInDb && !hasCeoInTr; + YahooQuoteSummaryModulesDto? modulesDto = null; if (!string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) && activeQueryTicker.Ticker != cleanIsin) { modulesDto = await _scraper.GetQuoteSummaryModulesAsync( activeQueryTicker.Ticker, - forceHtmlScrape: false, + forceHtmlScrape: forceHtmlFallback, + includeProfile: needProfile, cancellationToken: cancellationToken); } else @@ -211,51 +228,96 @@ public class FundamentalsDbService : IFundamentalsDbService "[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", activeQueryTicker.Ticker); } - // --- Update AssetDataEntity --- - if (shouldUpdateAssetData) + // Falls der Sekundär-Ticker (z. B. APC.DE) überhaupt keine Daten liefert, nutze den PrimaryTicker (z. B. AAPL) als Fallback + if (modulesDto == null && + !string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) && + yahooPrimaryTicker.Ticker != activeQueryTicker.Ticker && + yahooPrimaryTicker.Ticker != cleanIsin) { - if (assetData == null) + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-FALLBACK-PRIMARY] Sekundär-Ticker '{Active}' lieferte keine Daten. Versuche PrimaryTicker '{Primary}'...", + activeQueryTicker.Ticker, yahooPrimaryTicker.Ticker); + + modulesDto = await _scraper.GetQuoteSummaryModulesAsync( + yahooPrimaryTicker.Ticker, + forceHtmlScrape: forceHtmlFallback, + includeProfile: needProfile, + cancellationToken: cancellationToken); + } + + // --- Update AssetDataEntity --- + if (assetData == null) + { + assetData = new AssetDataEntity { - assetData = new AssetDataEntity + Isin = cleanIsin, + PrimaryTicker = new TickerEntity { - Isin = cleanIsin, - PrimaryTicker = new TickerEntity - { - Ticker = yahooPrimaryTicker.Ticker, - Exchange = yahooPrimaryTicker.Exchange ?? "Unknown" - }, - KeyExecutives = new List(), - AssetEvents = new List() - }; - context.AssetData.Add(assetData); - } + Ticker = yahooPrimaryTicker.Ticker, + Exchange = yahooPrimaryTicker.Exchange ?? "Unknown" + }, + KeyExecutives = new List(), + AssetEvents = new List() + }; + context.AssetData.Add(assetData); + } - string trName = trDetails?.Company?.Name ?? string.Empty; - string trDescription = trDetails?.Company?.Description ?? string.Empty; + string trName = trDetails?.Company?.Name?.Trim() ?? string.Empty; + string trDescription = trDetails?.Company?.Description?.Trim() ?? string.Empty; - string fallbackName = modulesDto?.QuoteType?.ShortName - ?? modulesDto?.QuoteType?.LongName - ?? activeQueryTicker.Ticker; + string yahooName = modulesDto?.QuoteType?.LongName?.Trim() + ?? modulesDto?.QuoteType?.ShortName?.Trim() + ?? string.Empty; + string yahooDesc = modulesDto?.AssetProfile?.LongBusinessSummary?.Trim() ?? string.Empty; - assetData.Name = !string.IsNullOrWhiteSpace(trName) ? trName : fallbackName; - assetData.Description = !string.IsNullOrWhiteSpace(trDescription) - ? trDescription - : (modulesDto?.AssetProfile?.LongBusinessSummary ?? string.Empty); + // Name nur aktualisieren, wenn ein echter Name vorliegt (Bestandsdaten niemals mit ISIN/Ticker überschreiben) + if (!string.IsNullOrWhiteSpace(trName)) + { + assetData.Name = trName; + } + else if (!string.IsNullOrWhiteSpace(yahooName)) + { + assetData.Name = yahooName; + } + else if (string.IsNullOrWhiteSpace(assetData.Name)) + { + assetData.Name = !string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) ? activeQueryTicker.Ticker : cleanIsin; + } - // PrimaryTicker ist FEST der erste von Yahoo Finance + // Description nur aktualisieren, wenn neue Beschreibung vorhanden ist + if (!string.IsNullOrWhiteSpace(trDescription)) + { + assetData.Description = trDescription; + } + else if (!string.IsNullOrWhiteSpace(yahooDesc)) + { + assetData.Description = yahooDesc; + } + + // PrimaryTicker aktualisieren falls vorhanden + if (!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) && yahooPrimaryTicker.Ticker != cleanIsin) + { assetData.PrimaryTicker = new TickerEntity { Ticker = yahooPrimaryTicker.Ticker, Exchange = yahooPrimaryTicker.Exchange ?? "Unknown" }; + } - if (!resolvedTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase))) + // AvailableTickers aktualisieren (nur echte Börsenticker, keine ISINs) + var validTickers = resolvedTickers + .Where(t => !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (validTickers.Count > 0) + { + if (!validTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase))) { - resolvedTickers.Insert(0, yahooPrimaryTicker); + validTickers.Insert(0, yahooPrimaryTicker); } assetData.AvailableTickers.Clear(); - foreach (var a in resolvedTickers) + foreach (var a in validTickers) { assetData.AvailableTickers.Add(new TickerEntity { @@ -263,21 +325,19 @@ public class FundamentalsDbService : IFundamentalsDbService Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker) }); } - - await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, - "[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}'", - assetData.Name, assetData.PrimaryTicker.Ticker); } + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}' | AvailableTickers: {Count}", + assetData.Name, assetData.PrimaryTicker?.Ticker ?? "NULL", assetData.AvailableTickers.Count); + // --- Process Trade Republic Corporate Events --- - if (trDetails != null && (shouldUpdateAssetData || effectiveForceRefresh) && assetData != null) + if (trDetails != null && assetData != null) { - // 1. Alte Events direkt in der DB löschen (bypasses Change Tracker) await context.AssetEvents .Where(e => e.AssetDataIsin == cleanIsin) .ExecuteDeleteAsync(cancellationToken); - // 2. ALLE tracked AssetEventEntity-Einträge aus dem Change Tracker entfernen foreach (var entry in context.ChangeTracker.Entries() .Where(e => e.Entity.AssetDataIsin == cleanIsin) .ToList()) @@ -285,7 +345,6 @@ public class FundamentalsDbService : IFundamentalsDbService entry.State = EntityState.Detached; } - // 3. Navigation-Collection zurücksetzen assetData.AssetEvents = new List(); var trEventList = new List(); @@ -324,18 +383,18 @@ public class FundamentalsDbService : IFundamentalsDbService } // --- Process Modules DTO (Executives & Fundamental Data) --- - if (modulesDto != null) + if (modulesDto != null || trDetails?.Company != null) { - // Update KeyExecutives - if (shouldUpdateExecutives && assetData != null) + // Update KeyExecutives wenn Executives aus TR oder Yahoo vorliegen + var yahooOfficers = modulesDto?.AssetProfile?.CompanyOfficers; + bool hasTrOfficers = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName); + + if (((yahooOfficers != null && yahooOfficers.Count > 0) || hasTrOfficers) && assetData != null) { - // 1. Alte Executives direkt in der DB löschen (bypasses Change Tracker) await context.KeyExecutives .Where(e => e.AssetDataIsin == cleanIsin) .ExecuteDeleteAsync(cancellationToken); - // 2. ALLE tracked KeyExecutiveEntity-Einträge aus dem Change Tracker entfernen - // (nicht nur die in der Navigation-Collection — der Tracker kann mehr halten) foreach (var entry in context.ChangeTracker.Entries() .Where(e => e.Entity.AssetDataIsin == cleanIsin) .ToList()) @@ -343,14 +402,12 @@ public class FundamentalsDbService : IFundamentalsDbService entry.State = EntityState.Detached; } - // 3. Navigation-Collection zurücksetzen assetData.KeyExecutives = new List(); - // 4. Neue Executives aufbauen und direkt über den DbSet hinzufügen - if (modulesDto.AssetProfile?.CompanyOfficers != null) + if (yahooOfficers != null && yahooOfficers.Count > 0) { int sortIdx = 0; - foreach (var officer in modulesDto.AssetProfile.CompanyOfficers) + foreach (var officer in yahooOfficers) { if (!string.IsNullOrWhiteSpace(officer.Name)) { @@ -368,6 +425,49 @@ public class FundamentalsDbService : IFundamentalsDbService } } } + else if (hasTrOfficers && trDetails?.Company != null) + { + int sortIdx = 0; + if (!string.IsNullOrWhiteSpace(trDetails.Company.CeoName)) + { + var ceo = new KeyExecutiveEntity + { + AssetDataIsin = cleanIsin, + Name = trDetails.Company.CeoName, + Title = "CEO", + Payment = string.Empty, + SortOrder = sortIdx++ + }; + context.KeyExecutives.Add(ceo); + assetData.KeyExecutives.Add(ceo); + } + if (!string.IsNullOrWhiteSpace(trDetails.Company.CfoName)) + { + var cfo = new KeyExecutiveEntity + { + AssetDataIsin = cleanIsin, + Name = trDetails.Company.CfoName, + Title = "CFO", + Payment = string.Empty, + SortOrder = sortIdx++ + }; + context.KeyExecutives.Add(cfo); + assetData.KeyExecutives.Add(cfo); + } + if (!string.IsNullOrWhiteSpace(trDetails.Company.CooName)) + { + var coo = new KeyExecutiveEntity + { + AssetDataIsin = cleanIsin, + Name = trDetails.Company.CooName, + Title = "COO", + Payment = string.Empty, + SortOrder = sortIdx++ + }; + context.KeyExecutives.Add(coo); + assetData.KeyExecutives.Add(coo); + } + } await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, "[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.", @@ -375,16 +475,23 @@ public class FundamentalsDbService : IFundamentalsDbService } // Update FundamentalDataEntity - if (shouldUpdateFundamentals) + if (modulesDto != null && (modulesDto.SummaryDetail != null || modulesDto.DefaultKeyStatistics != null || modulesDto.FinancialData != null)) { + if (fundamentalData == null || !string.Equals(fundamentalData.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase)) + { + fundamentalData = assetData?.FundamentalData? + .FirstOrDefault(f => string.Equals(f.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase)); + } + if (fundamentalData == null) { fundamentalData = new FundamentalDataEntity { - Isin = cleanIsin, + Id = Guid.NewGuid(), AssetDataIsin = cleanIsin }; context.FundamentalData.Add(fundamentalData); + assetData?.FundamentalData.Add(fundamentalData); } fundamentalData.Ticker = new TickerEntity @@ -542,7 +649,7 @@ public class FundamentalsDbService : IFundamentalsDbService : (assetData.PrimaryTicker != null ? new List { assetData.PrimaryTicker } : new List()); var tickerDtos = tickerEntities - .Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker)) + .Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(assetData.Isin, StringComparison.OrdinalIgnoreCase)) .Select(a => new TickerInfoDto { Ticker = a.Ticker, diff --git a/FinlyticFundamentals/Util/SettingKeys.cs b/FinlyticFundamentals/Util/SettingKeys.cs index a244bfd..d61bce9 100644 --- a/FinlyticFundamentals/Util/SettingKeys.cs +++ b/FinlyticFundamentals/Util/SettingKeys.cs @@ -13,6 +13,7 @@ public class SettingKeys // --- Features & Toggles --- public static readonly SettingKey EnableHtmlFallback = new("Feature.EnableHtmlFallback", true); + public static readonly SettingKey ForceHtmlFallback = new("Scraper.ForceHtmlFallback", false); public static readonly SettingKey AllowForceRefresh = new("Feature.AllowForceRefresh", true); public static readonly SettingKey FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30); } \ No newline at end of file diff --git a/FinlyticTrades/Services/TradeLifecycleService.cs b/FinlyticTrades/Services/TradeLifecycleService.cs index 088b4ae..911a607 100644 --- a/FinlyticTrades/Services/TradeLifecycleService.cs +++ b/FinlyticTrades/Services/TradeLifecycleService.cs @@ -154,58 +154,63 @@ public class TradeLifecycleService : ITradeLifecycleService { string targetUserId = !string.IsNullOrWhiteSpace(request.UserId) ? request.UserId : "default_user"; - var existingTrade = await _dbContext.Trades + // 1. Prüfen, ob DIESER spezifische Nutzer diesen Trade/AnalysisId bereits als aktiven Trade angenommen hat + var userExistingTrade = await _dbContext.Trades .FirstOrDefaultAsync(t => - (!string.IsNullOrEmpty(request.TradeId) && t.TradeId == request.TradeId) || - (!string.IsNullOrEmpty(request.AnalysisId) && t.AnalysisId == request.AnalysisId), cancellationToken); + !t.IsGlobalProposal && + t.UserId == targetUserId && + ((!string.IsNullOrEmpty(request.TradeId) && t.TradeId == request.TradeId) || + (!string.IsNullOrEmpty(request.AnalysisId) && t.AnalysisId == request.AnalysisId)), + cancellationToken); - if (existingTrade != null) + if (userExistingTrade != null) { - if (existingTrade.Status == TradeStatus.Closed) + if (userExistingTrade.Status == TradeStatus.Closed) { - await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Refused to accept trade {TradeId} because its status is CLOSED", existingTrade.TradeId); + await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Refused to accept trade {TradeId} because user's trade is already CLOSED", userExistingTrade.TradeId); return null; } - existingTrade.Status = TradeStatus.Active; - existingTrade.IsGlobalProposal = false; - existingTrade.UserId = targetUserId; - - if (request.ActualEntryPrice > 0) existingTrade.ActualEntryPrice = request.ActualEntryPrice; - if (request.EntryPrice > 0) existingTrade.EntryPrice = request.EntryPrice.Value; - if (request.PositionSize > 0) existingTrade.PositionSize = request.PositionSize; - if (request.LeverageUsed > 0) existingTrade.LeverageUsed = request.LeverageUsed; - if (request.Quantity > 0) existingTrade.Quantity = request.Quantity; - if (request.EntryFee.HasValue) existingTrade.EntryFee = request.EntryFee; - if (request.ExitFee.HasValue) existingTrade.ExitFee = request.ExitFee; - if (request.StopLoss > 0) existingTrade.StopLoss = request.StopLoss.Value; - if (request.TakeProfit > 0) existingTrade.TakeProfit = request.TakeProfit.Value; - if (request.KnockoutThreshold > 0) existingTrade.KnockoutThreshold = request.KnockoutThreshold; - if (!string.IsNullOrWhiteSpace(request.Timeframe)) existingTrade.Timeframe = request.Timeframe; - if (!string.IsNullOrWhiteSpace(request.DerivativeIsin)) existingTrade.DerivativeIsin = request.DerivativeIsin; - if (!string.IsNullOrWhiteSpace(request.Reasoning)) existingTrade.Reasoning = request.Reasoning; + // Bestehenden User-Trade mit neuen Parametern aktualisieren + if (request.ActualEntryPrice > 0) userExistingTrade.ActualEntryPrice = request.ActualEntryPrice; + if (request.EntryPrice > 0) userExistingTrade.EntryPrice = request.EntryPrice.Value; + if (request.PositionSize > 0) userExistingTrade.PositionSize = request.PositionSize; + if (request.LeverageUsed > 0) userExistingTrade.LeverageUsed = request.LeverageUsed; + if (request.Quantity > 0) userExistingTrade.Quantity = request.Quantity; + if (request.EntryFee.HasValue) userExistingTrade.EntryFee = request.EntryFee; + if (request.ExitFee.HasValue) userExistingTrade.ExitFee = request.ExitFee; + if (request.StopLoss > 0) userExistingTrade.StopLoss = request.StopLoss.Value; + if (request.TakeProfit > 0) userExistingTrade.TakeProfit = request.TakeProfit.Value; + if (request.KnockoutThreshold > 0) userExistingTrade.KnockoutThreshold = request.KnockoutThreshold; + if (!string.IsNullOrWhiteSpace(request.Timeframe)) userExistingTrade.Timeframe = request.Timeframe; + if (!string.IsNullOrWhiteSpace(request.DerivativeIsin)) userExistingTrade.DerivativeIsin = request.DerivativeIsin; + if (!string.IsNullOrWhiteSpace(request.Reasoning)) userExistingTrade.Reasoning = request.Reasoning; - existingTrade.ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow; + userExistingTrade.ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow; - existingTrade.PnlAbsolute = -(existingTrade.EntryFee ?? 0m) - (existingTrade.ExitFee ?? 0m); - if (existingTrade.PositionSize > 0) + userExistingTrade.PnlAbsolute = -(userExistingTrade.EntryFee ?? 0m) - (userExistingTrade.ExitFee ?? 0m); + if (userExistingTrade.PositionSize > 0) { - existingTrade.PnlPercent = (existingTrade.PnlAbsolute / existingTrade.PositionSize) * 100m; + userExistingTrade.PnlPercent = (userExistingTrade.PnlAbsolute / userExistingTrade.PositionSize) * 100m; } - _dbContext.Trades.Update(existingTrade); + _dbContext.Trades.Update(userExistingTrade); await _dbContext.SaveChangesAsync(cancellationToken); - await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully ACCEPTED and UPDATED trade {TradeId} for ISIN {Isin}, UserId: {UserId}", existingTrade.TradeId, existingTrade.Isin, existingTrade.UserId); - return existingTrade; + await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully UPDATED existing trade {TradeId} for ISIN {Isin}, UserId: {UserId}", userExistingTrade.TradeId, userExistingTrade.Isin, userExistingTrade.UserId); + return userExistingTrade; } + // 2. Globalen Trade-Vorschlag finden (dieser bleibt unverändert in der DB, damit andere Nutzer ihn ebenfalls annehmen können) var proposal = await _dbContext.Trades - .FirstOrDefaultAsync(t => t.IsGlobalProposal && - (!string.IsNullOrEmpty(request.AnalysisId) ? t.AnalysisId == request.AnalysisId : t.Isin == request.Isin), + .FirstOrDefaultAsync(t => + (t.IsGlobalProposal || t.Status == TradeStatus.Proposed) && + ((!string.IsNullOrEmpty(request.AnalysisId) && t.AnalysisId == request.AnalysisId) || + (!string.IsNullOrEmpty(request.TradeId) && t.TradeId == request.TradeId) || + (!string.IsNullOrEmpty(request.Isin) && t.Isin == request.Isin)), cancellationToken); - var targetTradeId = !string.IsNullOrWhiteSpace(request.TradeId) ? request.TradeId : ("TRD-" + Guid.NewGuid().ToString("N")[..10].ToUpperInvariant()); + var targetTradeId = "TRD-" + Guid.NewGuid().ToString("N")[..10].ToUpperInvariant(); var newTrade = new TradeEntity { @@ -248,9 +253,10 @@ public class TradeLifecycleService : ITradeLifecycleService EntryFee = request.EntryFee, ExitFee = request.ExitFee, ExecutionTimestamp = request.ExecutionTimestamp?.ToUniversalTime() ?? DateTime.UtcNow, - Quantity = request.Quantity, + Quantity = request.Quantity > 0 ? request.Quantity : 1m, KnockoutThreshold = request.KnockoutThreshold, - IsRecurring = request.IsRecurring + IsRecurring = request.IsRecurring, + DerivativeProductCategories = proposal?.DerivativeProductCategories != null ? new List(proposal.DerivativeProductCategories) : new List() }; newTrade.PnlAbsolute = -(newTrade.EntryFee ?? 0m) - (newTrade.ExitFee ?? 0m); @@ -262,58 +268,69 @@ public class TradeLifecycleService : ITradeLifecycleService _dbContext.Trades.Add(newTrade); await _dbContext.SaveChangesAsync(cancellationToken); - await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully created active trade {TradeId} for ISIN {Isin}, UserId: {UserId}", newTrade.TradeId, request.Isin, newTrade.UserId); + await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Successfully CREATED individual active trade {TradeId} for ISIN {Isin}, UserId: {UserId} from proposal {AnalysisId}", + newTrade.TradeId, newTrade.Isin, newTrade.UserId, newTrade.AnalysisId); + return newTrade; } public async Task AddHourlyUpdateAsync(TradeHourlyUpdateDto update, CancellationToken cancellationToken = default) { - var trade = await _dbContext.Trades - .FirstOrDefaultAsync(t => t.TradeId == update.TradeId || t.Id.ToString() == update.TradeId, cancellationToken); + var matchedTrades = await _dbContext.Trades + .Where(t => t.TradeId == update.TradeId || (t.AnalysisId != null && t.AnalysisId == update.TradeId) || t.Id.ToString() == update.TradeId) + .ToListAsync(cancellationToken); - if (trade == null || (trade.Status != TradeStatus.Active && trade.Status != TradeStatus.Proposed)) + if (matchedTrades.Count == 0) { - await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Cannot add hourly update: Trade {TradeId} not found or not active/proposed.", update.TradeId); + await _finlyticLogger.LogWarningAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Cannot add hourly update: No active or proposed trades found for identifier {TradeId}.", update.TradeId); return; } - var updateEntity = new TradeHourlyUpdateEntity + foreach (var trade in matchedTrades) { - TradeId = trade.Id, - Recommendation = update.Recommendation, - CurrentPrice = update.CurrentPrice, - SuggestedStopLoss = update.SuggestedStopLoss, - SuggestedTakeProfit = update.SuggestedTakeProfit, - VixValue = update.VixValue, - Reasoning = update.Reasoning, - Timestamp = update.Timestamp - }; - - _dbContext.TradeHourlyUpdates.Add(updateEntity); - - if (update.SuggestedStopLoss.HasValue && update.SuggestedStopLoss > 0) - trade.StopLoss = update.SuggestedStopLoss.Value; - if (update.SuggestedTakeProfit.HasValue && update.SuggestedTakeProfit > 0) - trade.TakeProfit = update.SuggestedTakeProfit.Value; - - if (string.Equals(update.Recommendation, "Close", StringComparison.OrdinalIgnoreCase)) - { - if (trade.IsGlobalProposal || trade.Status == TradeStatus.Proposed) + if (trade.Status != TradeStatus.Active && trade.Status != TradeStatus.Proposed) { - trade.Status = TradeStatus.Invalidated; - trade.CloseReason = "ProposalInvalidated"; - trade.ClosedAt = DateTime.UtcNow; + continue; } - else + + var updateEntity = new TradeHourlyUpdateEntity { - await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Active trade {TradeId} received Close recommendation ({Reasoning}). Trade kept Active for user action.", - trade.TradeId, update.Reasoning); + TradeId = trade.Id, + Recommendation = update.Recommendation, + CurrentPrice = update.CurrentPrice, + SuggestedStopLoss = update.SuggestedStopLoss, + SuggestedTakeProfit = update.SuggestedTakeProfit, + VixValue = update.VixValue, + Reasoning = update.Reasoning, + Timestamp = update.Timestamp + }; + + _dbContext.TradeHourlyUpdates.Add(updateEntity); + + if (update.SuggestedStopLoss.HasValue && update.SuggestedStopLoss > 0) + trade.StopLoss = update.SuggestedStopLoss.Value; + if (update.SuggestedTakeProfit.HasValue && update.SuggestedTakeProfit > 0) + trade.TakeProfit = update.SuggestedTakeProfit.Value; + + if (string.Equals(update.Recommendation, "Close", StringComparison.OrdinalIgnoreCase)) + { + if (trade.IsGlobalProposal || trade.Status == TradeStatus.Proposed) + { + trade.Status = TradeStatus.Invalidated; + trade.CloseReason = "ProposalInvalidated"; + trade.ClosedAt = DateTime.UtcNow; + } + else + { + await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Active trade {TradeId} (UserId: {UserId}) received Close recommendation ({Reasoning}). Trade kept Active for user action.", + trade.TradeId, trade.UserId, update.Reasoning); + } } } await _dbContext.SaveChangesAsync(cancellationToken); - await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Added hourly update for Trade {TradeId}. Recommendation: {Rec}, Price: {Price}", - update.TradeId, update.Recommendation, update.CurrentPrice); + await _finlyticLogger.LogInfoAsync(SettingKeys.TradesChannel, "[TradeLifecycleService] Added hourly update across {Count} matched trades for identifier {TradeId}. Rec: {Rec}, Price: {Price}", + matchedTrades.Count, update.TradeId, update.Recommendation, update.CurrentPrice); } public async Task> GetActiveTradesAsync(string? userId = null, CancellationToken cancellationToken = default)