feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings

2026-08-16 14:05:57 +02:00
parent b0f8d4b78b
commit 2ba54e8057
17 changed files with 1791 additions and 563 deletions
@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -17,14 +18,15 @@ public interface IYahooFinanceHtmlClient
 {
    Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
        string isinOrSymbol,
+        bool includeProfile = true,
        CancellationToken cancellationToken = default);
 }

 public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
 {
-    private const string _serviceName = nameof(YahooFinanceHtmlClient);
    private readonly IPlaywrightExecutionService _playwrightService;
    private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
+    private const string _serviceName = "YahooFinanceHtmlClient";

    public YahooFinanceHtmlClient(
        IPlaywrightExecutionService playwrightService,
@@ -36,55 +38,93 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient

    public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
        string isinOrSymbol,
+        bool includeProfile = true,
        CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;

        var symbol = isinOrSymbol.Trim().ToUpperInvariant();
-        await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'...");
+        await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Starting parallel fast Playwright HTML DOM scrape for symbol '{symbol}' (IncludeProfile: {includeProfile})...");

        return await _playwrightService.ExecuteInContextAsync(async context =>
        {
-            var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-            var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-            var analysisData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-
-            ProfileExtractionResult? profileResult = null;
-
+            var summaryUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/";
            var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
-            var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
            var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
-            var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/";
+            var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";

-            var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
-            var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken);
-            var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
-            var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken);
+            // 1. Initial Page: Authenticate session and pass Cookie Consent once for the entire context
+            var initialPage = await context.NewPageAsync();
+            var summaryData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

            try
            {
-                await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask);
+                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [1/2] Loading Summary & passing Consent: {summaryUrl}");
+                await initialPage.GotoAsync(summaryUrl, new PageGotoOptions
+                {
+                    WaitUntil = WaitUntilState.DOMContentLoaded,
+                    Timeout = 15_000
+                });

-                keyStatsData = await statsTask;
-                profileResult = await profileTask;
-                financialsData = await financialsTask;
-                analysisData = await analysisTask;
+                await HandleConsentAsync(initialPage);
+                await WaitForContentAsync(initialPage);
+                summaryData = await ExtractKeyValuePairsFromPageAsync(initialPage, summaryUrl);
            }
            catch (Exception ex)
            {
-                await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}.");
+                await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping initial Summary page for {symbol}.");
+            }
+            finally
+            {
+                await initialPage.CloseAsync();
+            }
+
+            // 2. Parallel Sub-Pages (Stats, Financials, and conditionally Profile)
+            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [2/2] Fetching Sub-Pages in parallel (IncludeProfile: {includeProfile})...");
+
+            var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
+            var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
+            Task<ProfileExtractionResult?> profileTask = includeProfile
+                ? ScrapeProfilePageAsync(context, profileUrl, cancellationToken)!
+                : Task.FromResult<ProfileExtractionResult?>(null);
+
+            var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            ProfileExtractionResult? profileResult = null;
+
+            try
+            {
+                await Task.WhenAll(statsTask, financialsTask, profileTask);
+
+                keyStatsData = await statsTask;
+                financialsData = await financialsTask;
+                profileResult = await profileTask;
+            }
+            catch (Exception ex)
+            {
+                await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error during parallel sub-page scrape for {symbol}.");
+            }
+
+            // Merge summary data into key stats
+            foreach (var (k, v) in summaryData)
+            {
+                if (!keyStatsData.ContainsKey(k))
+                {
+                    keyStatsData[k] = v;
+                }
            }

            var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
            var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();

-            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}");
+            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, 
+                $"[{_serviceName}] Fast HTML DOM scrape complete for '{symbol}'. Summary: {summaryData.Count}, Stats: {keyStatsData.Count}, Financials: {financialsData.Count}, Profile: {profileDict.Count}, Officers: {officers.Count}");

            return BuildModulesDto(
                keyStatsData, 
                profileDict, 
                financialsData, 
-                analysisData, 
+                new Dictionary<string, string>(), 
                officers, 
                profileResult?.Sector, 
                profileResult?.Industry, 
@@ -107,47 +147,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
            await page.GotoAsync(url, new PageGotoOptions
            {
                WaitUntil = WaitUntilState.DOMContentLoaded,
-                Timeout = 20_000
+                Timeout = 10_000
            });

-            await HandleConsentAsync(page);
-
-            var extracted = await page.EvaluateAsync<Dictionary<string, string>>(@"() => {
-                const results = {};
-                
-                const cleanKey = (str) => {
-                    return str.toLowerCase()
-                              .replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
-                              .replace(/\s*\d+\s*$/, '')
-                              .replace(/\s+/g, ' ')
-                              .trim();
-                };
-
-                document.querySelectorAll('table tr').forEach(tr => {
-                    const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
-                    if (cells.length >= 2 && cells[0] && cells[1]) {
-                        const key = cleanKey(cells[0]);
-                        const val = cells[1].replace(/\s+/g, ' ').trim();
-                        if (key && val && val !== 'N/A' && val !== '--' && val !== '-') {
-                            results[key] = val;
-                        }
-                    }
-                });
-
-                return results;
-            }");
-
-            if (extracted != null)
-            {
-                foreach (var (k, v) in extracted)
-                {
-                    targetDict[k] = v;
-                }
-            }
+            await WaitForContentAsync(page);
+            targetDict = await ExtractKeyValuePairsFromPageAsync(page, url);
        }
        catch (Exception ex)
        {
-            await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}");
+            await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping URL {url}");
        }
        finally
        {
@@ -157,7 +165,7 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
        return targetDict;
    }

-    private async Task<ProfileExtractionResult> ScrapeProfilePageAsync(
+    private async Task<ProfileExtractionResult?> ScrapeProfilePageAsync(
        IBrowserContext context,
        string url,
        CancellationToken cancellationToken)
@@ -176,15 +184,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
            await page.GotoAsync(url, new PageGotoOptions
            {
                WaitUntil = WaitUntilState.DOMContentLoaded,
-                Timeout = 20_000
+                Timeout = 10_000
            });

-            await HandleConsentAsync(page);
+            await WaitForContentAsync(page);

-            var metaInfo = await page.EvaluateAsync<ProfileMetaJsResult>(@"() => {
+            var jsonStr = await page.EvaluateAsync<string>(@"() => {
                let sector = null, industry = null, employees = null, description = null;
                
-                const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary');
+                const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary, section[data-testid=""asset-profile""] p');
                if (descEl) description = descEl.innerText.trim();

                const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
@@ -219,40 +227,51 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
                    }
                });

-                return { sector, industry, employees, description, officers };
+                return JSON.stringify({ sector, industry, employees, description, officers });
            }");

-            if (metaInfo != null)
+            if (!string.IsNullOrWhiteSpace(jsonStr))
            {
-                sector = metaInfo.Sector;
-                industry = metaInfo.Industry;
-                fullTimeEmployees = metaInfo.Employees;
-                description = metaInfo.Description;
+                var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
+                var metaInfo = JsonSerializer.Deserialize<ProfileMetaJsResult>(jsonStr, options);

-                if (metaInfo.Officers != null)
+                if (metaInfo != null)
                {
-                    foreach (var off in metaInfo.Officers)
+                    sector = metaInfo.Sector;
+                    industry = metaInfo.Industry;
+                    fullTimeEmployees = metaInfo.Employees;
+                    description = metaInfo.Description;
+
+                    if (metaInfo.Officers != null)
                    {
-                        if (!string.IsNullOrWhiteSpace(off.Name))
+                        foreach (var off in metaInfo.Officers)
                        {
-                            companyOfficers.Add(new YahooCompanyOfficerDto(
-                                Name: off.Name,
-                                Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
-                                Title: off.Title,
-                                YearBorn: off.YearBorn,
-                                FiscalYear: null,
-                                TotalPay: ParseYahooValue(off.Pay),
-                                ExercisedValue: ParseYahooValue(off.Exercised),
-                                UnexercisedValue: null
-                            ));
+                            if (!string.IsNullOrWhiteSpace(off.Name))
+                            {
+                                companyOfficers.Add(new YahooCompanyOfficerDto(
+                                    Name: off.Name,
+                                    Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
+                                    Title: off.Title,
+                                    YearBorn: off.YearBorn,
+                                    FiscalYear: null,
+                                    TotalPay: ParseYahooValue(off.Pay),
+                                    ExercisedValue: ParseYahooValue(off.Exercised),
+                                    UnexercisedValue: null
+                                ));
+                            }
                        }
                    }
+
+                    await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, 
+                        $"[{_serviceName}] HTML Profile extracted -> Sector: '{sector}', Industry: '{industry}', Employees: {fullTimeEmployees}, Officers: {companyOfficers.Count}, Desc length: {description?.Length ?? 0}");
                }
            }
+
+            profileDict = await ExtractKeyValuePairsFromPageAsync(page, url);
        }
        catch (Exception ex)
        {
-            await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}");
+            await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping Profile data from {url}");
        }
        finally
        {
@@ -262,21 +281,151 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
        return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
    }

-    private static async Task HandleConsentAsync(IPage page)
+    private static async Task WaitForContentAsync(IPage page)
    {
        try
        {
-            if (page.Url.Contains("consent.yahoo.com"))
+            await page.WaitForSelectorAsync("table tr, ul li, div[data-testid], section, main", new PageWaitForSelectorOptions
            {
-                var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button");
-                if (await consentBtn.CountAsync() > 0)
+                State = WaitForSelectorState.Attached,
+                Timeout = 2_500
+            });
+        }
+        catch { }
+    }
+
+    private async Task<Dictionary<string, string>> ExtractKeyValuePairsFromPageAsync(IPage page, string pageUrl)
+    {
+        var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+        try
+        {
+            var jsonStr = await page.EvaluateAsync<string>(@"() => {
+                const results = {};
+                
+                const cleanKey = (str) => {
+                    return str.toLowerCase()
+                              .replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
+                              .replace(/\s*\d+\s*$/, '')
+                              .replace(/\s+/g, ' ')
+                              .trim();
+                };
+
+                // 1. Standard HTML Table Rows (Financials, Balance Sheet, Key Stats)
+                document.querySelectorAll('table tr').forEach(tr => {
+                    const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
+                    if (cells.length >= 2 && cells[0] && cells[1]) {
+                        const key = cleanKey(cells[0]);
+                        const val = cells[1].replace(/\s+/g, ' ').trim();
+                        if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
+                            results[key] = val;
+                        }
+                    }
+                });
+
+                // 2. Modern Yahoo Finance List / Div pairs (Summary Table, Quote Statistics, Flex containers)
+                document.querySelectorAll('li, div[class*=""container""], div[class*=""row""], section div, div[data-testid]').forEach(el => {
+                    const children = Array.from(el.querySelectorAll(':scope > span, :scope > div, :scope > p')).map(s => s.innerText.trim()).filter(Boolean);
+                    if (children.length === 2) {
+                        const key = cleanKey(children[0]);
+                        const val = children[1].replace(/\s+/g, ' ').trim();
+                        if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
+                            if (!results[key]) {
+                                results[key] = val;
+                            }
+                        }
+                    }
+                });
+
+                return JSON.stringify(results);
+            }");
+
+            if (!string.IsNullOrWhiteSpace(jsonStr))
+            {
+                var parsed = JsonSerializer.Deserialize<Dictionary<string, string>>(jsonStr);
+                if (parsed != null)
                {
-                    await consentBtn.First.ClickAsync();
-                    await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 });
+                    foreach (var (k, v) in parsed)
+                    {
+                        targetDict[k] = v;
+                    }
+                }
+            }
+
+            var sampleKeys = targetDict.Keys.Take(6).ToList();
+            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, 
+                $"[{_serviceName}] Extracted {targetDict.Count} items from HTML of '{pageUrl}'. Sample keys: [{string.Join(", ", sampleKeys)}]");
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error extracting key-value pairs from HTML of '{pageUrl}'");
+        }
+
+        return targetDict;
+    }
+
+    private async Task HandleConsentAsync(IPage page)
+    {
+        try
+        {
+            var url = page.Url;
+            if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) || 
+                url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
+            {
+                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, 
+                    $"[{_serviceName}] Detected EU Cookie Consent redirect: '{url}'. Searching for Accept/Reject buttons...");
+
+                var selectors = new[]
+                {
+                    "button[name='agree']",
+                    "button[value='agree']",
+                    "button.accept-all",
+                    "button.btn.primary",
+                    "button.btn.secondary.accept-all",
+                    "button[name='reject']",
+                    "button[value='reject']",
+                    "button.reject-all",
+                    "form[action*='consent'] button[type='submit']",
+                    "button:has-text('Alle akzeptieren')",
+                    "button:has-text('Accept all')",
+                    "button:has-text('Alle ablehnen')",
+                    "button:has-text('Reject all')",
+                    "button:has-text('Akzeptieren')",
+                    "button:has-text('Ablehnen')",
+                    "button:has-text('Agree')",
+                    "button:has-text('I agree')"
+                };
+
+                foreach (var sel in selectors)
+                {
+                    var btn = page.Locator(sel);
+                    if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
+                    {
+                        var btnText = await btn.First.InnerTextAsync();
+                        await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, 
+                            $"[{_serviceName}] Found consent button '{btnText.Trim()}' with selector '{sel}'. Clicking...");
+
+                        // Fast click and wait for DOMContentLoaded on finance.yahoo.com
+                        await btn.First.ClickAsync();
+                        try
+                        {
+                            await page.WaitForURLAsync(u => u.Contains("finance.yahoo.com", StringComparison.OrdinalIgnoreCase), 
+                                new PageWaitForURLOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 5_000 });
+                        }
+                        catch { }
+
+                        await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, 
+                            $"[{_serviceName}] Successfully passed consent wall. Current URL: '{page.Url}'");
+                        break;
+                    }
                }
            }
        }
-        catch { /* Fallback */ }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, 
+                $"[{_serviceName}] Exception in HandleConsentAsync.");
+        }
    }

    private YahooQuoteSummaryModulesDto BuildModulesDto(
@@ -297,10 +446,8 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient

        var assetProfile = new YahooAssetProfileDto(
            Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
-            Industry: industry ?? GetString(allStats, "industry"),
-            IndustryKey: null, IndustryDisp: null,
-            Sector: sector ?? GetString(allStats, "sector"),
-            SectorKey: null, SectorDisp: null,
+            Industry: industry, IndustryKey: null, IndustryDisp: null,
+            Sector: sector, SectorKey: null, SectorDisp: null,
            LongBusinessSummary: description,
            FullTimeEmployees: fullTimeEmployees,
            CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
@@ -308,100 +455,104 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
            GovernanceEpochDate: null, CompensationAsOfEpochDate: null
        );

-        var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
-            PriceToBook: GetVal(allStats, "price/book", "price / book"),
-            EnterpriseValue: GetVal(allStats, "enterprise value"),
-            ForwardPE: GetVal(allStats, "forward p/e"),
-            ProfitMargins: GetVal(allStats, "profit margin"),
-            FloatShares: GetVal(allStats, "float"),
-            SharesOutstanding: GetVal(allStats, "shares outstanding"),
-            SharesShort: GetVal(allStats, "shares short"),
-            SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
-            SharesShortPreviousMonthDate: null, DateShortInterest: null,
-            SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"),
-            HeldPercentInsiders: GetVal(allStats, "% held by insiders"),
-            HeldPercentInstitutions: GetVal(allStats, "% held by institutions"),
-            ShortRatio: GetVal(allStats, "short ratio"),
-            ShortPercentOfFloat: GetVal(allStats, "short % of float"),
-            Beta: GetVal(allStats, "beta (5y monthly)", "beta"),
-            Category: null,
-            BookValue: GetVal(allStats, "book value per share", "book value"),
-            PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"),
-            LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"),
-            NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"),
-            MostRecentQuarter: GetVal(allStats, "most recent quarter"),
-            EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"),
-            NetIncomeToCommon: GetVal(allStats, "net income avi to common"),
-            TrailingEps: GetVal(allStats, "diluted eps"),
-            ForwardEps: GetVal(allStats, "forward eps"),
-            PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"),
-            EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"),
-            EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"),
-            FiftyTwoWeekChange: GetVal(allStats, "52-week change"),
-            SandP52WeekChange: GetVal(allStats, "s&p500 52-week change")
+        var financialData = new YahooFinancialDataDto(
+            CurrentPrice: GetVal(allStats, "previous close", "current price", "price", "regular market price"),
+            TargetHighPrice: GetVal(allStats, "target high price", "target high"),
+            TargetLowPrice: GetVal(allStats, "target low price", "target low"),
+            TargetMeanPrice: GetVal(allStats, "1y target est", "target mean price", "target est"),
+            TargetMedianPrice: GetVal(allStats, "target median price"),
+            RecommendationMean: GetVal(allStats, "recommendation mean"),
+            RecommendationKey: allStats.GetValueOrDefault("recommendation") ?? allStats.GetValueOrDefault("recommendation key"),
+            NumberOfAnalystOpinions: GetVal(allStats, "number of analyst opinions", "analyst opinions"),
+            TotalCash: GetVal(allStats, "total cash", "total cash (mrq)"),
+            TotalCashPerShare: GetVal(allStats, "total cash per share", "total cash per share (mrq)"),
+            Ebitda: GetVal(allStats, "ebitda"),
+            TotalDebt: GetVal(allStats, "total debt", "total debt (mrq)"),
+            QuickRatio: GetVal(allStats, "quick ratio"),
+            CurrentRatio: GetVal(allStats, "current ratio", "current ratio (mrq)"),
+            TotalRevenue: GetVal(allStats, "total revenue", "revenue", "revenue (ttm)"),
+            DebtToEquity: GetVal(allStats, "total debt/equity", "total debt/equity (mrq)", "debt to equity"),
+            RevenuePerShare: GetVal(allStats, "revenue per share", "revenue per share (ttm)"),
+            ReturnOnAssets: GetVal(allStats, "return on assets", "return on assets (ttm)"),
+            ReturnOnEquity: GetVal(allStats, "return on equity", "return on equity (ttm)"),
+            GrossProfits: GetVal(allStats, "gross profit", "gross profit (ttm)", "gross profits"),
+            FreeCashflow: GetVal(allStats, "levered free cash flow", "levered free cash flow (ttm)", "free cash flow"),
+            OperatingCashflow: GetVal(allStats, "operating cash flow", "operating cash flow (ttm)"),
+            RevenueGrowth: GetVal(allStats, "quarterly revenue growth", "quarterly revenue growth (yoy)", "revenue growth"),
+            GrossMargins: GetVal(allStats, "gross margin", "gross margins"),
+            EbitdaMargins: GetVal(allStats, "ebitda margin", "ebitda margins"),
+            OperatingMargins: GetVal(allStats, "operating margin", "operating margin (ttm)", "operating margins"),
+            ProfitMargins: GetVal(allStats, "profit margin", "profit margins"),
+            FinancialCurrency: null
        );

-        var financialData = new YahooFinancialDataDto(
-            CurrentPrice: GetVal(allStats, "current price", "price"),
-            TargetHighPrice: GetVal(allStats, "target high", "high target"),
-            TargetLowPrice: GetVal(allStats, "target low", "low target"),
-            TargetMeanPrice: GetVal(allStats, "target mean", "target est"),
-            TargetMedianPrice: GetVal(allStats, "target median"),
-            RecommendationMean: GetVal(allStats, "recommendation mean"),
-            RecommendationKey: GetString(allStats, "recommendation key"),
-            NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"),
-            TotalCash: GetVal(allStats, "total cash"),
-            TotalCashPerShare: GetVal(allStats, "total cash per share"),
-            Ebitda: GetVal(allStats, "ebitda"),
-            TotalDebt: GetVal(allStats, "total debt"),
-            QuickRatio: GetVal(allStats, "quick ratio"),
-            CurrentRatio: GetVal(allStats, "current ratio"),
-            TotalRevenue: GetVal(allStats, "revenue", "total revenue"),
-            DebtToEquity: GetVal(allStats, "total debt/equity"),
-            RevenuePerShare: GetVal(allStats, "revenue per share"),
-            ReturnOnAssets: GetVal(allStats, "return on assets"),
-            ReturnOnEquity: GetVal(allStats, "return on equity"),
-            GrossProfits: GetVal(allStats, "gross profit"),
-            FreeCashflow: GetVal(allStats, "levered free cash flow"),
-            OperatingCashflow: GetVal(allStats, "operating cash flow"),
-            RevenueGrowth: GetVal(allStats, "quarterly revenue growth"),
-            GrossMargins: GetVal(allStats, "gross margin"),
-            EbitdaMargins: GetVal(allStats, "ebitda margin"),
-            OperatingMargins: GetVal(allStats, "operating margin"),
-            ProfitMargins: GetVal(allStats, "profit margin"),
-            FinancialCurrency: "USD"
+        var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
+            PriceToBook: GetVal(allStats, "price/book", "price to book", "kbv", "kurs-buchwert-verhältnis"),
+            EnterpriseValue: GetVal(allStats, "enterprise value", "unternehmenswert"),
+            ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
+            ProfitMargins: GetVal(allStats, "profit margin", "gewinnmarge"),
+            FloatShares: GetVal(allStats, "float", "streubesitz"),
+            SharesOutstanding: GetVal(allStats, "shares outstanding", "ausstehende aktien"),
+            SharesShort: GetVal(allStats, "shares short", "leerverkaufte aktien"),
+            SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
+            SharesShortPreviousMonthDate: null,
+            DateShortInterest: null,
+            SharesPercentSharesOut: GetVal(allStats, "shares % of shares outstanding", "short % of shares outstanding"),
+            HeldPercentInsiders: GetVal(allStats, "% held by insiders", "insider anteil"),
+            HeldPercentInstitutions: GetVal(allStats, "% held by institutions", "institutioneller anteil"),
+            ShortRatio: GetVal(allStats, "short ratio"),
+            ShortPercentOfFloat: GetVal(allStats, "short % of float", "short percent of float"),
+            Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
+            Category: null,
+            BookValue: GetVal(allStats, "book value per share", "book value per share (mrq)", "book value", "buchwert"),
+            PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price to sales", "kuv"),
+            LastFiscalYearEnd: null,
+            NextFiscalYearEnd: null,
+            MostRecentQuarter: null,
+            EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth", "quarterly earnings growth (yoy)", "earnings growth"),
+            NetIncomeToCommon: GetVal(allStats, "net income avi to common", "net income avi to common (ttm)", "net income avail. to common", "net income"),
+            TrailingEps: GetVal(allStats, "diluted eps", "diluted eps (ttm)", "trailing eps", "eps (ttm)", "gewinn je aktie"),
+            ForwardEps: GetVal(allStats, "forward eps"),
+            PegRatio: GetVal(allStats, "peg ratio (5 yr expected)", "peg ratio (5yr expected)", "peg ratio", "peg-verhältnis"),
+            EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue", "ev/revenue"),
+            EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda", "ev/ebitda"),
+            FiftyTwoWeekChange: GetVal(allStats, "52 week change", "52-week change", "52-wochen-änderung"),
+            SandP52WeekChange: GetVal(allStats, "s&p 500 52-week change", "s&p500 52-week change", "s&p 500 52 week change")
        );

        var summaryDetail = new YahooSummaryDetailDto(
-            MaxAge: 86400, PriceHint: null,
-            PreviousClose: GetVal(allStats, "previous close"),
-            Open: GetVal(allStats, "open"),
-            DayLow: GetVal(allStats, "day low"),
-            DayHigh: GetVal(allStats, "day high"),
-            RegularMarketPreviousClose: GetVal(allStats, "previous close"),
-            RegularMarketOpen: GetVal(allStats, "open"),
-            RegularMarketDayLow: GetVal(allStats, "day low"),
-            RegularMarketDayHigh: GetVal(allStats, "day high"),
-            DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"),
-            DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"),
-            ExDividendDate: GetVal(allStats, "ex-dividend date"),
-            PayoutRatio: GetVal(allStats, "payout ratio"),
-            FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"),
-            Beta: GetVal(allStats, "beta"),
-            TrailingPE: GetVal(allStats, "trailing p/e"),
-            ForwardPE: GetVal(allStats, "forward p/e"),
-            Volume: GetVal(allStats, "volume"),
-            RegularMarketVolume: GetVal(allStats, "volume"),
-            AverageVolume: GetVal(allStats, "avg. volume", "average volume"),
-            AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"),
-            AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"),
-            Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"),
-            BidSize: null, AskSize: null,
-            MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"),
-            FiftyTwoWeekLow: GetVal(allStats, "52 week low"),
-            FiftyTwoWeekHigh: GetVal(allStats, "52 week high"),
-            PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"),
-            Currency: "USD"
+            MaxAge: null,
+            PriceHint: null,
+            PreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
+            Open: GetVal(allStats, "open", "eröffnung"),
+            DayLow: null,
+            DayHigh: null,
+            RegularMarketPreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
+            RegularMarketOpen: GetVal(allStats, "open", "eröffnung"),
+            RegularMarketDayLow: null,
+            RegularMarketDayHigh: null,
+            DividendRate: GetVal(allStats, "forward annual dividend rate", "trailing annual dividend rate", "forward dividend & yield", "dividend rate", "dividende"),
+            DividendYield: GetVal(allStats, "forward annual dividend yield", "trailing annual dividend yield", "dividend yield", "dividendenrendite"),
+            ExDividendDate: null,
+            PayoutRatio: GetVal(allStats, "payout ratio", "ausschüttungsquote"),
+            FiveYearAvgDividendYield: GetVal(allStats, "5 year average dividend yield"),
+            Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
+            TrailingPE: GetVal(allStats, "pe ratio (ttm)", "trailing p/e", "p/e ratio", "pe", "trailing pe", "kgv (ttm)", "kgv"),
+            ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
+            Volume: GetVal(allStats, "volume", "volumen"),
+            RegularMarketVolume: GetVal(allStats, "volume", "volumen"),
+            AverageVolume: GetVal(allStats, "avg vol (3 month)", "avg. volume", "average volume", "durchschnittsvolumen"),
+            AverageVolume10days: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
+            AverageDailyVolume10Day: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
+            Bid: GetVal(allStats, "bid", "geld"),
+            Ask: GetVal(allStats, "ask", "brief"),
+            BidSize: null,
+            AskSize: null,
+            MarketCap: GetVal(allStats, "market cap", "market capitalization", "market cap (intraday)", "marktkapitalisierung"),
+            FiftyTwoWeekLow: GetVal(allStats, "52 week low", "52-week low", "52 wochen tief"),
+            FiftyTwoWeekHigh: GetVal(allStats, "52 week high", "52-week high", "52 wochen hoch"),
+            PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "kuv"),
+            Currency: null
        );

        return new YahooQuoteSummaryModulesDto(
@@ -420,66 +571,59 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
        );
    }

-    private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] keys)
+    private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] candidateKeys)
    {
-        foreach (var k in keys)
+        foreach (var candidate in candidateKeys)
        {
-            if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
-                return ParseYahooValue(val);
-
-            var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
-            if (!string.IsNullOrWhiteSpace(match.Value))
-                return ParseYahooValue(match.Value);
+            if (dict.TryGetValue(candidate, out var valStr) && !string.IsNullOrWhiteSpace(valStr))
+            {
+                var parsed = ParseYahooValue(valStr);
+                if (parsed != null) return parsed;
+            }
        }
        return null;
    }

-    private static string? GetString(Dictionary<string, string> dict, params string[] keys)
+    private static YahooValueDto? ParseYahooValue(string? raw)
    {
-        foreach (var k in keys)
-        {
-            if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
-                return val.Trim();
+        if (string.IsNullOrWhiteSpace(raw) || raw == "N/A" || raw == "--" || raw == "-") return null;

-            var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
-            if (!string.IsNullOrWhiteSpace(match.Value))
-                return match.Value.Trim();
-        }
-        return null;
-    }
-
-    /// <summary>
-    /// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln.
-    /// </summary>
-    public static YahooValueDto? ParseYahooValue(string? text)
-    {
-        if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-")
-            return null;
-
-        var trimmed = text.Trim();
-        bool isPercent = trimmed.EndsWith("%");
+        var clean = raw.Trim().Replace(" ", "").Replace("$", "").Replace("€", "").Replace("£", "");
+        var isPercent = clean.EndsWith("%");
+        if (isPercent) clean = clean.TrimEnd('%');

        double multiplier = 1.0;
-        if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0;
-        else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0;
-        else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0;
-        else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0;
+        if (clean.EndsWith("T", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000_000.0; clean = clean[..^1]; }
+        else if (clean.EndsWith("B", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000.0; clean = clean[..^1]; }
+        else if (clean.EndsWith("M", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000.0; clean = clean[..^1]; }
+        else if (clean.EndsWith("K", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000.0; clean = clean[..^1]; }

-        // Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt
-        var numPart = Regex.Replace(trimmed, @"[^\d.-]", "");
-
-        if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal))
+        if (double.TryParse(clean, NumberStyles.Any, CultureInfo.InvariantCulture, out var num))
        {
-            double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier);
-            return new YahooValueDto
-            {
-                Raw = finalVal,
-                Fmt = trimmed,
-                LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture)
-            };
+            var finalRaw = num * multiplier;
+            if (isPercent) finalRaw /= 100.0;
+            return new YahooValueDto { Raw = finalRaw, Fmt = raw };
        }

-        return null;
+        return new YahooValueDto { Raw = null, Fmt = raw };
+    }
+
+    private class ProfileMetaJsResult
+    {
+        public string? Sector { get; set; }
+        public string? Industry { get; set; }
+        public int? Employees { get; set; }
+        public string? Description { get; set; }
+        public List<JsOfficer>? Officers { get; set; }
+    }
+
+    private class JsOfficer
+    {
+        public string? Name { get; set; }
+        public string? Title { get; set; }
+        public string? Pay { get; set; }
+        public string? Exercised { get; set; }
+        public int? YearBorn { get; set; }
    }

    private record ProfileExtractionResult(
@@ -488,24 +632,5 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
        string? Sector,
        string? Industry,
        int? Employees,
-        string? Description
-    );
-
-    private class ProfileMetaJsResult
-    {
-        public string? Sector { get; set; }
-        public string? Industry { get; set; }
-        public int? Employees { get; set; }
-        public string? Description { get; set; }
-        public List<OfficerJsResult>? Officers { get; set; }
-    }
-
-    private class OfficerJsResult
-    {
-        public string? Name { get; set; }
-        public string? Title { get; set; }
-        public string? Pay { get; set; }
-        public string? Exercised { get; set; }
-        public int? YearBorn { get; set; }
-    }
+        string? Description);
 }