feat(core): update DTOs, Trade Republic client, Yahoo scrapers, and dynamic settings

2026-08-14 23:55:02 +02:00
parent 3d8af3940b
commit 4f733bf8c3
511 changed files with 1990 additions and 1080548 deletions
@@ -0,0 +1,519 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Dtos.Yahoo;
+using FinlyticCore.Models.Settings;
+using FinlyticCore.Services;
+using FinlyticCore.Services.PlaywrightScrapper;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Playwright;
+
+namespace FinlyticCore.Clients;
+
+public interface IYahooFinanceHtmlClient
+{
+    Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
+        string isinOrSymbol,
+        CancellationToken cancellationToken = default);
+}
+
+public interface IYahooFinanceHtmlClient<TDbContext> : IYahooFinanceHtmlClient
+    where TDbContext : DbContext
+{
+}
+
+public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHtmlClient<TDbContext>
+    where TDbContext : DbContext
+{
+    private readonly IPlaywrightExecutionService _playwrightService;
+    private readonly IFinlyticLogger<TContextClass, TDbContext> _finlyticLogger;
+    private readonly string _serviceName;
+
+    public YahooFinanceHtmlClient(
+        IPlaywrightExecutionService playwrightService,
+        IFinlyticLogger<TContextClass, TDbContext> finlyticLogger)
+    {
+        _playwrightService = playwrightService;
+        _finlyticLogger = finlyticLogger;
+        _serviceName = typeof(TContextClass).Name;
+    }
+
+    public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
+        string isinOrSymbol,
+        CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;
+
+        var symbol = isinOrSymbol.Trim().ToUpperInvariant();
+        await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'...");
+
+        return await _playwrightService.ExecuteInContextAsync(async context =>
+        {
+            var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            var analysisData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+            ProfileExtractionResult? profileResult = null;
+
+            var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
+            var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
+            var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
+            var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/";
+
+            var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
+            var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken);
+            var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
+            var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken);
+
+            try
+            {
+                await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask);
+
+                keyStatsData = await statsTask;
+                profileResult = await profileTask;
+                financialsData = await financialsTask;
+                analysisData = await analysisTask;
+            }
+            catch (Exception ex)
+            {
+                await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}.");
+            }
+
+            var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();
+
+            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}");
+
+            return BuildModulesDto(
+                keyStatsData, 
+                profileDict, 
+                financialsData, 
+                analysisData, 
+                officers, 
+                profileResult?.Sector, 
+                profileResult?.Industry, 
+                profileResult?.Employees, 
+                profileResult?.Description);
+
+        }, PlaywrightBrowserFactory.GetDefaultContextOptions(), cancellationToken);
+    }
+
+    private async Task<Dictionary<string, string>> ScrapePagePairsAsync(
+        IBrowserContext context,
+        string url,
+        CancellationToken cancellationToken)
+    {
+        var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        var page = await context.NewPageAsync();
+
+        try
+        {
+            await page.GotoAsync(url, new PageGotoOptions
+            {
+                WaitUntil = WaitUntilState.DOMContentLoaded,
+                Timeout = 20_000
+            });
+
+            await HandleConsentAsync(page);
+
+            var extracted = await page.EvaluateAsync<Dictionary<string, string>>(@"() => {
+                const results = {};
+                
+                const cleanKey = (str) => {
+                    return str.toLowerCase()
+                              .replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
+                              .replace(/\s*\d+\s*$/, '')
+                              .replace(/\s+/g, ' ')
+                              .trim();
+                };
+
+                document.querySelectorAll('table tr').forEach(tr => {
+                    const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
+                    if (cells.length >= 2 && cells[0] && cells[1]) {
+                        const key = cleanKey(cells[0]);
+                        const val = cells[1].replace(/\s+/g, ' ').trim();
+                        if (key && val && val !== 'N/A' && val !== '--' && val !== '-') {
+                            results[key] = val;
+                        }
+                    }
+                });
+
+                return results;
+            }");
+
+            if (extracted != null)
+            {
+                foreach (var (k, v) in extracted)
+                {
+                    targetDict[k] = v;
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}");
+        }
+        finally
+        {
+            await page.CloseAsync();
+        }
+
+        return targetDict;
+    }
+
+    private async Task<ProfileExtractionResult> ScrapeProfilePageAsync(
+        IBrowserContext context,
+        string url,
+        CancellationToken cancellationToken)
+    {
+        var profileDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        var companyOfficers = new List<YahooCompanyOfficerDto>();
+        string? sector = null;
+        string? industry = null;
+        int? fullTimeEmployees = null;
+        string? description = null;
+
+        var page = await context.NewPageAsync();
+
+        try
+        {
+            await page.GotoAsync(url, new PageGotoOptions
+            {
+                WaitUntil = WaitUntilState.DOMContentLoaded,
+                Timeout = 20_000
+            });
+
+            await HandleConsentAsync(page);
+
+            var metaInfo = await page.EvaluateAsync<ProfileMetaJsResult>(@"() => {
+                let sector = null, industry = null, employees = null, description = null;
+                
+                const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary');
+                if (descEl) description = descEl.innerText.trim();
+
+                const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
+                if (profileSec) {
+                    const text = profileSec.innerText;
+                    const sectorMatch = text.match(/Sector\(s\)\s*:?\s*([^\n\r]+)/i) || text.match(/Sector\s*:?\s*([^\n\r]+)/i);
+                    if (sectorMatch) sector = sectorMatch[1].trim();
+
+                    const indMatch = text.match(/Industry\s*:?\s*([^\n\r]+)/i);
+                    if (indMatch) industry = indMatch[1].trim();
+
+                    const empMatch = text.match(/Full Time Employees\s*:?\s*([\d,]+)/i);
+                    if (empMatch) {
+                        const cleanNum = empMatch[1].replace(/,/g, '');
+                        employees = parseInt(cleanNum, 10);
+                    }
+                }
+
+                const officers = [];
+                const officerRows = document.querySelectorAll('section[data-testid=""asset-profile""] table tr, table.officers tr, main table tr');
+                officerRows.forEach((tr, index) => {
+                    if (index === 0) return;
+                    const tds = Array.from(tr.querySelectorAll('td')).map(td => td.innerText.trim());
+                    if (tds.length >= 2) {
+                        officers.push({
+                            name: tds[0] || null,
+                            title: tds[1] || null,
+                            pay: tds[2] || null,
+                            exercised: tds[3] || null,
+                            yearBorn: tds[4] ? parseInt(tds[4], 10) : null
+                        });
+                    }
+                });
+
+                return { sector, industry, employees, description, officers };
+            }");
+
+            if (metaInfo != null)
+            {
+                sector = metaInfo.Sector;
+                industry = metaInfo.Industry;
+                fullTimeEmployees = metaInfo.Employees;
+                description = metaInfo.Description;
+
+                if (metaInfo.Officers != null)
+                {
+                    foreach (var off in metaInfo.Officers)
+                    {
+                        if (!string.IsNullOrWhiteSpace(off.Name))
+                        {
+                            companyOfficers.Add(new YahooCompanyOfficerDto(
+                                Name: off.Name,
+                                Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
+                                Title: off.Title,
+                                YearBorn: off.YearBorn,
+                                FiscalYear: null,
+                                TotalPay: ParseYahooValue(off.Pay),
+                                ExercisedValue: ParseYahooValue(off.Exercised),
+                                UnexercisedValue: null
+                            ));
+                        }
+                    }
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}");
+        }
+        finally
+        {
+            await page.CloseAsync();
+        }
+
+        return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
+    }
+
+    private static async Task HandleConsentAsync(IPage page)
+    {
+        try
+        {
+            if (page.Url.Contains("consent.yahoo.com"))
+            {
+                var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button");
+                if (await consentBtn.CountAsync() > 0)
+                {
+                    await consentBtn.First.ClickAsync();
+                    await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 });
+                }
+            }
+        }
+        catch { /* Fallback */ }
+    }
+
+    private YahooQuoteSummaryModulesDto BuildModulesDto(
+        Dictionary<string, string> keyStatsData,
+        Dictionary<string, string> profileData,
+        Dictionary<string, string> financialsData,
+        Dictionary<string, string> analysisData,
+        List<YahooCompanyOfficerDto> companyOfficers,
+        string? sector,
+        string? industry,
+        int? fullTimeEmployees,
+        string? description)
+    {
+        var allStats = new Dictionary<string, string>(keyStatsData, StringComparer.OrdinalIgnoreCase);
+        foreach (var (k, v) in profileData) allStats[k] = v;
+        foreach (var (k, v) in financialsData) allStats[k] = v;
+        foreach (var (k, v) in analysisData) allStats[k] = v;
+
+        var assetProfile = new YahooAssetProfileDto(
+            Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
+            Industry: industry ?? GetString(allStats, "industry"),
+            IndustryKey: null, IndustryDisp: null,
+            Sector: sector ?? GetString(allStats, "sector"),
+            SectorKey: null, SectorDisp: null,
+            LongBusinessSummary: description,
+            FullTimeEmployees: fullTimeEmployees,
+            CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
+            AuditRisk: null, BoardRisk: null, CompensationRisk: null, ShareHolderRightsRisk: null, OverallRisk: null,
+            GovernanceEpochDate: null, CompensationAsOfEpochDate: null
+        );
+
+        var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
+            PriceToBook: GetVal(allStats, "price/book", "price / book"),
+            EnterpriseValue: GetVal(allStats, "enterprise value"),
+            ForwardPE: GetVal(allStats, "forward p/e"),
+            ProfitMargins: GetVal(allStats, "profit margin"),
+            FloatShares: GetVal(allStats, "float"),
+            SharesOutstanding: GetVal(allStats, "shares outstanding"),
+            SharesShort: GetVal(allStats, "shares short"),
+            SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
+            SharesShortPreviousMonthDate: null, DateShortInterest: null,
+            SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"),
+            HeldPercentInsiders: GetVal(allStats, "% held by insiders"),
+            HeldPercentInstitutions: GetVal(allStats, "% held by institutions"),
+            ShortRatio: GetVal(allStats, "short ratio"),
+            ShortPercentOfFloat: GetVal(allStats, "short % of float"),
+            Beta: GetVal(allStats, "beta (5y monthly)", "beta"),
+            Category: null,
+            BookValue: GetVal(allStats, "book value per share", "book value"),
+            PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"),
+            LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"),
+            NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"),
+            MostRecentQuarter: GetVal(allStats, "most recent quarter"),
+            EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"),
+            NetIncomeToCommon: GetVal(allStats, "net income avi to common"),
+            TrailingEps: GetVal(allStats, "diluted eps"),
+            ForwardEps: GetVal(allStats, "forward eps"),
+            PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"),
+            EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"),
+            EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"),
+            FiftyTwoWeekChange: GetVal(allStats, "52-week change"),
+            SandP52WeekChange: GetVal(allStats, "s&p500 52-week change")
+        );
+
+        var financialData = new YahooFinancialDataDto(
+            CurrentPrice: GetVal(allStats, "current price", "price"),
+            TargetHighPrice: GetVal(allStats, "target high", "high target"),
+            TargetLowPrice: GetVal(allStats, "target low", "low target"),
+            TargetMeanPrice: GetVal(allStats, "target mean", "target est"),
+            TargetMedianPrice: GetVal(allStats, "target median"),
+            RecommendationMean: GetVal(allStats, "recommendation mean"),
+            RecommendationKey: GetString(allStats, "recommendation key"),
+            NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"),
+            TotalCash: GetVal(allStats, "total cash"),
+            TotalCashPerShare: GetVal(allStats, "total cash per share"),
+            Ebitda: GetVal(allStats, "ebitda"),
+            TotalDebt: GetVal(allStats, "total debt"),
+            QuickRatio: GetVal(allStats, "quick ratio"),
+            CurrentRatio: GetVal(allStats, "current ratio"),
+            TotalRevenue: GetVal(allStats, "revenue", "total revenue"),
+            DebtToEquity: GetVal(allStats, "total debt/equity"),
+            RevenuePerShare: GetVal(allStats, "revenue per share"),
+            ReturnOnAssets: GetVal(allStats, "return on assets"),
+            ReturnOnEquity: GetVal(allStats, "return on equity"),
+            GrossProfits: GetVal(allStats, "gross profit"),
+            FreeCashflow: GetVal(allStats, "levered free cash flow"),
+            OperatingCashflow: GetVal(allStats, "operating cash flow"),
+            RevenueGrowth: GetVal(allStats, "quarterly revenue growth"),
+            GrossMargins: GetVal(allStats, "gross margin"),
+            EbitdaMargins: GetVal(allStats, "ebitda margin"),
+            OperatingMargins: GetVal(allStats, "operating margin"),
+            ProfitMargins: GetVal(allStats, "profit margin"),
+            FinancialCurrency: "USD"
+        );
+
+        var summaryDetail = new YahooSummaryDetailDto(
+            MaxAge: 86400, PriceHint: null,
+            PreviousClose: GetVal(allStats, "previous close"),
+            Open: GetVal(allStats, "open"),
+            DayLow: GetVal(allStats, "day low"),
+            DayHigh: GetVal(allStats, "day high"),
+            RegularMarketPreviousClose: GetVal(allStats, "previous close"),
+            RegularMarketOpen: GetVal(allStats, "open"),
+            RegularMarketDayLow: GetVal(allStats, "day low"),
+            RegularMarketDayHigh: GetVal(allStats, "day high"),
+            DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"),
+            DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"),
+            ExDividendDate: GetVal(allStats, "ex-dividend date"),
+            PayoutRatio: GetVal(allStats, "payout ratio"),
+            FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"),
+            Beta: GetVal(allStats, "beta"),
+            TrailingPE: GetVal(allStats, "trailing p/e"),
+            ForwardPE: GetVal(allStats, "forward p/e"),
+            Volume: GetVal(allStats, "volume"),
+            RegularMarketVolume: GetVal(allStats, "volume"),
+            AverageVolume: GetVal(allStats, "avg. volume", "average volume"),
+            AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"),
+            AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"),
+            Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"),
+            BidSize: null, AskSize: null,
+            MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"),
+            FiftyTwoWeekLow: GetVal(allStats, "52 week low"),
+            FiftyTwoWeekHigh: GetVal(allStats, "52 week high"),
+            PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"),
+            Currency: "USD"
+        );
+
+        return new YahooQuoteSummaryModulesDto(
+            QuoteType: null,
+            AssetProfile: assetProfile,
+            FinancialData: financialData,
+            DefaultKeyStatistics: defaultKeyStatistics,
+            SummaryDetail: summaryDetail,
+            IncomeStatementHistory: null,
+            IncomeStatementHistoryQuarterly: null,
+            BalanceSheetHistory: null,
+            BalanceSheetHistoryQuarterly: null,
+            CashflowStatementHistory: null,
+            CashflowStatementHistoryQuarterly: null,
+            CalendarEvents: null
+        );
+    }
+
+    private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] keys)
+    {
+        foreach (var k in keys)
+        {
+            if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
+                return ParseYahooValue(val);
+
+            var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
+            if (!string.IsNullOrWhiteSpace(match.Value))
+                return ParseYahooValue(match.Value);
+        }
+        return null;
+    }
+
+    private static string? GetString(Dictionary<string, string> dict, params string[] keys)
+    {
+        foreach (var k in keys)
+        {
+            if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
+                return val.Trim();
+
+            var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
+            if (!string.IsNullOrWhiteSpace(match.Value))
+                return match.Value.Trim();
+        }
+        return null;
+    }
+
+    /// <summary>
+    /// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln.
+    /// </summary>
+    public static YahooValueDto? ParseYahooValue(string? text)
+    {
+        if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-")
+            return null;
+
+        var trimmed = text.Trim();
+        bool isPercent = trimmed.EndsWith("%");
+
+        double multiplier = 1.0;
+        if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0;
+        else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0;
+        else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0;
+        else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0;
+
+        // Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt
+        var numPart = Regex.Replace(trimmed, @"[^\d.-]", "");
+
+        if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal))
+        {
+            double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier);
+            return new YahooValueDto
+            {
+                Raw = finalVal,
+                Fmt = trimmed,
+                LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture)
+            };
+        }
+
+        return null;
+    }
+
+    private record ProfileExtractionResult(
+        Dictionary<string, string> ProfileDict,
+        List<YahooCompanyOfficerDto> Officers,
+        string? Sector,
+        string? Industry,
+        int? Employees,
+        string? Description
+    );
+
+    private class ProfileMetaJsResult
+    {
+        public string? Sector { get; set; }
+        public string? Industry { get; set; }
+        public int? Employees { get; set; }
+        public string? Description { get; set; }
+        public List<OfficerJsResult>? Officers { get; set; }
+    }
+
+    private class OfficerJsResult
+    {
+        public string? Name { get; set; }
+        public string? Title { get; set; }
+        public string? Pay { get; set; }
+        public string? Exercised { get; set; }
+        public int? YearBorn { get; set; }
+    }
+}