using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Clients; using FinlyticCore.Dtos.Fundamentals; using FinlyticCore.Dtos.Yahoo; using FinlyticCore.Models.Settings; using FinlyticCore.Services; using FinlyticCore.Utils; using Microsoft.Extensions.Configuration; namespace FinlyticCore.Services.Yahoo; public interface IYahooFinanceScraper { /// /// Ermittelt den primären Börsenticker zu einer ISIN anhand von Börsenplatz-Prioritäten. /// Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// /// Ermittelt alle gefundenen Börsenticker zu einer ISIN, sortiert nach Priorität. /// Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// /// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab /// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind. /// Task GetQuoteSummaryModulesAsync( string symbolOrIsin, bool forceHtmlScrape = false, bool includeProfile = true, CancellationToken cancellationToken = default); } public class YahooFinanceScraper : IYahooFinanceScraper { private const string _serviceName = nameof(YahooFinanceScraper); private readonly YahooFinanceClient _yahooApiClient; private readonly IYahooFinanceHtmlClient _htmlScraperClient; private readonly IConfiguration _configuration; private readonly IFinlyticLogger _finlyticLogger; public YahooFinanceScraper( YahooFinanceClient yahooApiClient, IYahooFinanceHtmlClient htmlScraperClient, IConfiguration configuration, IFinlyticLogger finlyticLogger) { _yahooApiClient = yahooApiClient; _htmlScraperClient = htmlScraperClient; _configuration = configuration; _finlyticLogger = finlyticLogger; } /// public async Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default) { var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken); return tickers.FirstOrDefault(); } /// public async Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(isin)) return new List(); var cleanIsin = isin.Trim().ToUpperInvariant(); var symbols = new List<(string symbol, string exchange, int priority)>(); // Crypto / Trade Republic interne ISINs (beginnend mit 'X', z. B. XF000BTC0017) if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase)) { var (cryptoSubtitle, cryptoName) = await CryptoSubtitleResolver.ResolveCryptoInfoAsync( cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken); if (!string.IsNullOrWhiteSpace(cryptoSubtitle)) { var cryptoEur = $"{cryptoSubtitle}-EUR"; var cryptoUsd = $"{cryptoSubtitle}-USD"; symbols.Add((cryptoEur, "Crypto", 0)); symbols.Add((cryptoUsd, "Crypto", 1)); try { var searchRes = await _yahooApiClient.SearchAsync(cryptoSubtitle, quotesCount: 10, cancellationToken: cancellationToken); if (searchRes?.Quotes != null) { foreach (var q in searchRes.Quotes.Where(q => !string.IsNullOrEmpty(q.Symbol))) { if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase))) { symbols.Add((q.Symbol, q.Exchange ?? "Crypto", 2)); } } } } catch (Exception ex) { await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, $"[{_serviceName}] Crypto search failed for {cryptoSubtitle}"); } await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel, $"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}"); return symbols .OrderBy(s => s.priority) .Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange }) .ToList(); } } try { // 1. Suche via ISIN - der allererste Ticker von Yahoo Finance ist der absolute Primary Ticker var primary = await _yahooApiClient.SearchAsync(cleanIsin, quotesCount: 20, cancellationToken: cancellationToken); var quotes = primary?.Quotes ?? new List(); var validQuotes = quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)).ToList(); if (validQuotes.Count > 0) { var first = validQuotes[0]; symbols.Add((first.Symbol, first.Exchange ?? string.Empty, 0)); foreach (var q in validQuotes.Skip(1)) { if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase))) { symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin)))); } } } // 2. Falls Ticker gefunden, mit Unternehmensname noch mehr internationale Exchangeticker suchen (z.B. APC.DE) if (validQuotes.Count > 0) { var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName; if (!string.IsNullOrWhiteSpace(companyName)) { var secondary = await _yahooApiClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken); foreach (var q in secondary?.Quotes ?? new List()) { if (!string.IsNullOrEmpty(q.Symbol) && !symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase))) { symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin)))); } } } } } catch (Exception ex) { await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, $"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'"); } var result = symbols .OrderBy(s => s.priority) .Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = !string.IsNullOrWhiteSpace(s.exchange) ? s.exchange : "Unknown" }) .DistinctBy(s => s.Ticker, StringComparer.OrdinalIgnoreCase) .ToList(); return result; } /// public async Task GetQuoteSummaryModulesAsync( string symbolOrIsin, bool forceHtmlScrape = false, bool includeProfile = true, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null; var symbol = symbolOrIsin.Trim().ToUpperInvariant(); // Falls eine ISIN übergeben wurde, zuerst Ticker auflösen if (IsIsin(symbol)) { var resolvedTicker = await ResolveTickerFromIsinAsync(symbol, cancellationToken); if (resolvedTicker != null) { symbol = resolvedTicker.Ticker; } } YahooQuoteSummaryModulesDto? apiModules = null; // ------------------------------------------------------------- // 1. PRIMÄRE DATENQUELLE: Yahoo Finance API (Cookie/Crumb) // ------------------------------------------------------------- if (!forceHtmlScrape) { try { await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, $"[{_serviceName}] Starte primären API-Abruf für '{symbol}'..."); var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken); apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault(); if (apiModules != null && HasSufficientData(apiModules)) { await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, $"[{_serviceName}] Erfolgreich Daten über API bezogen für '{symbol}'."); return apiModules; } await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, $"[{_serviceName}] API lieferte unvollständige Daten für '{symbol}'. Initiiere Fallback..."); } catch (Exception ex) { await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, $"[{_serviceName}] API-Abruf fehlgeschlagen für '{symbol}'. Wechsle zu Scraper..."); } } // ------------------------------------------------------------- // 2. FALLBACK DATENQUELLE: Playwright HTML Scraper // ------------------------------------------------------------- YahooQuoteSummaryModulesDto? htmlModules = null; try { await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, $"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}' (IncludeProfile: {includeProfile})..."); htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, includeProfile, cancellationToken); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, $"[{_serviceName}] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{symbol}'."); } // ------------------------------------------------------------- // 3. Zusammenführen (Merge API & HTML Fallback) // ------------------------------------------------------------- if (apiModules == null) return htmlModules; if (htmlModules == null) return apiModules; return MergeModules(apiModules, htmlModules); } private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules) { return modules.SummaryDetail != null || modules.FinancialData != null || modules.DefaultKeyStatistics != null; } public static YahooQuoteSummaryModulesDto? MergeModules( YahooQuoteSummaryModulesDto? primary, YahooQuoteSummaryModulesDto? secondary) { if (primary == null && secondary == null) return null; if (primary == null) return secondary; if (secondary == null) return primary; return new YahooQuoteSummaryModulesDto( QuoteType: primary.QuoteType ?? secondary.QuoteType, AssetProfile: primary.AssetProfile ?? secondary.AssetProfile, FinancialData: MergeFinancialData(primary.FinancialData, secondary.FinancialData), DefaultKeyStatistics: MergeDefaultKeyStatistics(primary.DefaultKeyStatistics, secondary.DefaultKeyStatistics), SummaryDetail: MergeSummaryDetail(primary.SummaryDetail, secondary.SummaryDetail), IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory, IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly, BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory, BalanceSheetHistoryQuarterly: primary.BalanceSheetHistoryQuarterly ?? secondary.BalanceSheetHistoryQuarterly, CashflowStatementHistory: primary.CashflowStatementHistory ?? secondary.CashflowStatementHistory, CashflowStatementHistoryQuarterly: primary.CashflowStatementHistoryQuarterly ?? secondary.CashflowStatementHistoryQuarterly, CalendarEvents: primary.CalendarEvents ?? secondary.CalendarEvents ); } private static YahooFinancialDataDto? MergeFinancialData(YahooFinancialDataDto? a, YahooFinancialDataDto? b) { if (a == null) return b; if (b == null) return a; return new YahooFinancialDataDto( CurrentPrice: a.CurrentPrice ?? b.CurrentPrice, TargetHighPrice: a.TargetHighPrice ?? b.TargetHighPrice, TargetLowPrice: a.TargetLowPrice ?? b.TargetLowPrice, TargetMeanPrice: a.TargetMeanPrice ?? b.TargetMeanPrice, TargetMedianPrice: a.TargetMedianPrice ?? b.TargetMedianPrice, RecommendationMean: a.RecommendationMean ?? b.RecommendationMean, RecommendationKey: !string.IsNullOrWhiteSpace(a.RecommendationKey) && a.RecommendationKey != "none" ? a.RecommendationKey : b.RecommendationKey, NumberOfAnalystOpinions: a.NumberOfAnalystOpinions ?? b.NumberOfAnalystOpinions, TotalCash: a.TotalCash ?? b.TotalCash, TotalCashPerShare: a.TotalCashPerShare ?? b.TotalCashPerShare, Ebitda: a.Ebitda ?? b.Ebitda, TotalDebt: a.TotalDebt ?? b.TotalDebt, QuickRatio: a.QuickRatio ?? b.QuickRatio, CurrentRatio: a.CurrentRatio ?? b.CurrentRatio, TotalRevenue: a.TotalRevenue ?? b.TotalRevenue, DebtToEquity: a.DebtToEquity ?? b.DebtToEquity, RevenuePerShare: a.RevenuePerShare ?? b.RevenuePerShare, ReturnOnAssets: a.ReturnOnAssets ?? b.ReturnOnAssets, ReturnOnEquity: a.ReturnOnEquity ?? b.ReturnOnEquity, GrossProfits: a.GrossProfits ?? b.GrossProfits, FreeCashflow: a.FreeCashflow ?? b.FreeCashflow, OperatingCashflow: a.OperatingCashflow ?? b.OperatingCashflow, RevenueGrowth: a.RevenueGrowth ?? b.RevenueGrowth, GrossMargins: a.GrossMargins ?? b.GrossMargins, EbitdaMargins: a.EbitdaMargins ?? b.EbitdaMargins, OperatingMargins: a.OperatingMargins ?? b.OperatingMargins, ProfitMargins: a.ProfitMargins ?? b.ProfitMargins, FinancialCurrency: a.FinancialCurrency ?? b.FinancialCurrency ); } private static YahooDefaultKeyStatisticsDto? MergeDefaultKeyStatistics(YahooDefaultKeyStatisticsDto? a, YahooDefaultKeyStatisticsDto? b) { if (a == null) return b; if (b == null) return a; return new YahooDefaultKeyStatisticsDto( PriceToBook: a.PriceToBook ?? b.PriceToBook, EnterpriseValue: a.EnterpriseValue ?? b.EnterpriseValue, ForwardPE: a.ForwardPE ?? b.ForwardPE, ProfitMargins: a.ProfitMargins ?? b.ProfitMargins, FloatShares: a.FloatShares ?? b.FloatShares, SharesOutstanding: a.SharesOutstanding ?? b.SharesOutstanding, SharesShort: a.SharesShort ?? b.SharesShort, SharesShortPriorMonth: a.SharesShortPriorMonth ?? b.SharesShortPriorMonth, SharesShortPreviousMonthDate: a.SharesShortPreviousMonthDate ?? b.SharesShortPreviousMonthDate, DateShortInterest: a.DateShortInterest ?? b.DateShortInterest, SharesPercentSharesOut: a.SharesPercentSharesOut ?? b.SharesPercentSharesOut, HeldPercentInsiders: a.HeldPercentInsiders ?? b.HeldPercentInsiders, HeldPercentInstitutions: a.HeldPercentInstitutions ?? b.HeldPercentInstitutions, ShortRatio: a.ShortRatio ?? b.ShortRatio, ShortPercentOfFloat: a.ShortPercentOfFloat ?? b.ShortPercentOfFloat, Beta: a.Beta ?? b.Beta, Category: a.Category ?? b.Category, BookValue: a.BookValue ?? b.BookValue, PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months, LastFiscalYearEnd: a.LastFiscalYearEnd ?? b.LastFiscalYearEnd, NextFiscalYearEnd: a.NextFiscalYearEnd ?? b.NextFiscalYearEnd, MostRecentQuarter: a.MostRecentQuarter ?? b.MostRecentQuarter, EarningsQuarterlyGrowth: a.EarningsQuarterlyGrowth ?? b.EarningsQuarterlyGrowth, NetIncomeToCommon: a.NetIncomeToCommon ?? b.NetIncomeToCommon, TrailingEps: a.TrailingEps ?? b.TrailingEps, ForwardEps: a.ForwardEps ?? b.ForwardEps, PegRatio: a.PegRatio ?? b.PegRatio, EnterpriseToRevenue: a.EnterpriseToRevenue ?? b.EnterpriseToRevenue, EnterpriseToEbitda: a.EnterpriseToEbitda ?? b.EnterpriseToEbitda, FiftyTwoWeekChange: a.FiftyTwoWeekChange ?? b.FiftyTwoWeekChange, SandP52WeekChange: a.SandP52WeekChange ?? b.SandP52WeekChange ); } private static YahooSummaryDetailDto? MergeSummaryDetail(YahooSummaryDetailDto? a, YahooSummaryDetailDto? b) { if (a == null) return b; if (b == null) return a; return new YahooSummaryDetailDto( MaxAge: a.MaxAge ?? b.MaxAge, PriceHint: a.PriceHint ?? b.PriceHint, PreviousClose: a.PreviousClose ?? b.PreviousClose, Open: a.Open ?? b.Open, DayLow: a.DayLow ?? b.DayLow, DayHigh: a.DayHigh ?? b.DayHigh, RegularMarketPreviousClose: a.RegularMarketPreviousClose ?? b.RegularMarketPreviousClose, RegularMarketOpen: a.RegularMarketOpen ?? b.RegularMarketOpen, RegularMarketDayLow: a.RegularMarketDayLow ?? b.RegularMarketDayLow, RegularMarketDayHigh: a.RegularMarketDayHigh ?? b.RegularMarketDayHigh, DividendRate: a.DividendRate ?? b.DividendRate, DividendYield: a.DividendYield ?? b.DividendYield, ExDividendDate: a.ExDividendDate ?? b.ExDividendDate, PayoutRatio: a.PayoutRatio ?? b.PayoutRatio, FiveYearAvgDividendYield: a.FiveYearAvgDividendYield ?? b.FiveYearAvgDividendYield, Beta: a.Beta ?? b.Beta, TrailingPE: a.TrailingPE ?? b.TrailingPE, ForwardPE: a.ForwardPE ?? b.ForwardPE, Volume: a.Volume ?? b.Volume, RegularMarketVolume: a.RegularMarketVolume ?? b.RegularMarketVolume, AverageVolume: a.AverageVolume ?? b.AverageVolume, AverageVolume10days: a.AverageVolume10days ?? b.AverageVolume10days, AverageDailyVolume10Day: a.AverageDailyVolume10Day ?? b.AverageDailyVolume10Day, Bid: a.Bid ?? b.Bid, Ask: a.Ask ?? b.Ask, BidSize: a.BidSize ?? b.BidSize, AskSize: a.AskSize ?? b.AskSize, MarketCap: a.MarketCap ?? b.MarketCap, FiftyTwoWeekLow: a.FiftyTwoWeekLow ?? b.FiftyTwoWeekLow, FiftyTwoWeekHigh: a.FiftyTwoWeekHigh ?? b.FiftyTwoWeekHigh, PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months, Currency: a.Currency ?? b.Currency ); } private static bool IsIsin(string value) { return value.Length == 12 && char.IsLetter(value[0]) && char.IsLetter(value[1]) && value.All(char.IsLetterOrDigit); } private static int GetExchangePriority(string symbol, string isin) { bool isGermanIsin = isin.StartsWith("DE", StringComparison.OrdinalIgnoreCase); if (isGermanIsin) { if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 1; // Xetra if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 2; // Frankfurt if (symbol.EndsWith(".STU", StringComparison.OrdinalIgnoreCase)) return 3; // Stuttgart if (symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 4; // Stuttgart (alt) if (symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) return 5; // Hamburg if (!symbol.Contains('.')) return 6; // US Primary } else { if (!symbol.Contains('.')) return 1; // US Primary (NASDAQ, NYSE) if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; // Xetra if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 3; // Frankfurt if (symbol.EndsWith(".L", StringComparison.OrdinalIgnoreCase)) return 4; // London if (symbol.EndsWith(".PA", StringComparison.OrdinalIgnoreCase)) return 5; // Paris } return 10; } }