using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.Yahoo; using FinlyticCore.Services.Yahoo; using FinlyticFundamentals.Entities; using Microsoft.Extensions.Logging; namespace FinlyticFundamentals.Services; public interface IYahooFinanceScraper { /// /// Resolves ticker from ISIN. /// Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// /// Resolves all tickers from ISIN. /// Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// /// Scrapes fundamentals. /// Task ScrapeFundamentalsAsync(string isin, string ticker, CancellationToken cancellationToken = default); } public record ScrapedFundamentalsData( AssetFundamentalsEntity Fundamentals, TickerFundamentalsEntity TickerData, List Executives, List Statements, List Estimates ); public class YahooFinanceScraper : IYahooFinanceScraper { private readonly HttpClient _httpClient; private readonly YahooFinanceClient _yahooClient; private readonly ILogger _logger; public YahooFinanceScraper(HttpClient httpClient, YahooFinanceClient yahooClient, ILogger logger) { _httpClient = httpClient; _yahooClient = yahooClient; _logger = logger; } /// public async Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default) { var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken); return tickers.FirstOrDefault(); } /// public async Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(isin)) return new(); var symbols = new List<(string symbol, int priority)>(); var primary = await _yahooClient.SearchAsync(isin, quotesCount: 20, cancellationToken: cancellationToken); var quotes = primary?.Quotes ?? new(); foreach (var q in quotes.Where(q => !string.IsNullOrEmpty(q.Symbol))) { symbols.Add((q.Symbol, GetExchangePriority(q.Symbol, isin))); } if (quotes.Count == 0) return []; // 2. Namenssuche für deutsche/andere Handelsplätze var companyName = quotes[0].LongName!; var secondary = await _yahooClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken); foreach (var q in secondary?.Quotes ?? new()) { if (!string.IsNullOrEmpty(q.Symbol) && !symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase))) { symbols.Add((q.Symbol, GetExchangePriority(q.Symbol, isin))); } } // 3. Sortieren und zurückgeben return symbols .OrderBy(s => s.priority) .Select(s => s.symbol) .Distinct(StringComparer.OrdinalIgnoreCase) .Take(20) .ToList(); } private int GetExchangePriority(string symbol, string isin) { if (!string.IsNullOrEmpty(isin) && isin.StartsWith("US", StringComparison.OrdinalIgnoreCase)) { if (!symbol.Contains('.')) return 1; if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase) || symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 3; return 4; } if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) { return 1; // XETRA } else if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) { return 2; // Frankfurt } else if (symbol.EndsWith(".TG", StringComparison.OrdinalIgnoreCase)) { return 3; // Gettex } else if (symbol.EndsWith(".MU", StringComparison.OrdinalIgnoreCase) || symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase) || symbol.EndsWith(".BE", StringComparison.OrdinalIgnoreCase) || symbol.EndsWith(".DU", StringComparison.OrdinalIgnoreCase) || symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) { return 4; // Other German regional exchanges } else if (symbol.Contains('.') && !symbol.EndsWith(".OB", StringComparison.OrdinalIgnoreCase) && !symbol.EndsWith(".PK", StringComparison.OrdinalIgnoreCase)) { return 5; // Domestic/home non-US exchanges } else { return 6; // Other } } /// public async Task ScrapeFundamentalsAsync(string isin, string ticker, CancellationToken cancellationToken = default) { _logger.LogInformation( "[{Channel}] Fetching fundamental data for Ticker {Ticker} (ISIN: {Isin}) using YahooFinanceClient...", "FundamentalsChannel", ticker, isin); try { var summaryResponse = await _yahooClient.GetFullQuoteSummaryAsync(ticker, cancellationToken); if (summaryResponse?.QuoteSummary?.Result == null || summaryResponse.QuoteSummary.Result.Count == 0) { _logger.LogWarning("[{Channel}] YahooFinanceClient returned no result for ticker {Ticker}", "FundamentalsChannel", ticker); return null; } var root = summaryResponse.QuoteSummary.Result[0]; var assetProfile = root.AssetProfile; var financialData = root.FinancialData; var defaultKeyStatistics = root.DefaultKeyStatistics; var summaryDetail = root.SummaryDetail; var calendarEvents = root.CalendarEvents; // Instantiate entities var fundamentals = new AssetFundamentalsEntity { Isin = isin, PrimaryTicker = ticker, LastUpdatedAt = DateTime.UtcNow, LastStaticUpdatedAt = DateTime.UtcNow }; var tickerData = new TickerFundamentalsEntity { Ticker = ticker, Isin = isin, LastUpdatedAt = DateTime.UtcNow }; // 1. Static Profile Data if (assetProfile != null) { fundamentals.BusinessSummary = assetProfile.LongBusinessSummary; fundamentals.Sector = assetProfile.Sector; fundamentals.Industry = assetProfile.Industry; fundamentals.Country = assetProfile.Country; fundamentals.Employees = assetProfile.FullTimeEmployees; } // Company Name fundamentals.CompanyName = ticker; // 2. Exchange & Trading Currency for Ticker if (financialData != null && !string.IsNullOrWhiteSpace(financialData.FinancialCurrency)) { tickerData.TradingCurrency = financialData.FinancialCurrency; } if (summaryDetail != null && !string.IsNullOrWhiteSpace(summaryDetail.Currency)) { tickerData.TradingCurrency = summaryDetail.Currency; } // 3. Dynamic Price & Valuation Data if (financialData != null) { tickerData.CurrentPrice = financialData.CurrentPrice?.DecimalValue ?? 0; tickerData.GrossMargin = financialData.GrossMargins?.DecimalValue; tickerData.OperatingMargin = financialData.OperatingMargins?.DecimalValue; tickerData.NetProfitMargin = financialData.ProfitMargins?.DecimalValue; tickerData.ReturnOnEquity = financialData.ReturnOnEquity?.DecimalValue; tickerData.ReturnOnAssets = financialData.ReturnOnAssets?.DecimalValue; tickerData.CurrentRatio = financialData.CurrentRatio?.DecimalValue; tickerData.QuickRatio = financialData.QuickRatio?.DecimalValue; tickerData.DebtToEquity = financialData.DebtToEquity?.DecimalValue; // Targets on Company Level fundamentals.PriceTargetLow = financialData.TargetLowPrice?.DecimalValue; fundamentals.PriceTargetHigh = financialData.TargetHighPrice?.DecimalValue; fundamentals.PriceTargetMedian = financialData.TargetMedianPrice?.DecimalValue; fundamentals.PriceTargetMean = financialData.TargetMeanPrice?.DecimalValue; } if (summaryDetail != null) { if (tickerData.CurrentPrice == 0) { tickerData.CurrentPrice = summaryDetail.Open?.DecimalValue ?? summaryDetail.PreviousClose?.DecimalValue ?? 0; } tickerData.FiftyTwoWeekHigh = summaryDetail.FiftyTwoWeekHigh?.DecimalValue ?? 0; tickerData.FiftyTwoWeekLow = summaryDetail.FiftyTwoWeekLow?.DecimalValue ?? 0; } var mCap = defaultKeyStatistics?.SharesOutstanding?.DecimalValue; mCap ??= summaryDetail?.MarketCap?.DecimalValue; tickerData.MarketCapitalization = mCap ?? 0; var ev = defaultKeyStatistics?.EnterpriseValue?.DecimalValue; tickerData.EnterpriseValue = ev ?? 0; tickerData.PeRatioTrailing = defaultKeyStatistics?.TrailingEps?.DecimalValue ?? summaryDetail?.TrailingPE?.DecimalValue; tickerData.PeRatioForward = defaultKeyStatistics?.ForwardPE?.DecimalValue ?? summaryDetail?.ForwardPE?.DecimalValue; if (defaultKeyStatistics != null) { tickerData.PegRatio = defaultKeyStatistics.PegRatio?.DecimalValue; tickerData.PbRatio = defaultKeyStatistics.PriceToBook?.DecimalValue; fundamentals.ShortRatio = defaultKeyStatistics.ShortRatio?.DecimalValue; fundamentals.ShortPercentOfFloat = defaultKeyStatistics.ShortPercentOfFloat?.DecimalValue; fundamentals.PercentHeldByInstitutions = defaultKeyStatistics.HeldPercentInstitutions?.DecimalValue; fundamentals.PercentHeldByInsiders = defaultKeyStatistics.HeldPercentInsiders?.DecimalValue; } tickerData.PsRatio = defaultKeyStatistics?.PriceToSalesTrailing12Months?.DecimalValue ?? summaryDetail?.PriceToSalesTrailing12Months?.DecimalValue; tickerData.EvToEbitda = defaultKeyStatistics?.EnterpriseToEbitda?.DecimalValue; tickerData.EvToRevenue = defaultKeyStatistics?.EnterpriseToRevenue?.DecimalValue; tickerData.DividendYield = summaryDetail?.DividendYield?.DecimalValue; tickerData.PayoutRatio = summaryDetail?.PayoutRatio?.DecimalValue; if (financialData != null) { if (!string.IsNullOrWhiteSpace(financialData.RecommendationKey) && !financialData.RecommendationKey.Equals("none", StringComparison.OrdinalIgnoreCase)) { fundamentals.ConsensusRating = financialData.RecommendationKey; } else if (financialData.RecommendationMean != null && financialData.RecommendationMean.Raw.HasValue) { double mean = financialData.RecommendationMean.Raw.Value; fundamentals.ConsensusRating = mean <= 1.8 ? "strong_buy" : (mean <= 2.5 ? "buy" : (mean <= 3.5 ? "hold" : (mean <= 4.2 ? "sell" : "strong_sell"))); } } // 4. Calendar Events Data if (calendarEvents != null) { if (calendarEvents.ExDividendDate?.Raw.HasValue == true) { long seconds = (long)calendarEvents.ExDividendDate.Raw.Value; if (seconds > 0) fundamentals.ExDividendDate = DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime; } if (calendarEvents.Earnings?.EarningsDate != null && calendarEvents.Earnings.EarningsDate.Count > 0) { var firstDate = calendarEvents.Earnings.EarningsDate[0]; if (firstDate.Raw.HasValue && firstDate.Raw.Value > 0) { fundamentals.NextEarningsDate = DateTimeOffset.FromUnixTimeSeconds((long)firstDate.Raw.Value).UtcDateTime; } } } if (!fundamentals.ExDividendDate.HasValue && summaryDetail?.ExDividendDate?.Raw.HasValue == true) { long seconds = (long)summaryDetail.ExDividendDate.Raw.Value; if (seconds > 0) fundamentals.ExDividendDate = DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime; } tickerData.ExDividendDate = fundamentals.ExDividendDate; // 5. Executives List var executives = new List(); if (assetProfile?.CompanyOfficers != null) { foreach (var officer in assetProfile.CompanyOfficers) { var exec = new CompanyExecutiveEntity { Isin = isin, Name = !string.IsNullOrWhiteSpace(officer.Name) ? officer.Name : "Unknown", Title = !string.IsNullOrWhiteSpace(officer.Title) ? officer.Title : "Officer", Age = officer.Age, Compensation = officer.TotalPay?.DecimalValue }; executives.Add(exec); } } // 6. Financial Statements var statements = new List(); // A. Annual Statements if (root.IncomeStatementHistory?.IncomeStatementHistory != null) { foreach (var item in root.IncomeStatementHistory.IncomeStatementHistory) { MapIncomeStatement(item, isin, "Annual", statements); } } if (root.BalanceSheetHistory?.BalanceSheetStatements != null) { foreach (var item in root.BalanceSheetHistory.BalanceSheetStatements) { MapBalanceSheet(item, isin, "Annual", statements); } } if (root.CashflowStatementHistory?.CashflowStatements != null) { foreach (var item in root.CashflowStatementHistory.CashflowStatements) { MapCashflowStatement(item, isin, "Annual", statements); } } // B. Quarterly Statements if (root.IncomeStatementHistoryQuarterly?.IncomeStatementHistory != null) { foreach (var item in root.IncomeStatementHistoryQuarterly.IncomeStatementHistory) { MapIncomeStatement(item, isin, "Quarterly", statements); } } if (root.BalanceSheetHistoryQuarterly?.BalanceSheetStatements != null) { foreach (var item in root.BalanceSheetHistoryQuarterly.BalanceSheetStatements) { MapBalanceSheet(item, isin, "Quarterly", statements); } } if (root.CashflowStatementHistoryQuarterly?.CashflowStatements != null) { foreach (var item in root.CashflowStatementHistoryQuarterly.CashflowStatements) { MapCashflowStatement(item, isin, "Quarterly", statements); } } // 7. Forward Estimates var estimates = new List(); return new ScrapedFundamentalsData(fundamentals, tickerData, executives, statements, estimates); } catch (Exception ex) { _logger.LogError(ex, "[{Channel}] Failed to scrape fundamentals for ISIN {Isin} (Ticker: {Ticker})", "FundamentalsChannel", isin, ticker); return null; } } private static void MapIncomeStatement(YahooIncomeStatementDto item, string isin, string periodType, List statements) { if (item.EndDate?.Raw.HasValue != true) return; var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date; var statement = GetOrCreateStatement(statements, isin, periodType, endDate); if (item.TotalRevenue?.Raw.HasValue == true) statement.TotalRevenue = item.TotalRevenue.DecimalValue; if (item.CostOfRevenue?.Raw.HasValue == true) statement.CostOfRevenue = item.CostOfRevenue.DecimalValue; if (item.GrossProfit?.Raw.HasValue == true) statement.GrossProfit = item.GrossProfit.DecimalValue; else if (statement.TotalRevenue.HasValue && statement.CostOfRevenue.HasValue) statement.GrossProfit = statement.TotalRevenue - statement.CostOfRevenue; if (item.TotalOperatingExpenses?.Raw.HasValue == true) statement.OperatingExpenses = item.TotalOperatingExpenses.DecimalValue; if (item.OperatingIncome?.Raw.HasValue == true) statement.OperatingIncome = item.OperatingIncome.DecimalValue; else if (statement.GrossProfit.HasValue && statement.OperatingExpenses.HasValue) statement.OperatingIncome = statement.GrossProfit - statement.OperatingExpenses; if (item.Ebit?.Raw.HasValue == true) statement.Ebitda = item.Ebit.DecimalValue; if (item.NetIncome?.Raw.HasValue == true) statement.NetIncome = item.NetIncome.DecimalValue; } private static void MapBalanceSheet(YahooBalanceSheetStatementDto item, string isin, string periodType, List statements) { if (item.EndDate?.Raw.HasValue != true) return; var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date; var statement = GetOrCreateStatement(statements, isin, periodType, endDate); if (item.Cash?.Raw.HasValue == true) statement.CashAndCashEquivalents = item.Cash.DecimalValue; if (item.NetReceivables?.Raw.HasValue == true) statement.AccountsReceivable = item.NetReceivables.DecimalValue; if (item.Inventory?.Raw.HasValue == true) statement.Inventory = item.Inventory.DecimalValue; if (item.TotalCurrentAssets?.Raw.HasValue == true) statement.TotalCurrentAssets = item.TotalCurrentAssets.DecimalValue; if (item.TotalCurrentLiabilities?.Raw.HasValue == true) statement.CurrentLiabilities = item.TotalCurrentLiabilities.DecimalValue; if (item.LongTermDebt?.Raw.HasValue == true) statement.LongTermDebt = item.LongTermDebt.DecimalValue; if (item.TotalLiab?.Raw.HasValue == true) statement.TotalLiabilities = item.TotalLiab.DecimalValue; if (item.TotalStockholderEquity?.Raw.HasValue == true) statement.TotalStockholdersEquity = item.TotalStockholderEquity.DecimalValue; } private static void MapCashflowStatement(YahooCashflowStatementDto item, string isin, string periodType, List statements) { if (item.EndDate?.Raw.HasValue != true) return; var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date; var statement = GetOrCreateStatement(statements, isin, periodType, endDate); if (item.TotalCashFromOperatingActivities?.Raw.HasValue == true) statement.OperatingCashFlow = item.TotalCashFromOperatingActivities.DecimalValue; if (item.TotalCashflowsFromInvestingActivities?.Raw.HasValue == true) statement.InvestingCashFlow = item.TotalCashflowsFromInvestingActivities.DecimalValue; if (item.CapitalExpenditures?.Raw.HasValue == true) statement.CapitalExpenditures = item.CapitalExpenditures.DecimalValue; if (item.TotalCashFromFinancingActivities?.Raw.HasValue == true) statement.FinancingCashFlow = item.TotalCashFromFinancingActivities.DecimalValue; if (statement.OperatingCashFlow.HasValue) { var capex = statement.CapitalExpenditures ?? 0m; statement.FreeCashFlow = statement.OperatingCashFlow.Value - Math.Abs(capex); } } private static FinancialStatementEntity GetOrCreateStatement(List statements, string isin, string periodType, DateTime endDate) { var existing = statements.FirstOrDefault(s => s.PeriodType == periodType && s.EndDate.Date == endDate.Date); if (existing == null) { existing = new FinancialStatementEntity { Isin = isin, PeriodType = periodType, EndDate = endDate.Date }; statements.Add(existing); } return existing; } }