Files
Finlytic/FinlyticFundamentals/Services/YahooFinanceScraper.cs
T

491 lines
21 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Services.Yahoo;
using FinlyticFundamentals.Entities;
using Microsoft.Extensions.Logging;
namespace FinlyticFundamentals.Services;
public interface IYahooFinanceScraper
{
/// <summary>
/// Resolves ticker from ISIN.
/// </summary>
Task<string?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Resolves all tickers from ISIN.
/// </summary>
Task<List<string>> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Scrapes fundamentals.
/// </summary>
Task<ScrapedFundamentalsData?> ScrapeFundamentalsAsync(string isin, string ticker,
CancellationToken cancellationToken = default);
}
public record ScrapedFundamentalsData(
AssetFundamentalsEntity Fundamentals,
TickerFundamentalsEntity TickerData,
List<CompanyExecutiveEntity> Executives,
List<FinancialStatementEntity> Statements,
List<ForwardEstimateEntity> Estimates
);
public class YahooFinanceScraper : IYahooFinanceScraper
{
private readonly HttpClient _httpClient;
private readonly YahooFinanceClient _yahooClient;
private readonly ILogger<YahooFinanceScraper> _logger;
public YahooFinanceScraper(HttpClient httpClient, YahooFinanceClient yahooClient,
ILogger<YahooFinanceScraper> logger)
{
_httpClient = httpClient;
_yahooClient = yahooClient;
_logger = logger;
}
/// <inheritdoc />
public async Task<string?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default)
{
var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken);
return tickers.FirstOrDefault();
}
/// <inheritdoc />
public async Task<List<string>> ResolveAllTickersFromIsinAsync(string isin,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(isin)) return new();
var symbols = new List<(string symbol, int priority)>();
var primary = await _yahooClient.SearchAsync(isin, quotesCount: 20, cancellationToken: cancellationToken);
var quotes = primary?.Quotes ?? new();
foreach (var q in quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)))
{
symbols.Add((q.Symbol, GetExchangePriority(q.Symbol, isin)));
}
if (quotes.Count == 0) return [];
// 2. Namenssuche für deutsche/andere Handelsplätze
var companyName = quotes[0].LongName!;
var secondary =
await _yahooClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken);
foreach (var q in secondary?.Quotes ?? new())
{
if (!string.IsNullOrEmpty(q.Symbol) &&
!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, GetExchangePriority(q.Symbol, isin)));
}
}
// 3. Sortieren und zurückgeben
return symbols
.OrderBy(s => s.priority)
.Select(s => s.symbol)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(20)
.ToList();
}
private int GetExchangePriority(string symbol, string isin)
{
if (!string.IsNullOrEmpty(isin) && isin.StartsWith("US", StringComparison.OrdinalIgnoreCase))
{
if (!symbol.Contains('.')) return 1;
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2;
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase) ||
symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 3;
return 4;
}
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase))
{
return 1; // XETRA
}
else if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase))
{
return 2; // Frankfurt
}
else if (symbol.EndsWith(".TG", StringComparison.OrdinalIgnoreCase))
{
return 3; // Gettex
}
else if (symbol.EndsWith(".MU", StringComparison.OrdinalIgnoreCase) ||
symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase) ||
symbol.EndsWith(".BE", StringComparison.OrdinalIgnoreCase) ||
symbol.EndsWith(".DU", StringComparison.OrdinalIgnoreCase) ||
symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase))
{
return 4; // Other German regional exchanges
}
else if (symbol.Contains('.') && !symbol.EndsWith(".OB", StringComparison.OrdinalIgnoreCase) &&
!symbol.EndsWith(".PK", StringComparison.OrdinalIgnoreCase))
{
return 5; // Domestic/home non-US exchanges
}
else
{
return 6; // Other
}
}
/// <inheritdoc />
public async Task<ScrapedFundamentalsData?> ScrapeFundamentalsAsync(string isin, string ticker,
CancellationToken cancellationToken = default)
{
_logger.LogInformation(
"[{Channel}] Fetching fundamental data for Ticker {Ticker} (ISIN: {Isin}) using YahooFinanceClient...",
"FundamentalsChannel", ticker, isin);
try
{
var summaryResponse = await _yahooClient.GetFullQuoteSummaryAsync(ticker, cancellationToken);
if (summaryResponse?.QuoteSummary?.Result == null || summaryResponse.QuoteSummary.Result.Count == 0)
{
_logger.LogWarning("[{Channel}] YahooFinanceClient returned no result for ticker {Ticker}",
"FundamentalsChannel", ticker);
return null;
}
var root = summaryResponse.QuoteSummary.Result[0];
var assetProfile = root.AssetProfile;
var financialData = root.FinancialData;
var defaultKeyStatistics = root.DefaultKeyStatistics;
var summaryDetail = root.SummaryDetail;
var calendarEvents = root.CalendarEvents;
// Instantiate entities
var fundamentals = new AssetFundamentalsEntity
{
Isin = isin,
PrimaryTicker = ticker,
LastUpdatedAt = DateTime.UtcNow,
LastStaticUpdatedAt = DateTime.UtcNow
};
var tickerData = new TickerFundamentalsEntity
{
Ticker = ticker,
Isin = isin,
LastUpdatedAt = DateTime.UtcNow
};
// 1. Static Profile Data
if (assetProfile != null)
{
fundamentals.BusinessSummary = assetProfile.LongBusinessSummary;
fundamentals.Sector = assetProfile.Sector;
fundamentals.Industry = assetProfile.Industry;
fundamentals.Country = assetProfile.Country;
fundamentals.Employees = assetProfile.FullTimeEmployees;
}
// Company Name
fundamentals.CompanyName = ticker;
// 2. Exchange & Trading Currency for Ticker
if (financialData != null && !string.IsNullOrWhiteSpace(financialData.FinancialCurrency))
{
tickerData.TradingCurrency = financialData.FinancialCurrency;
}
if (summaryDetail != null && !string.IsNullOrWhiteSpace(summaryDetail.Currency))
{
tickerData.TradingCurrency = summaryDetail.Currency;
}
// 3. Dynamic Price & Valuation Data
if (financialData != null)
{
tickerData.CurrentPrice = financialData.CurrentPrice?.DecimalValue ?? 0;
tickerData.GrossMargin = financialData.GrossMargins?.DecimalValue;
tickerData.OperatingMargin = financialData.OperatingMargins?.DecimalValue;
tickerData.NetProfitMargin = financialData.ProfitMargins?.DecimalValue;
tickerData.ReturnOnEquity = financialData.ReturnOnEquity?.DecimalValue;
tickerData.ReturnOnAssets = financialData.ReturnOnAssets?.DecimalValue;
tickerData.CurrentRatio = financialData.CurrentRatio?.DecimalValue;
tickerData.QuickRatio = financialData.QuickRatio?.DecimalValue;
tickerData.DebtToEquity = financialData.DebtToEquity?.DecimalValue;
// Targets on Company Level
fundamentals.PriceTargetLow = financialData.TargetLowPrice?.DecimalValue;
fundamentals.PriceTargetHigh = financialData.TargetHighPrice?.DecimalValue;
fundamentals.PriceTargetMedian = financialData.TargetMedianPrice?.DecimalValue;
fundamentals.PriceTargetMean = financialData.TargetMeanPrice?.DecimalValue;
}
if (summaryDetail != null)
{
if (tickerData.CurrentPrice == 0)
{
tickerData.CurrentPrice = summaryDetail.Open?.DecimalValue ??
summaryDetail.PreviousClose?.DecimalValue ?? 0;
}
tickerData.FiftyTwoWeekHigh = summaryDetail.FiftyTwoWeekHigh?.DecimalValue ?? 0;
tickerData.FiftyTwoWeekLow = summaryDetail.FiftyTwoWeekLow?.DecimalValue ?? 0;
}
var mCap = defaultKeyStatistics?.SharesOutstanding?.DecimalValue;
mCap ??= summaryDetail?.MarketCap?.DecimalValue;
tickerData.MarketCapitalization = mCap ?? 0;
var ev = defaultKeyStatistics?.EnterpriseValue?.DecimalValue;
tickerData.EnterpriseValue = ev ?? 0;
tickerData.PeRatioTrailing = defaultKeyStatistics?.TrailingEps?.DecimalValue ??
summaryDetail?.TrailingPE?.DecimalValue;
tickerData.PeRatioForward =
defaultKeyStatistics?.ForwardPE?.DecimalValue ?? summaryDetail?.ForwardPE?.DecimalValue;
if (defaultKeyStatistics != null)
{
tickerData.PegRatio = defaultKeyStatistics.PegRatio?.DecimalValue;
tickerData.PbRatio = defaultKeyStatistics.PriceToBook?.DecimalValue;
fundamentals.ShortRatio = defaultKeyStatistics.ShortRatio?.DecimalValue;
fundamentals.ShortPercentOfFloat = defaultKeyStatistics.ShortPercentOfFloat?.DecimalValue;
fundamentals.PercentHeldByInstitutions = defaultKeyStatistics.HeldPercentInstitutions?.DecimalValue;
fundamentals.PercentHeldByInsiders = defaultKeyStatistics.HeldPercentInsiders?.DecimalValue;
}
tickerData.PsRatio = defaultKeyStatistics?.PriceToSalesTrailing12Months?.DecimalValue ??
summaryDetail?.PriceToSalesTrailing12Months?.DecimalValue;
tickerData.EvToEbitda = defaultKeyStatistics?.EnterpriseToEbitda?.DecimalValue;
tickerData.EvToRevenue = defaultKeyStatistics?.EnterpriseToRevenue?.DecimalValue;
tickerData.DividendYield = summaryDetail?.DividendYield?.DecimalValue;
tickerData.PayoutRatio = summaryDetail?.PayoutRatio?.DecimalValue;
if (financialData != null)
{
if (!string.IsNullOrWhiteSpace(financialData.RecommendationKey) &&
!financialData.RecommendationKey.Equals("none", StringComparison.OrdinalIgnoreCase))
{
fundamentals.ConsensusRating = financialData.RecommendationKey;
}
else if (financialData.RecommendationMean != null && financialData.RecommendationMean.Raw.HasValue)
{
double mean = financialData.RecommendationMean.Raw.Value;
fundamentals.ConsensusRating = mean <= 1.8
? "strong_buy"
: (mean <= 2.5 ? "buy" : (mean <= 3.5 ? "hold" : (mean <= 4.2 ? "sell" : "strong_sell")));
}
}
// 4. Calendar Events Data
if (calendarEvents != null)
{
if (calendarEvents.ExDividendDate?.Raw.HasValue == true)
{
long seconds = (long)calendarEvents.ExDividendDate.Raw.Value;
if (seconds > 0)
fundamentals.ExDividendDate = DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime;
}
if (calendarEvents.Earnings?.EarningsDate != null && calendarEvents.Earnings.EarningsDate.Count > 0)
{
var firstDate = calendarEvents.Earnings.EarningsDate[0];
if (firstDate.Raw.HasValue && firstDate.Raw.Value > 0)
{
fundamentals.NextEarningsDate =
DateTimeOffset.FromUnixTimeSeconds((long)firstDate.Raw.Value).UtcDateTime;
}
}
}
if (!fundamentals.ExDividendDate.HasValue && summaryDetail?.ExDividendDate?.Raw.HasValue == true)
{
long seconds = (long)summaryDetail.ExDividendDate.Raw.Value;
if (seconds > 0) fundamentals.ExDividendDate = DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime;
}
tickerData.ExDividendDate = fundamentals.ExDividendDate;
// 5. Executives List
var executives = new List<CompanyExecutiveEntity>();
if (assetProfile?.CompanyOfficers != null)
{
foreach (var officer in assetProfile.CompanyOfficers)
{
var exec = new CompanyExecutiveEntity
{
Isin = isin,
Name = !string.IsNullOrWhiteSpace(officer.Name) ? officer.Name : "Unknown",
Title = !string.IsNullOrWhiteSpace(officer.Title) ? officer.Title : "Officer",
Age = officer.Age,
Compensation = officer.TotalPay?.DecimalValue
};
executives.Add(exec);
}
}
// 6. Financial Statements
var statements = new List<FinancialStatementEntity>();
// A. Annual Statements
if (root.IncomeStatementHistory?.IncomeStatementHistory != null)
{
foreach (var item in root.IncomeStatementHistory.IncomeStatementHistory)
{
MapIncomeStatement(item, isin, "Annual", statements);
}
}
if (root.BalanceSheetHistory?.BalanceSheetStatements != null)
{
foreach (var item in root.BalanceSheetHistory.BalanceSheetStatements)
{
MapBalanceSheet(item, isin, "Annual", statements);
}
}
if (root.CashflowStatementHistory?.CashflowStatements != null)
{
foreach (var item in root.CashflowStatementHistory.CashflowStatements)
{
MapCashflowStatement(item, isin, "Annual", statements);
}
}
// B. Quarterly Statements
if (root.IncomeStatementHistoryQuarterly?.IncomeStatementHistory != null)
{
foreach (var item in root.IncomeStatementHistoryQuarterly.IncomeStatementHistory)
{
MapIncomeStatement(item, isin, "Quarterly", statements);
}
}
if (root.BalanceSheetHistoryQuarterly?.BalanceSheetStatements != null)
{
foreach (var item in root.BalanceSheetHistoryQuarterly.BalanceSheetStatements)
{
MapBalanceSheet(item, isin, "Quarterly", statements);
}
}
if (root.CashflowStatementHistoryQuarterly?.CashflowStatements != null)
{
foreach (var item in root.CashflowStatementHistoryQuarterly.CashflowStatements)
{
MapCashflowStatement(item, isin, "Quarterly", statements);
}
}
// 7. Forward Estimates
var estimates = new List<ForwardEstimateEntity>();
return new ScrapedFundamentalsData(fundamentals, tickerData, executives, statements, estimates);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to scrape fundamentals for ISIN {Isin} (Ticker: {Ticker})",
"FundamentalsChannel", isin, ticker);
return null;
}
}
private static void MapIncomeStatement(YahooIncomeStatementDto item, string isin, string periodType,
List<FinancialStatementEntity> statements)
{
if (item.EndDate?.Raw.HasValue != true) return;
var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date;
var statement = GetOrCreateStatement(statements, isin, periodType, endDate);
if (item.TotalRevenue?.Raw.HasValue == true) statement.TotalRevenue = item.TotalRevenue.DecimalValue;
if (item.CostOfRevenue?.Raw.HasValue == true) statement.CostOfRevenue = item.CostOfRevenue.DecimalValue;
if (item.GrossProfit?.Raw.HasValue == true) statement.GrossProfit = item.GrossProfit.DecimalValue;
else if (statement.TotalRevenue.HasValue && statement.CostOfRevenue.HasValue)
statement.GrossProfit = statement.TotalRevenue - statement.CostOfRevenue;
if (item.TotalOperatingExpenses?.Raw.HasValue == true)
statement.OperatingExpenses = item.TotalOperatingExpenses.DecimalValue;
if (item.OperatingIncome?.Raw.HasValue == true) statement.OperatingIncome = item.OperatingIncome.DecimalValue;
else if (statement.GrossProfit.HasValue && statement.OperatingExpenses.HasValue)
statement.OperatingIncome = statement.GrossProfit - statement.OperatingExpenses;
if (item.Ebit?.Raw.HasValue == true) statement.Ebitda = item.Ebit.DecimalValue;
if (item.NetIncome?.Raw.HasValue == true) statement.NetIncome = item.NetIncome.DecimalValue;
}
private static void MapBalanceSheet(YahooBalanceSheetStatementDto item, string isin, string periodType,
List<FinancialStatementEntity> statements)
{
if (item.EndDate?.Raw.HasValue != true) return;
var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date;
var statement = GetOrCreateStatement(statements, isin, periodType, endDate);
if (item.Cash?.Raw.HasValue == true) statement.CashAndCashEquivalents = item.Cash.DecimalValue;
if (item.NetReceivables?.Raw.HasValue == true) statement.AccountsReceivable = item.NetReceivables.DecimalValue;
if (item.Inventory?.Raw.HasValue == true) statement.Inventory = item.Inventory.DecimalValue;
if (item.TotalCurrentAssets?.Raw.HasValue == true)
statement.TotalCurrentAssets = item.TotalCurrentAssets.DecimalValue;
if (item.TotalCurrentLiabilities?.Raw.HasValue == true)
statement.CurrentLiabilities = item.TotalCurrentLiabilities.DecimalValue;
if (item.LongTermDebt?.Raw.HasValue == true) statement.LongTermDebt = item.LongTermDebt.DecimalValue;
if (item.TotalLiab?.Raw.HasValue == true) statement.TotalLiabilities = item.TotalLiab.DecimalValue;
if (item.TotalStockholderEquity?.Raw.HasValue == true)
statement.TotalStockholdersEquity = item.TotalStockholderEquity.DecimalValue;
}
private static void MapCashflowStatement(YahooCashflowStatementDto item, string isin, string periodType,
List<FinancialStatementEntity> statements)
{
if (item.EndDate?.Raw.HasValue != true) return;
var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date;
var statement = GetOrCreateStatement(statements, isin, periodType, endDate);
if (item.TotalCashFromOperatingActivities?.Raw.HasValue == true)
statement.OperatingCashFlow = item.TotalCashFromOperatingActivities.DecimalValue;
if (item.TotalCashflowsFromInvestingActivities?.Raw.HasValue == true)
statement.InvestingCashFlow = item.TotalCashflowsFromInvestingActivities.DecimalValue;
if (item.CapitalExpenditures?.Raw.HasValue == true)
statement.CapitalExpenditures = item.CapitalExpenditures.DecimalValue;
if (item.TotalCashFromFinancingActivities?.Raw.HasValue == true)
statement.FinancingCashFlow = item.TotalCashFromFinancingActivities.DecimalValue;
if (statement.OperatingCashFlow.HasValue)
{
var capex = statement.CapitalExpenditures ?? 0m;
statement.FreeCashFlow = statement.OperatingCashFlow.Value - Math.Abs(capex);
}
}
private static FinancialStatementEntity GetOrCreateStatement(List<FinancialStatementEntity> statements, string isin,
string periodType, DateTime endDate)
{
var existing = statements.FirstOrDefault(s => s.PeriodType == periodType && s.EndDate.Date == endDate.Date);
if (existing == null)
{
existing = new FinancialStatementEntity
{
Isin = isin,
PeriodType = periodType,
EndDate = endDate.Date
};
statements.Add(existing);
}
return existing;
}
}