Files

437 lines
21 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Clients;
using FinlyticCore.Dtos.Fundamentals;
using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.Extensions.Configuration;
namespace FinlyticCore.Services.Yahoo;
public interface IYahooFinanceScraper
{
/// <summary>
/// Ermittelt den primären Börsenticker zu einer ISIN anhand von Börsenplatz-Prioritäten.
/// </summary>
Task<TickerInfoDto?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Ermittelt alle gefundenen Börsenticker zu einer ISIN, sortiert nach Priorität.
/// </summary>
Task<List<TickerInfoDto>> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab
/// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind.
/// </summary>
Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
bool includeProfile = true,
CancellationToken cancellationToken = default);
}
public class YahooFinanceScraper : IYahooFinanceScraper
{
private const string _serviceName = nameof(YahooFinanceScraper);
private readonly YahooFinanceClient _yahooApiClient;
private readonly IYahooFinanceHtmlClient _htmlScraperClient;
private readonly IConfiguration _configuration;
private readonly IFinlyticLogger<YahooFinanceScraper> _finlyticLogger;
public YahooFinanceScraper(
YahooFinanceClient yahooApiClient,
IYahooFinanceHtmlClient htmlScraperClient,
IConfiguration configuration,
IFinlyticLogger<YahooFinanceScraper> finlyticLogger)
{
_yahooApiClient = yahooApiClient;
_htmlScraperClient = htmlScraperClient;
_configuration = configuration;
_finlyticLogger = finlyticLogger;
}
/// <inheritdoc />
public async Task<TickerInfoDto?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default)
{
var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken);
return tickers.FirstOrDefault();
}
/// <inheritdoc />
public async Task<List<TickerInfoDto>> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(isin)) return new List<TickerInfoDto>();
var cleanIsin = isin.Trim().ToUpperInvariant();
var symbols = new List<(string symbol, string exchange, int priority)>();
// Crypto / Trade Republic interne ISINs (beginnend mit 'X', z. B. XF000BTC0017)
if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase))
{
var (cryptoSubtitle, cryptoName) = await CryptoSubtitleResolver.ResolveCryptoInfoAsync(
cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken);
if (!string.IsNullOrWhiteSpace(cryptoSubtitle))
{
var cryptoEur = $"{cryptoSubtitle}-EUR";
var cryptoUsd = $"{cryptoSubtitle}-USD";
symbols.Add((cryptoEur, "Crypto", 0));
symbols.Add((cryptoUsd, "Crypto", 1));
try
{
var searchRes = await _yahooApiClient.SearchAsync(cryptoSubtitle, quotesCount: 10, cancellationToken: cancellationToken);
if (searchRes?.Quotes != null)
{
foreach (var q in searchRes.Quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)))
{
if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, q.Exchange ?? "Crypto", 2));
}
}
}
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] Crypto search failed for {cryptoSubtitle}");
}
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel,
$"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}");
return symbols
.OrderBy(s => s.priority)
.Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange })
.ToList();
}
}
try
{
// 1. Suche via ISIN - der allererste Ticker von Yahoo Finance ist der absolute Primary Ticker
var primary = await _yahooApiClient.SearchAsync(cleanIsin, quotesCount: 20, cancellationToken: cancellationToken);
var quotes = primary?.Quotes ?? new List<YahooSearchQuoteDto>();
var validQuotes = quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)).ToList();
if (validQuotes.Count > 0)
{
var first = validQuotes[0];
symbols.Add((first.Symbol, first.Exchange ?? string.Empty, 0));
foreach (var q in validQuotes.Skip(1))
{
if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin))));
}
}
}
// 2. Falls Ticker gefunden, mit Unternehmensname noch mehr internationale Exchangeticker suchen (z.B. APC.DE)
if (validQuotes.Count > 0)
{
var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName;
if (!string.IsNullOrWhiteSpace(companyName))
{
var secondary = await _yahooApiClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken);
foreach (var q in secondary?.Quotes ?? new List<YahooSearchQuoteDto>())
{
if (!string.IsNullOrEmpty(q.Symbol) &&
!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin))));
}
}
}
}
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'");
}
var result = symbols
.OrderBy(s => s.priority)
.Select(s => new TickerInfoDto
{
Ticker = s.symbol,
Exchange = !string.IsNullOrWhiteSpace(s.exchange) ? s.exchange : "Unknown"
})
.DistinctBy(s => s.Ticker, StringComparer.OrdinalIgnoreCase)
.ToList();
return result;
}
/// <inheritdoc />
public async Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
bool includeProfile = true,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null;
var symbol = symbolOrIsin.Trim().ToUpperInvariant();
// Falls eine ISIN übergeben wurde, zuerst Ticker auflösen
if (IsIsin(symbol))
{
var resolvedTicker = await ResolveTickerFromIsinAsync(symbol, cancellationToken);
if (resolvedTicker != null)
{
symbol = resolvedTicker.Ticker;
}
}
YahooQuoteSummaryModulesDto? apiModules = null;
// -------------------------------------------------------------
// 1. PRIMÄRE DATENQUELLE: Yahoo Finance API (Cookie/Crumb)
// -------------------------------------------------------------
if (!forceHtmlScrape)
{
try
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte primären API-Abruf für '{symbol}'...");
var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken);
apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault();
if (apiModules != null && HasSufficientData(apiModules))
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Erfolgreich Daten über API bezogen für '{symbol}'.");
return apiModules;
}
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] API lieferte unvollständige Daten für '{symbol}'. Initiiere Fallback...");
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] API-Abruf fehlgeschlagen für '{symbol}'. Wechsle zu Scraper...");
}
}
// -------------------------------------------------------------
// 2. FALLBACK DATENQUELLE: Playwright HTML Scraper
// -------------------------------------------------------------
YahooQuoteSummaryModulesDto? htmlModules = null;
try
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}' (IncludeProfile: {includeProfile})...");
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, includeProfile, cancellationToken);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{symbol}'.");
}
// -------------------------------------------------------------
// 3. Zusammenführen (Merge API & HTML Fallback)
// -------------------------------------------------------------
if (apiModules == null) return htmlModules;
if (htmlModules == null) return apiModules;
return MergeModules(apiModules, htmlModules);
}
private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules)
{
return modules.SummaryDetail != null ||
modules.FinancialData != null ||
modules.DefaultKeyStatistics != null;
}
public static YahooQuoteSummaryModulesDto? MergeModules(
YahooQuoteSummaryModulesDto? primary,
YahooQuoteSummaryModulesDto? secondary)
{
if (primary == null && secondary == null) return null;
if (primary == null) return secondary;
if (secondary == null) return primary;
return new YahooQuoteSummaryModulesDto(
QuoteType: primary.QuoteType ?? secondary.QuoteType,
AssetProfile: primary.AssetProfile ?? secondary.AssetProfile,
FinancialData: MergeFinancialData(primary.FinancialData, secondary.FinancialData),
DefaultKeyStatistics: MergeDefaultKeyStatistics(primary.DefaultKeyStatistics, secondary.DefaultKeyStatistics),
SummaryDetail: MergeSummaryDetail(primary.SummaryDetail, secondary.SummaryDetail),
IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory,
IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly,
BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory,
BalanceSheetHistoryQuarterly: primary.BalanceSheetHistoryQuarterly ?? secondary.BalanceSheetHistoryQuarterly,
CashflowStatementHistory: primary.CashflowStatementHistory ?? secondary.CashflowStatementHistory,
CashflowStatementHistoryQuarterly: primary.CashflowStatementHistoryQuarterly ?? secondary.CashflowStatementHistoryQuarterly,
CalendarEvents: primary.CalendarEvents ?? secondary.CalendarEvents
);
}
private static YahooFinancialDataDto? MergeFinancialData(YahooFinancialDataDto? a, YahooFinancialDataDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooFinancialDataDto(
CurrentPrice: a.CurrentPrice ?? b.CurrentPrice,
TargetHighPrice: a.TargetHighPrice ?? b.TargetHighPrice,
TargetLowPrice: a.TargetLowPrice ?? b.TargetLowPrice,
TargetMeanPrice: a.TargetMeanPrice ?? b.TargetMeanPrice,
TargetMedianPrice: a.TargetMedianPrice ?? b.TargetMedianPrice,
RecommendationMean: a.RecommendationMean ?? b.RecommendationMean,
RecommendationKey: !string.IsNullOrWhiteSpace(a.RecommendationKey) && a.RecommendationKey != "none" ? a.RecommendationKey : b.RecommendationKey,
NumberOfAnalystOpinions: a.NumberOfAnalystOpinions ?? b.NumberOfAnalystOpinions,
TotalCash: a.TotalCash ?? b.TotalCash,
TotalCashPerShare: a.TotalCashPerShare ?? b.TotalCashPerShare,
Ebitda: a.Ebitda ?? b.Ebitda,
TotalDebt: a.TotalDebt ?? b.TotalDebt,
QuickRatio: a.QuickRatio ?? b.QuickRatio,
CurrentRatio: a.CurrentRatio ?? b.CurrentRatio,
TotalRevenue: a.TotalRevenue ?? b.TotalRevenue,
DebtToEquity: a.DebtToEquity ?? b.DebtToEquity,
RevenuePerShare: a.RevenuePerShare ?? b.RevenuePerShare,
ReturnOnAssets: a.ReturnOnAssets ?? b.ReturnOnAssets,
ReturnOnEquity: a.ReturnOnEquity ?? b.ReturnOnEquity,
GrossProfits: a.GrossProfits ?? b.GrossProfits,
FreeCashflow: a.FreeCashflow ?? b.FreeCashflow,
OperatingCashflow: a.OperatingCashflow ?? b.OperatingCashflow,
RevenueGrowth: a.RevenueGrowth ?? b.RevenueGrowth,
GrossMargins: a.GrossMargins ?? b.GrossMargins,
EbitdaMargins: a.EbitdaMargins ?? b.EbitdaMargins,
OperatingMargins: a.OperatingMargins ?? b.OperatingMargins,
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
FinancialCurrency: a.FinancialCurrency ?? b.FinancialCurrency
);
}
private static YahooDefaultKeyStatisticsDto? MergeDefaultKeyStatistics(YahooDefaultKeyStatisticsDto? a, YahooDefaultKeyStatisticsDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooDefaultKeyStatisticsDto(
PriceToBook: a.PriceToBook ?? b.PriceToBook,
EnterpriseValue: a.EnterpriseValue ?? b.EnterpriseValue,
ForwardPE: a.ForwardPE ?? b.ForwardPE,
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
FloatShares: a.FloatShares ?? b.FloatShares,
SharesOutstanding: a.SharesOutstanding ?? b.SharesOutstanding,
SharesShort: a.SharesShort ?? b.SharesShort,
SharesShortPriorMonth: a.SharesShortPriorMonth ?? b.SharesShortPriorMonth,
SharesShortPreviousMonthDate: a.SharesShortPreviousMonthDate ?? b.SharesShortPreviousMonthDate,
DateShortInterest: a.DateShortInterest ?? b.DateShortInterest,
SharesPercentSharesOut: a.SharesPercentSharesOut ?? b.SharesPercentSharesOut,
HeldPercentInsiders: a.HeldPercentInsiders ?? b.HeldPercentInsiders,
HeldPercentInstitutions: a.HeldPercentInstitutions ?? b.HeldPercentInstitutions,
ShortRatio: a.ShortRatio ?? b.ShortRatio,
ShortPercentOfFloat: a.ShortPercentOfFloat ?? b.ShortPercentOfFloat,
Beta: a.Beta ?? b.Beta,
Category: a.Category ?? b.Category,
BookValue: a.BookValue ?? b.BookValue,
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
LastFiscalYearEnd: a.LastFiscalYearEnd ?? b.LastFiscalYearEnd,
NextFiscalYearEnd: a.NextFiscalYearEnd ?? b.NextFiscalYearEnd,
MostRecentQuarter: a.MostRecentQuarter ?? b.MostRecentQuarter,
EarningsQuarterlyGrowth: a.EarningsQuarterlyGrowth ?? b.EarningsQuarterlyGrowth,
NetIncomeToCommon: a.NetIncomeToCommon ?? b.NetIncomeToCommon,
TrailingEps: a.TrailingEps ?? b.TrailingEps,
ForwardEps: a.ForwardEps ?? b.ForwardEps,
PegRatio: a.PegRatio ?? b.PegRatio,
EnterpriseToRevenue: a.EnterpriseToRevenue ?? b.EnterpriseToRevenue,
EnterpriseToEbitda: a.EnterpriseToEbitda ?? b.EnterpriseToEbitda,
FiftyTwoWeekChange: a.FiftyTwoWeekChange ?? b.FiftyTwoWeekChange,
SandP52WeekChange: a.SandP52WeekChange ?? b.SandP52WeekChange
);
}
private static YahooSummaryDetailDto? MergeSummaryDetail(YahooSummaryDetailDto? a, YahooSummaryDetailDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooSummaryDetailDto(
MaxAge: a.MaxAge ?? b.MaxAge,
PriceHint: a.PriceHint ?? b.PriceHint,
PreviousClose: a.PreviousClose ?? b.PreviousClose,
Open: a.Open ?? b.Open,
DayLow: a.DayLow ?? b.DayLow,
DayHigh: a.DayHigh ?? b.DayHigh,
RegularMarketPreviousClose: a.RegularMarketPreviousClose ?? b.RegularMarketPreviousClose,
RegularMarketOpen: a.RegularMarketOpen ?? b.RegularMarketOpen,
RegularMarketDayLow: a.RegularMarketDayLow ?? b.RegularMarketDayLow,
RegularMarketDayHigh: a.RegularMarketDayHigh ?? b.RegularMarketDayHigh,
DividendRate: a.DividendRate ?? b.DividendRate,
DividendYield: a.DividendYield ?? b.DividendYield,
ExDividendDate: a.ExDividendDate ?? b.ExDividendDate,
PayoutRatio: a.PayoutRatio ?? b.PayoutRatio,
FiveYearAvgDividendYield: a.FiveYearAvgDividendYield ?? b.FiveYearAvgDividendYield,
Beta: a.Beta ?? b.Beta,
TrailingPE: a.TrailingPE ?? b.TrailingPE,
ForwardPE: a.ForwardPE ?? b.ForwardPE,
Volume: a.Volume ?? b.Volume,
RegularMarketVolume: a.RegularMarketVolume ?? b.RegularMarketVolume,
AverageVolume: a.AverageVolume ?? b.AverageVolume,
AverageVolume10days: a.AverageVolume10days ?? b.AverageVolume10days,
AverageDailyVolume10Day: a.AverageDailyVolume10Day ?? b.AverageDailyVolume10Day,
Bid: a.Bid ?? b.Bid,
Ask: a.Ask ?? b.Ask,
BidSize: a.BidSize ?? b.BidSize,
AskSize: a.AskSize ?? b.AskSize,
MarketCap: a.MarketCap ?? b.MarketCap,
FiftyTwoWeekLow: a.FiftyTwoWeekLow ?? b.FiftyTwoWeekLow,
FiftyTwoWeekHigh: a.FiftyTwoWeekHigh ?? b.FiftyTwoWeekHigh,
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
Currency: a.Currency ?? b.Currency
);
}
private static bool IsIsin(string value)
{
return value.Length == 12 &&
char.IsLetter(value[0]) &&
char.IsLetter(value[1]) &&
value.All(char.IsLetterOrDigit);
}
private static int GetExchangePriority(string symbol, string isin)
{
bool isGermanIsin = isin.StartsWith("DE", StringComparison.OrdinalIgnoreCase);
if (isGermanIsin)
{
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 1; // Xetra
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 2; // Frankfurt
if (symbol.EndsWith(".STU", StringComparison.OrdinalIgnoreCase)) return 3; // Stuttgart
if (symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 4; // Stuttgart (alt)
if (symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) return 5; // Hamburg
if (!symbol.Contains('.')) return 6; // US Primary
}
else
{
if (!symbol.Contains('.')) return 1; // US Primary (NASDAQ, NYSE)
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; // Xetra
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 3; // Frankfurt
if (symbol.EndsWith(".L", StringComparison.OrdinalIgnoreCase)) return 4; // London
if (symbol.EndsWith(".PA", StringComparison.OrdinalIgnoreCase)) return 5; // Paris
}
return 10;
}
}