using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Clients;
using FinlyticCore.Dtos.Fundamentals;
using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Util;
using Microsoft.Extensions.Logging;
namespace FinlyticFundamentals.Services;
public interface IYahooFinanceScraper
{
///
/// Ermittelt den primären Börsenticker zu einer ISIN anhand von Börsenplatz-Prioritäten.
///
Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default);
///
/// Ermittelt alle gefundenen Börsenticker zu einer ISIN, sortiert nach Priorität.
///
Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default);
///
/// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab
/// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind.
///
/// Das Tickersymbol (z. B. "MSFT") oder die ISIN.
/// Erzwingt sofortiges HTML-Scraping ohne API-Vorprüfung.
/// Abbruch-Token.
/// Das aggregierte oder null.
Task GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
CancellationToken cancellationToken = default);
}
public class YahooFinanceScraper : IYahooFinanceScraper
{
private readonly YahooFinanceClient _yahooApiClient;
private readonly IYahooFinanceHtmlClient _htmlScraperClient;
private readonly IFinlyticLogger _finlyticLogger;
public YahooFinanceScraper(
YahooFinanceClient yahooApiClient,
IYahooFinanceHtmlClient htmlScraperClient,
IFinlyticLogger finlyticLogger)
{
_yahooApiClient = yahooApiClient;
_htmlScraperClient = htmlScraperClient;
_finlyticLogger = finlyticLogger;
}
///
public async Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default)
{
var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken);
return tickers.FirstOrDefault();
}
///
public async Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(isin)) return new List();
var cleanIsin = isin.Trim().ToUpperInvariant();
var symbols = new List<(string symbol, string exchange, int priority)>();
try
{
// 1. Suche via ISIN
var primary = await _yahooApiClient.SearchAsync(cleanIsin, quotesCount: 20, cancellationToken: cancellationToken);
var quotes = primary?.Quotes ?? new List();
foreach (var q in quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)))
{
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, GetExchangePriority(q.Symbol, cleanIsin)));
}
// 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind
if (quotes.Count > 0)
{
var companyName = quotes[0].LongName ?? quotes[0].ShortName;
if (!string.IsNullOrWhiteSpace(companyName))
{
var secondary = await _yahooApiClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken);
foreach (var q in secondary?.Quotes ?? new List())
{
if (!string.IsNullOrEmpty(q.Symbol) &&
!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, GetExchangePriority(q.Symbol, cleanIsin)));
}
}
}
}
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] Fehler beim Auflösen des Tickers für ISIN '{Isin}'", cleanIsin);
}
return symbols
.OrderBy(s => s.priority)
.Select(s => new TickerInfoDto(){Ticker = s.symbol, Exchange = s.exchange})
.ToList();
}
///
public async Task GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null;
var symbol = symbolOrIsin.Trim().ToUpperInvariant();
// Falls eine ISIN übergeben wurde, zuerst Ticker auflösen
if (IsIsin(symbol))
{
var resolvedTicker = await ResolveTickerFromIsinAsync(symbol, cancellationToken);
if (resolvedTicker != null)
{
symbol = resolvedTicker.Ticker;
}
}
YahooQuoteSummaryModulesDto? apiModules = null;
// -------------------------------------------------------------
// 1. PRIMÄRE DATENQUELLE: Yahoo Finance API (Cookie/Crumb)
// -------------------------------------------------------------
if (!forceHtmlScrape)
{
try
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Starte primären API-Abruf für '{Symbol}'...", symbol);
var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken);
apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault();
if (apiModules != null && HasSufficientData(apiModules))
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Erfolgreich Daten über API bezogen für '{Symbol}'.", symbol);
return apiModules;
}
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] API lieferte unvollständige Daten für '{Symbol}'. Initiiere Fallback...", symbol);
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] API-Abruf fehlgeschlagen für '{Symbol}'. Wechsle zu Scraper...", symbol);
}
}
// -------------------------------------------------------------
// 2. FALLBACK DATENQUELLE: Playwright HTML Scraper
// -------------------------------------------------------------
YahooQuoteSummaryModulesDto? htmlModules = null;
try
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Starte HTML-Scraper Fallback für '{Symbol}'...", symbol);
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{Symbol}'.", symbol);
}
// -------------------------------------------------------------
// 3. Zusammenführen (Merge API & HTML Fallback)
// -------------------------------------------------------------
if (apiModules == null) return htmlModules;
if (htmlModules == null) return apiModules;
return MergeModules(apiModules, htmlModules);
}
///
/// Prüft, ob das Modul-DTO die wesentlichen Fundamentalblöcke enthält.
///
private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules)
{
return modules.SummaryDetail != null ||
modules.FinancialData != null ||
modules.DefaultKeyStatistics != null;
}
///
/// Führt API- und Scraper-Daten zusammen, damit Lücken in API-Responses geschlossen werden.
///
private static YahooQuoteSummaryModulesDto MergeModules(
YahooQuoteSummaryModulesDto primary,
YahooQuoteSummaryModulesDto secondary)
{
return new YahooQuoteSummaryModulesDto(
QuoteType: primary.QuoteType ?? secondary.QuoteType,
AssetProfile: primary.AssetProfile ?? secondary.AssetProfile,
FinancialData: primary.FinancialData ?? secondary.FinancialData,
DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics,
SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail,
IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory,
IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly,
BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory,
BalanceSheetHistoryQuarterly: primary.BalanceSheetHistoryQuarterly ?? secondary.BalanceSheetHistoryQuarterly,
CashflowStatementHistory: primary.CashflowStatementHistory ?? secondary.CashflowStatementHistory,
CashflowStatementHistoryQuarterly: primary.CashflowStatementHistoryQuarterly ?? secondary.CashflowStatementHistoryQuarterly,
CalendarEvents: primary.CalendarEvents ?? secondary.CalendarEvents
);
}
private static bool IsIsin(string value)
{
return value.Length == 12 &&
char.IsLetter(value[0]) &&
char.IsLetter(value[1]) &&
value.All(char.IsLetterOrDigit);
}
private static int GetExchangePriority(string symbol, string isin)
{
bool isGermanIsin = isin.StartsWith("DE", StringComparison.OrdinalIgnoreCase);
if (isGermanIsin)
{
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 1; // Xetra
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 2; // Frankfurt
if (symbol.EndsWith(".STU", StringComparison.OrdinalIgnoreCase)) return 3; // Stuttgart
if (symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 4; // Stuttgart (alt)
if (symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) return 5; // Hamburg
if (!symbol.Contains('.')) return 6; // US Primary
}
else
{
if (!symbol.Contains('.')) return 1; // US Primary (NASDAQ, NYSE)
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; // Xetra
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 3; // Frankfurt
if (symbol.EndsWith(".L", StringComparison.OrdinalIgnoreCase)) return 4; // London
if (symbol.EndsWith(".PA", StringComparison.OrdinalIgnoreCase)) return 5; // Paris
}
return 10;
}
}