feat(core): dynamic settings service, IFinlyticLogger, log broadcaster, and persistent Yahoo auth
This commit is contained in:
@@ -0,0 +1,301 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Clients;
|
||||
using FinlyticCore.Dtos.Fundamentals;
|
||||
using FinlyticCore.Dtos.Yahoo;
|
||||
using FinlyticCore.Models.Settings;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticCore.Utils;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace FinlyticCore.Services.Yahoo;
|
||||
|
||||
public interface IYahooFinanceScraper
|
||||
{
|
||||
/// <summary>
|
||||
/// Ermittelt den primären Börsenticker zu einer ISIN anhand von Börsenplatz-Prioritäten.
|
||||
/// </summary>
|
||||
Task<TickerInfoDto?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Ermittelt alle gefundenen Börsenticker zu einer ISIN, sortiert nach Priorität.
|
||||
/// </summary>
|
||||
Task<List<TickerInfoDto>> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab
|
||||
/// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind.
|
||||
/// </summary>
|
||||
Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
|
||||
string symbolOrIsin,
|
||||
bool forceHtmlScrape = false,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
{
|
||||
private const string _serviceName = nameof(YahooFinanceScraper);
|
||||
private readonly YahooFinanceClient _yahooApiClient;
|
||||
private readonly IYahooFinanceHtmlClient _htmlScraperClient;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly IFinlyticLogger<YahooFinanceScraper> _finlyticLogger;
|
||||
|
||||
public YahooFinanceScraper(
|
||||
YahooFinanceClient yahooApiClient,
|
||||
IYahooFinanceHtmlClient htmlScraperClient,
|
||||
IConfiguration configuration,
|
||||
IFinlyticLogger<YahooFinanceScraper> finlyticLogger)
|
||||
{
|
||||
_yahooApiClient = yahooApiClient;
|
||||
_htmlScraperClient = htmlScraperClient;
|
||||
_configuration = configuration;
|
||||
_finlyticLogger = finlyticLogger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<TickerInfoDto?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken);
|
||||
return tickers.FirstOrDefault();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<TickerInfoDto>> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isin)) return new List<TickerInfoDto>();
|
||||
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
var symbols = new List<(string symbol, string exchange, int priority)>();
|
||||
|
||||
// Crypto / Trade Republic interne ISINs (beginnend mit 'X', z. B. XF000BTC0017)
|
||||
if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var (cryptoSubtitle, cryptoName) = await CryptoSubtitleResolver.ResolveCryptoInfoAsync(
|
||||
cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cryptoSubtitle))
|
||||
{
|
||||
var cryptoEur = $"{cryptoSubtitle}-EUR";
|
||||
var cryptoUsd = $"{cryptoSubtitle}-USD";
|
||||
|
||||
symbols.Add((cryptoEur, "Crypto", 0));
|
||||
symbols.Add((cryptoUsd, "Crypto", 1));
|
||||
|
||||
try
|
||||
{
|
||||
var searchRes = await _yahooApiClient.SearchAsync(cryptoSubtitle, quotesCount: 10, cancellationToken: cancellationToken);
|
||||
if (searchRes?.Quotes != null)
|
||||
{
|
||||
foreach (var q in searchRes.Quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)))
|
||||
{
|
||||
if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
symbols.Add((q.Symbol, q.Exchange ?? "Crypto", 2));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel,
|
||||
$"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}");
|
||||
|
||||
return symbols
|
||||
.OrderBy(s => s.priority)
|
||||
.Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange })
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 1. Suche via ISIN - der allererste Ticker von Yahoo Finance ist der absolute Primary Ticker
|
||||
var primary = await _yahooApiClient.SearchAsync(cleanIsin, quotesCount: 20, cancellationToken: cancellationToken);
|
||||
var quotes = primary?.Quotes ?? new List<YahooSearchQuoteDto>();
|
||||
var validQuotes = quotes.Where(q => !string.IsNullOrEmpty(q.Symbol)).ToList();
|
||||
|
||||
if (validQuotes.Count > 0)
|
||||
{
|
||||
var first = validQuotes[0];
|
||||
symbols.Add((first.Symbol, first.Exchange ?? string.Empty, 0));
|
||||
|
||||
foreach (var q in validQuotes.Skip(1))
|
||||
{
|
||||
if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind
|
||||
if (validQuotes.Count > 0)
|
||||
{
|
||||
var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName;
|
||||
if (!string.IsNullOrWhiteSpace(companyName))
|
||||
{
|
||||
var secondary = await _yahooApiClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken);
|
||||
foreach (var q in secondary?.Quotes ?? new List<YahooSearchQuoteDto>())
|
||||
{
|
||||
if (!string.IsNullOrEmpty(q.Symbol) &&
|
||||
!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin))));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
|
||||
$"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'");
|
||||
}
|
||||
|
||||
return symbols
|
||||
.OrderBy(s => s.priority)
|
||||
.Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange })
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
|
||||
string symbolOrIsin,
|
||||
bool forceHtmlScrape = false,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null;
|
||||
|
||||
var symbol = symbolOrIsin.Trim().ToUpperInvariant();
|
||||
|
||||
// Falls eine ISIN übergeben wurde, zuerst Ticker auflösen
|
||||
if (IsIsin(symbol))
|
||||
{
|
||||
var resolvedTicker = await ResolveTickerFromIsinAsync(symbol, cancellationToken);
|
||||
if (resolvedTicker != null)
|
||||
{
|
||||
symbol = resolvedTicker.Ticker;
|
||||
}
|
||||
}
|
||||
|
||||
YahooQuoteSummaryModulesDto? apiModules = null;
|
||||
|
||||
// -------------------------------------------------------------
|
||||
// 1. PRIMÄRE DATENQUELLE: Yahoo Finance API (Cookie/Crumb)
|
||||
// -------------------------------------------------------------
|
||||
if (!forceHtmlScrape)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
|
||||
$"[{_serviceName}] Starte primären API-Abruf für '{symbol}'...");
|
||||
|
||||
var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken);
|
||||
apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault();
|
||||
|
||||
if (apiModules != null && HasSufficientData(apiModules))
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
|
||||
$"[{_serviceName}] Erfolgreich Daten über API bezogen für '{symbol}'.");
|
||||
return apiModules;
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
|
||||
$"[{_serviceName}] API lieferte unvollständige Daten für '{symbol}'. Initiiere Fallback...");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
|
||||
$"[{_serviceName}] API-Abruf fehlgeschlagen für '{symbol}'. Wechsle zu Scraper...");
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------
|
||||
// 2. FALLBACK DATENQUELLE: Playwright HTML Scraper
|
||||
// -------------------------------------------------------------
|
||||
YahooQuoteSummaryModulesDto? htmlModules = null;
|
||||
try
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
|
||||
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}'...");
|
||||
|
||||
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex,
|
||||
$"[{_serviceName}] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{symbol}'.");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------
|
||||
// 3. Zusammenführen (Merge API & HTML Fallback)
|
||||
// -------------------------------------------------------------
|
||||
if (apiModules == null) return htmlModules;
|
||||
if (htmlModules == null) return apiModules;
|
||||
|
||||
return MergeModules(apiModules, htmlModules);
|
||||
}
|
||||
|
||||
private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules)
|
||||
{
|
||||
return modules.SummaryDetail != null ||
|
||||
modules.FinancialData != null ||
|
||||
modules.DefaultKeyStatistics != null;
|
||||
}
|
||||
|
||||
private static YahooQuoteSummaryModulesDto MergeModules(
|
||||
YahooQuoteSummaryModulesDto primary,
|
||||
YahooQuoteSummaryModulesDto secondary)
|
||||
{
|
||||
return new YahooQuoteSummaryModulesDto(
|
||||
QuoteType: primary.QuoteType ?? secondary.QuoteType,
|
||||
AssetProfile: primary.AssetProfile ?? secondary.AssetProfile,
|
||||
FinancialData: primary.FinancialData ?? secondary.FinancialData,
|
||||
DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics,
|
||||
SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail,
|
||||
IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory,
|
||||
IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly,
|
||||
BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory,
|
||||
BalanceSheetHistoryQuarterly: primary.BalanceSheetHistoryQuarterly ?? secondary.BalanceSheetHistoryQuarterly,
|
||||
CashflowStatementHistory: primary.CashflowStatementHistory ?? secondary.CashflowStatementHistory,
|
||||
CashflowStatementHistoryQuarterly: primary.CashflowStatementHistoryQuarterly ?? secondary.CashflowStatementHistoryQuarterly,
|
||||
CalendarEvents: primary.CalendarEvents ?? secondary.CalendarEvents
|
||||
);
|
||||
}
|
||||
|
||||
private static bool IsIsin(string value)
|
||||
{
|
||||
return value.Length == 12 &&
|
||||
char.IsLetter(value[0]) &&
|
||||
char.IsLetter(value[1]) &&
|
||||
value.All(char.IsLetterOrDigit);
|
||||
}
|
||||
|
||||
private static int GetExchangePriority(string symbol, string isin)
|
||||
{
|
||||
bool isGermanIsin = isin.StartsWith("DE", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (isGermanIsin)
|
||||
{
|
||||
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 1; // Xetra
|
||||
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 2; // Frankfurt
|
||||
if (symbol.EndsWith(".STU", StringComparison.OrdinalIgnoreCase)) return 3; // Stuttgart
|
||||
if (symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 4; // Stuttgart (alt)
|
||||
if (symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) return 5; // Hamburg
|
||||
if (!symbol.Contains('.')) return 6; // US Primary
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!symbol.Contains('.')) return 1; // US Primary (NASDAQ, NYSE)
|
||||
if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; // Xetra
|
||||
if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 3; // Frankfurt
|
||||
if (symbol.EndsWith(".L", StringComparison.OrdinalIgnoreCase)) return 4; // London
|
||||
if (symbol.EndsWith(".PA", StringComparison.OrdinalIgnoreCase)) return 5; // Paris
|
||||
}
|
||||
|
||||
return 10;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user