feat(core): update DTOs, Trade Republic client, Yahoo scrapers, and dynamic settings
This commit is contained in:
@@ -0,0 +1,419 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Dtos.Yahoo;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FinlyticCore.Services.Yahoo;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Thread-sicherer Client für den Zugriff auf die internen Yahoo Finance APIs.
|
||||
/// Verwaltet automatisch den erforderlichen Cookie- (A3) und Crumb-Token-Authentifizierungs-Flow.
|
||||
/// </summary>
|
||||
public interface IYahooFinanceClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Stellt sicher, dass die aktuelle Session über ein gültiges Cookie und einen Crumb-Token verfügt.
|
||||
/// </summary>
|
||||
/// <param name="forceRefresh">Erzwingt das Erneuern des Authentifizierungs-Tokens, selbst wenn die Frist noch nicht abgelaufen ist.</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Der aktuelle Crumb-Token oder <c>null</c>, wenn die Authentifizierung fehlgeschlagen ist.</returns>
|
||||
Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Sucht nach Tickern, Namen, ISINs oder Firmen über die Yahoo Finance Such-API.
|
||||
/// erfordert keine Cookie/Crumb-Authentifizierung.
|
||||
/// </summary>
|
||||
/// <param name="query">Der Suchbegriff (z. B. "Apple", "US0378331005", "AAPL").</param>
|
||||
/// <param name="quotesCount">Die maximale Anzahl an Treffern für Wertpapiere/Aktien.</param>
|
||||
/// <param name="newsCount">Die maximale Anzahl an News-Treffern.</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Das Suchergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
|
||||
Task<YahooSearchResponseDto?> SearchAsync(
|
||||
string query,
|
||||
int quotesCount = 10,
|
||||
int newsCount = 0,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Ruft Fundamentaldaten und Unternehmens-Metadaten für ein bestimmtes Symbol über den quoteSummary-Endpunkt ab.
|
||||
/// </summary>
|
||||
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL", "MSFT").</param>
|
||||
/// <param name="modules">Die abzufragenden Yahoo-Module (z. B. "assetProfile", "financialData").</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Die Abfrageergebnisse als DTO oder <c>null</c> bei Fehlern.</returns>
|
||||
Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
|
||||
string symbol,
|
||||
IEnumerable<string> modules,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Hilfsmethode zum Abrufen aller vordefinierten Standard-Module für ein Tickersymbol.
|
||||
/// </summary>
|
||||
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Das vollständige QuoteSummary-DTO oder <c>null</c> bei Fehlern.</returns>
|
||||
Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(
|
||||
string symbol,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Ruft historische Chart- und Kursdaten (OHLCV) für ein Symbol ab.
|
||||
/// </summary>
|
||||
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
|
||||
/// <param name="range">Der Abfragezeitraum (z. B. "1d", "1m", "1y", "5y").</param>
|
||||
/// <param name="interval">Das Intervall der Datenpunkte (z. B. "1m", "5m", "1d", "1wk").</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Das Chart-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
|
||||
Task<YahooChartResponseDto?> GetChartAsync(
|
||||
string symbol,
|
||||
string range = "1y",
|
||||
string interval = "1d",
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Ruft schnelle Realtime-Preise für eine Liste von Tickersymbolen ab.
|
||||
/// </summary>
|
||||
/// <param name="symbols">Eine Liste von Tickersymbolen (z. B. <c>["AAPL", "MSFT", "^GSPC"]</c>).</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Das Quote-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
|
||||
Task<YahooQuoteResponseDto?> GetQuotesAsync(
|
||||
IEnumerable<string> symbols,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Bequeme Hilfsmethode, um den aktuellen regulären Marktpreis für ein einzelnes Tickersymbol abzufragen.
|
||||
/// </summary>
|
||||
/// <param name="symbol">Das Tickersymbol (z. B. "^VIX", "AAPL").</param>
|
||||
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
|
||||
/// <returns>Der aktuelle Preis als <see cref="decimal"/> oder <c>null</c>, wenn kein Preis ermittelt werden konnte.</returns>
|
||||
Task<decimal?> GetLivePriceAsync(
|
||||
string symbol,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
|
||||
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
|
||||
/// </summary>
|
||||
public class YahooFinanceClient
|
||||
{
|
||||
private const string DefaultUserAgent =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly CookieContainer _cookieContainer;
|
||||
private readonly ILogger<YahooFinanceClient>? _logger;
|
||||
private readonly SemaphoreSlim _authLock = new(1, 1);
|
||||
|
||||
private string? _crumb;
|
||||
private DateTime _lastAuthTime = DateTime.MinValue;
|
||||
|
||||
/// <summary>
|
||||
/// Standard modules available for the quoteSummary endpoint.
|
||||
/// </summary>
|
||||
public static readonly string[] StandardQuoteSummaryModules = new[]
|
||||
{
|
||||
"assetProfile",
|
||||
"financialData",
|
||||
"defaultKeyStatistics",
|
||||
"summaryDetail",
|
||||
"incomeStatementHistory",
|
||||
"incomeStatementHistoryQuarterly",
|
||||
"balanceSheetHistory",
|
||||
"balanceSheetHistoryQuarterly",
|
||||
"cashflowStatementHistory",
|
||||
"cashflowStatementHistoryQuarterly",
|
||||
"calendarEvents"
|
||||
};
|
||||
|
||||
public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_cookieContainer = new CookieContainer();
|
||||
|
||||
if (httpClient != null)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
else
|
||||
{
|
||||
var handler = new HttpClientHandler
|
||||
{
|
||||
CookieContainer = _cookieContainer,
|
||||
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
|
||||
};
|
||||
_httpClient = new HttpClient(handler);
|
||||
}
|
||||
|
||||
if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
|
||||
{
|
||||
_httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the Cookie (A3) & Crumb token authentication flow.
|
||||
/// 1. GET https://fc.yahoo.com (sets session A3 cookie)
|
||||
/// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
|
||||
/// </summary>
|
||||
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
|
||||
{
|
||||
return _crumb;
|
||||
}
|
||||
|
||||
await _authLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
|
||||
(DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
|
||||
{
|
||||
return _crumb;
|
||||
}
|
||||
|
||||
_logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");
|
||||
|
||||
// 1. Send GET request to fc.yahoo.com to obtain session cookie A3
|
||||
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
|
||||
{
|
||||
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
|
||||
// CookieContainer automatically intercepts and stores 'A3' cookie
|
||||
}
|
||||
|
||||
// 2. Send GET request to getcrumb to obtain the dynamic crumb token
|
||||
using (var crumbRequest =
|
||||
new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
|
||||
{
|
||||
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
|
||||
if (!crumbResponse.IsSuccessStatusCode)
|
||||
{
|
||||
_logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
|
||||
crumbResponse.StatusCode);
|
||||
return null;
|
||||
}
|
||||
|
||||
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
|
||||
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
|
||||
_lastAuthTime = DateTime.UtcNow;
|
||||
|
||||
_logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
|
||||
return _crumb;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_authLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
|
||||
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&quotesCount={quotesCount}&newsCount={newsCount}
|
||||
/// Note: Does not require Cookie/Crumb authentication.
|
||||
/// </summary>
|
||||
public async Task<YahooSearchResponseDto?> SearchAsync(
|
||||
string query,
|
||||
int quotesCount = 10,
|
||||
int newsCount = 0,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query)) return null;
|
||||
|
||||
try
|
||||
{
|
||||
var url =
|
||||
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
|
||||
response.StatusCode);
|
||||
return null;
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
|
||||
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&modules={modules}
|
||||
/// </summary>
|
||||
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
|
||||
string symbol,
|
||||
IEnumerable<string> modules,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(symbol)) return null;
|
||||
|
||||
var moduleList = string.Join(",", modules);
|
||||
return await ExecuteWithRetryAsync(async (crumb) =>
|
||||
{
|
||||
var url =
|
||||
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
|
||||
symbol, response.StatusCode);
|
||||
return (
|
||||
response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden, null);
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
|
||||
return (false, dto);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Convenience method to fetch all standard quoteSummary modules for a given symbol.
|
||||
/// </summary>
|
||||
public Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(string symbol,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return GetQuoteSummaryAsync(symbol, StandardQuoteSummaryModules, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves historical OHLCV chart data for a given symbol.
|
||||
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&interval={interval}&crumb={crumb}
|
||||
/// </summary>
|
||||
public async Task<YahooChartResponseDto?> GetChartAsync(
|
||||
string symbol,
|
||||
string range = "1y",
|
||||
string interval = "1d",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(symbol)) return null;
|
||||
|
||||
return await ExecuteWithRetryAsync(async (crumb) =>
|
||||
{
|
||||
var url =
|
||||
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
|
||||
response.StatusCode);
|
||||
return (
|
||||
response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden, null);
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
|
||||
return (false, dto);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves quick real-time price quotes for one or more symbols.
|
||||
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&crumb={crumb}
|
||||
/// </summary>
|
||||
public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
|
||||
IEnumerable<string> symbols,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var symbolList = symbols.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
|
||||
if (symbolList.Count == 0) return null;
|
||||
|
||||
var symbolsParam = string.Join(",", symbolList);
|
||||
return await ExecuteWithRetryAsync(async (crumb) =>
|
||||
{
|
||||
var url =
|
||||
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
|
||||
return (
|
||||
response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden, null);
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
|
||||
return (false, dto);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Convenient helper method to fetch the current live price for a single symbol (e.g., "^VIX").
|
||||
/// </summary>
|
||||
public async Task<decimal?> GetLivePriceAsync(string symbol, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(symbol)) return null;
|
||||
|
||||
var quotes = await GetQuotesAsync(new[] { symbol }, cancellationToken);
|
||||
var item = quotes?.QuoteResponse?.Result?.FirstOrDefault();
|
||||
|
||||
if (item?.RegularMarketPrice.HasValue == true && item.RegularMarketPrice.Value > 0)
|
||||
{
|
||||
return Convert.ToDecimal(item.RegularMarketPrice.Value);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<T?> ExecuteWithRetryAsync<T>(
|
||||
Func<string, Task<(bool isAuthError, T? result)>> action,
|
||||
CancellationToken cancellationToken) where T : class
|
||||
{
|
||||
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
|
||||
if (string.IsNullOrEmpty(crumb)) return null;
|
||||
|
||||
var (isAuthError, result) = await action(crumb);
|
||||
if (!isAuthError && result != null)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
if (isAuthError)
|
||||
{
|
||||
_logger?.LogInformation(
|
||||
"[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
|
||||
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
|
||||
if (string.IsNullOrEmpty(crumb)) return null;
|
||||
|
||||
var (_, retryResult) = await action(crumb);
|
||||
return retryResult;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static JsonSerializerOptions GetJsonOptions()
|
||||
{
|
||||
return new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user