Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core): dynamic settings service, IFinlyticLogger, log broadcaster, and persistent Yahoo auth

Browse Source
This commit is contained in:
Kleidukos
2026-08-15 21:29:38 +02:00
parent 34fa774cbf
commit 3dbee36ca0
15 changed files with 920 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticCore/Clients/YahooFinanceClient.cs
+166 -144
View File
@@ -7,109 +7,17 @@ using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using Microsoft.Extensions.Logging;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
namespace FinlyticCore.Services.Yahoo;
/// <summary>
/// Thread-sicherer Client für den Zugriff auf die internen Yahoo Finance APIs.
/// Verwaltet automatisch den erforderlichen Cookie- (A3) und Crumb-Token-Authentifizierungs-Flow.
/// </summary>
public interface IYahooFinanceClient
{
/// <summary>
/// Stellt sicher, dass die aktuelle Session über ein gültiges Cookie und einen Crumb-Token verfügt.
/// </summary>
/// <param name="forceRefresh">Erzwingt das Erneuern des Authentifizierungs-Tokens, selbst wenn die Frist noch nicht abgelaufen ist.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Crumb-Token oder <c>null</c>, wenn die Authentifizierung fehlgeschlagen ist.</returns>
Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>
/// Sucht nach Tickern, Namen, ISINs oder Firmen über die Yahoo Finance Such-API.
/// erfordert keine Cookie/Crumb-Authentifizierung.
/// </summary>
/// <param name="query">Der Suchbegriff (z. B. "Apple", "US0378331005", "AAPL").</param>
/// <param name="quotesCount">Die maximale Anzahl an Treffern für Wertpapiere/Aktien.</param>
/// <param name="newsCount">Die maximale Anzahl an News-Treffern.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Suchergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooSearchResponseDto?> SearchAsync(
string query,
int quotesCount = 10,
int newsCount = 0,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft Fundamentaldaten und Unternehmens-Metadaten für ein bestimmtes Symbol über den quoteSummary-Endpunkt ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL", "MSFT").</param>
/// <param name="modules">Die abzufragenden Yahoo-Module (z. B. "assetProfile", "financialData").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Die Abfrageergebnisse als DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
IEnumerable<string> modules,
CancellationToken cancellationToken = default);
/// <summary>
/// Hilfsmethode zum Abrufen aller vordefinierten Standard-Module für ein Tickersymbol.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das vollständige QuoteSummary-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(
string symbol,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft historische Chart- und Kursdaten (OHLCV) für ein Symbol ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="range">Der Abfragezeitraum (z. B. "1d", "1m", "1y", "5y").</param>
/// <param name="interval">Das Intervall der Datenpunkte (z. B. "1m", "5m", "1d", "1wk").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Chart-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
string range = "1y",
string interval = "1d",
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft schnelle Realtime-Preise für eine Liste von Tickersymbolen ab.
/// </summary>
/// <param name="symbols">Eine Liste von Tickersymbolen (z. B. <c>["AAPL", "MSFT", "^GSPC"]</c>).</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Quote-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
CancellationToken cancellationToken = default);
/// <summary>
/// Bequeme Hilfsmethode, um den aktuellen regulären Marktpreis für ein einzelnes Tickersymbol abzufragen.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "^VIX", "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Preis als <see cref="decimal"/> oder <c>null</c>, wenn kein Preis ermittelt werden konnte.</returns>
Task<decimal?> GetLivePriceAsync(
string symbol,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
/// </summary>
public class YahooFinanceClient
{
private const string DefaultUserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
private readonly HttpClient _httpClient;
private readonly CookieContainer _cookieContainer;
private readonly ILogger<YahooFinanceClient>? _logger;
private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
private readonly ISettingsService? _settingsService;
private readonly SemaphoreSlim _authLock = new(1, 1);
private string? _crumb;
@@ -133,9 +41,13 @@ public class YahooFinanceClient
"calendarEvents"
};
public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
public YahooFinanceClient(
IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
ISettingsService? settingsService = null,
HttpClient? httpClient = null)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_settingsService = settingsService;
_cookieContainer = new CookieContainer();
if (httpClient != null)
@@ -147,71 +59,137 @@ public class YahooFinanceClient
var handler = new HttpClientHandler
{
CookieContainer = _cookieContainer,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
UseCookies = true,
AllowAutoRedirect = true
};
_httpClient = new HttpClient(handler);
}
if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
{
_httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
_httpClient = new HttpClient(handler);
_httpClient.DefaultRequestHeaders.Add("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
}
}
/// <summary>
/// Executes the Cookie (A3) &amp; Crumb token authentication flow.
/// 1. GET https://fc.yahoo.com (sets session A3 cookie)
/// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
/// Ensures that an active Yahoo session (Cookie + dynamic Crumb token) is initialized.
/// Uses persistent DB caching and only refreshes when the crumb is invalid or forceRefresh is true.
/// </summary>
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
CancellationToken cancellationToken = default)
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
await _authLock.WaitAsync(cancellationToken);
try
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
(DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
// 1. Check in-memory crumb
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb))
{
return _crumb;
}
_logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");
// 2. Check persistent DB cache via SettingsService
if (!forceRefresh && _settingsService != null)
{
try
{
var cachedCrumb = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCrumb, cancellationToken);
var cachedCookies = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCookie, cancellationToken);
// 1. Send GET request to fc.yahoo.com to obtain session cookie A3
if (!string.IsNullOrWhiteSpace(cachedCrumb) && !string.IsNullOrWhiteSpace(cachedCookies))
{
RestoreCookies(cachedCookies);
_crumb = cachedCrumb;
_lastAuthTime = DateTime.UtcNow;
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Restored cached Yahoo session & crumb from database ({Crumb}).", _crumb);
return _crumb;
}
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Error restoring cached Yahoo session from DB.");
}
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");
// 3. Send GET request to fc.yahoo.com to obtain session cookie A3
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
{
initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
// CookieContainer automatically intercepts and stores 'A3' cookie
}
// 2. Send GET request to getcrumb to obtain the dynamic crumb token
using (var crumbRequest =
new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
// 4. Send GET request to getcrumb to obtain dynamic crumb token
string[] crumbUrls = new[]
{
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (!crumbResponse.IsSuccessStatusCode)
"https://query1.finance.yahoo.com/v1/test/getcrumb",
"https://query2.finance.yahoo.com/v1/test/getcrumb"
};
foreach (var url in crumbUrls)
{
try
{
_logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
crumbResponse.StatusCode);
return null;
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
crumbRequest.Headers.Add("Accept", "*/*");
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (crumbResponse.IsSuccessStatusCode)
{
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
if (!string.IsNullOrWhiteSpace(crumbText))
{
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
// Persist to DB cache via SettingsService
if (_settingsService != null)
{
try
{
var serializedCookies = SerializeCookies();
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken);
if (!string.IsNullOrWhiteSpace(serializedCookies))
{
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
}
}
catch (Exception persistEx)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database.");
}
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb);
return _crumb;
}
}
else
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode);
}
}
catch
{
// Fallthrough to next endpoint
}
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
_logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
return _crumb;
}
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Failed to fetch crumb token from all endpoints.");
return null;
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
if (_finlyticLogger != null)
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
return null;
}
finally
@@ -220,6 +198,48 @@ public class YahooFinanceClient
}
}
private string SerializeCookies()
{
try
{
var cookies = _cookieContainer.GetAllCookies();
var pairs = new List<string>();
foreach (Cookie cookie in cookies)
{
pairs.Add($"{cookie.Name}={cookie.Value}");
}
return string.Join(";", pairs);
}
catch
{
return string.Empty;
}
}
private void RestoreCookies(string serializedCookies)
{
if (string.IsNullOrWhiteSpace(serializedCookies)) return;
try
{
var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
foreach (var part in parts)
{
var eqIdx = part.IndexOf('=');
if (eqIdx > 0 && eqIdx < part.Length - 1)
{
var name = part.Substring(0, eqIdx).Trim();
var val = part.Substring(eqIdx + 1).Trim();
_cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com"));
}
}
}
catch
{
// Ignore cookie restore errors
}
}
/// <summary>
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
@@ -241,8 +261,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode);
return null;
}
@@ -251,7 +271,8 @@ public class YahooFinanceClient
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
if (_finlyticLogger != null)
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
return null;
}
}
@@ -276,8 +297,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
symbol, response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
@@ -318,8 +339,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
@@ -351,7 +372,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
@@ -396,8 +418,8 @@ public class YahooFinanceClient
if (isAuthError)
{
_logger?.LogInformation(
"[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
@@ -413,7 +435,7 @@ public class YahooFinanceClient
return new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
}
}
FinlyticCore/Clients/YahooFinanceHtmlClient.cs
+4 -12
View File
@@ -9,7 +9,6 @@ using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Playwright;
namespace FinlyticCore.Clients;
@@ -21,25 +20,18 @@ public interface IYahooFinanceHtmlClient
CancellationToken cancellationToken = default);
}
public interface IYahooFinanceHtmlClient<TDbContext> : IYahooFinanceHtmlClient
where TDbContext : DbContext
{
}
public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHtmlClient<TDbContext>
where TDbContext : DbContext
public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
{
private const string _serviceName = nameof(YahooFinanceHtmlClient);
private readonly IPlaywrightExecutionService _playwrightService;
private readonly IFinlyticLogger<TContextClass, TDbContext> _finlyticLogger;
private readonly string _serviceName;
private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
public YahooFinanceHtmlClient(
IPlaywrightExecutionService playwrightService,
IFinlyticLogger<TContextClass, TDbContext> finlyticLogger)
IFinlyticLogger<YahooFinanceHtmlClient> finlyticLogger)
{
_playwrightService = playwrightService;
_finlyticLogger = finlyticLogger;
_serviceName = typeof(TContextClass).Name;
}
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(