Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core): update DTOs, Trade Republic client, Yahoo scrapers, and dynamic settings

Browse Source
This commit is contained in:
Kleidukos
2026-08-14 23:55:02 +02:00
parent 3d8af3940b
commit 4f733bf8c3
511 changed files with 1990 additions and 1080548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticCore/Services/Yahoo/YahooFinanceClient.cs
-332
View File
@@ -1,332 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services.Yahoo;
/// <summary>
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
/// </summary>
public class YahooFinanceClient
{
private const string DefaultUserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
private readonly HttpClient _httpClient;
private readonly CookieContainer _cookieContainer;
private readonly ILogger<YahooFinanceClient>? _logger;
private readonly SemaphoreSlim _authLock = new(1, 1);
private string? _crumb;
private DateTime _lastAuthTime = DateTime.MinValue;
/// <summary>
/// Standard modules available for the quoteSummary endpoint.
/// </summary>
public static readonly string[] StandardQuoteSummaryModules = new[]
{
"assetProfile",
"financialData",
"defaultKeyStatistics",
"summaryDetail",
"incomeStatementHistory",
"incomeStatementHistoryQuarterly",
"balanceSheetHistory",
"balanceSheetHistoryQuarterly",
"cashflowStatementHistory",
"cashflowStatementHistoryQuarterly",
"calendarEvents"
};
public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
{
_logger = logger;
_cookieContainer = new CookieContainer();
if (httpClient != null)
{
_httpClient = httpClient;
}
else
{
var handler = new HttpClientHandler
{
CookieContainer = _cookieContainer,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
_httpClient = new HttpClient(handler);
}
if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
{
_httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
}
}
/// <summary>
/// Executes the Cookie (A3) &amp; Crumb token authentication flow.
/// 1. GET https://fc.yahoo.com (sets session A3 cookie)
/// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
/// </summary>
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
CancellationToken cancellationToken = default)
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
await _authLock.WaitAsync(cancellationToken);
try
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
(DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
_logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");
// 1. Send GET request to fc.yahoo.com to obtain session cookie A3
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
{
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
// CookieContainer automatically intercepts and stores 'A3' cookie
}
// 2. Send GET request to getcrumb to obtain the dynamic crumb token
using (var crumbRequest =
new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
{
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (!crumbResponse.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
crumbResponse.StatusCode);
return null;
}
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
_logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
return _crumb;
}
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
return null;
}
finally
{
_authLock.Release();
}
}
/// <summary>
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
/// Note: Does not require Cookie/Crumb authentication.
/// </summary>
public async Task<YahooSearchResponseDto?> SearchAsync(
string query,
int quotesCount = 10,
int newsCount = 0,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(query)) return null;
try
{
var url =
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
response.StatusCode);
return null;
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
return null;
}
}
/// <summary>
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&amp;modules={modules}
/// </summary>
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
IEnumerable<string> modules,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbol)) return null;
var moduleList = string.Join(",", modules);
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken);
}
/// <summary>
/// Convenience method to fetch all standard quoteSummary modules for a given symbol.
/// </summary>
public Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(string symbol,
CancellationToken cancellationToken = default)
{
return GetQuoteSummaryAsync(symbol, StandardQuoteSummaryModules, cancellationToken);
}
/// <summary>
/// Retrieves historical OHLCV chart data for a given symbol.
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&amp;interval={interval}&amp;crumb={crumb}
/// </summary>
public async Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
string range = "1y",
string interval = "1d",
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbol)) return null;
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken);
}
/// <summary>
/// Retrieves quick real-time price quotes for one or more symbols.
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&amp;crumb={crumb}
/// </summary>
public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
CancellationToken cancellationToken = default)
{
var symbolList = symbols.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
if (symbolList.Count == 0) return null;
var symbolsParam = string.Join(",", symbolList);
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken);
}
/// <summary>
/// Convenient helper method to fetch the current live price for a single symbol (e.g., "^VIX").
/// </summary>
public async Task<decimal?> GetLivePriceAsync(string symbol, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbol)) return null;
var quotes = await GetQuotesAsync(new[] { symbol }, cancellationToken);
var item = quotes?.QuoteResponse?.Result?.FirstOrDefault();
if (item?.RegularMarketPrice.HasValue == true && item.RegularMarketPrice.Value > 0)
{
return Convert.ToDecimal(item.RegularMarketPrice.Value);
}
return null;
}
private async Task<T?> ExecuteWithRetryAsync<T>(
Func<string, Task<(bool isAuthError, T? result)>> action,
CancellationToken cancellationToken) where T : class
{
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (isAuthError, result) = await action(crumb);
if (!isAuthError && result != null)
{
return result;
}
if (isAuthError)
{
_logger?.LogInformation(
"[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (_, retryResult) = await action(crumb);
return retryResult;
}
return result;
}
private static JsonSerializerOptions GetJsonOptions()
{
return new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
};
}
}