441 lines
19 KiB
C#
441 lines
19 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FinlyticCore.Dtos.Yahoo;
|
|
using FinlyticCore.Models.Settings;
|
|
using FinlyticCore.Services;
|
|
|
|
namespace FinlyticCore.Services.Yahoo;
|
|
|
|
public class YahooFinanceClient
|
|
{
|
|
private readonly HttpClient _httpClient;
|
|
private readonly CookieContainer _cookieContainer;
|
|
private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
|
|
private readonly ISettingsService? _settingsService;
|
|
private readonly SemaphoreSlim _authLock = new(1, 1);
|
|
|
|
private string? _crumb;
|
|
private DateTime _lastAuthTime = DateTime.MinValue;
|
|
|
|
/// <summary>
|
|
/// Standard modules available for the quoteSummary endpoint.
|
|
/// </summary>
|
|
public static readonly string[] StandardQuoteSummaryModules = new[]
|
|
{
|
|
"assetProfile",
|
|
"financialData",
|
|
"defaultKeyStatistics",
|
|
"summaryDetail",
|
|
"incomeStatementHistory",
|
|
"incomeStatementHistoryQuarterly",
|
|
"balanceSheetHistory",
|
|
"balanceSheetHistoryQuarterly",
|
|
"cashflowStatementHistory",
|
|
"cashflowStatementHistoryQuarterly",
|
|
"calendarEvents"
|
|
};
|
|
|
|
public YahooFinanceClient(
|
|
IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
|
|
ISettingsService? settingsService = null,
|
|
HttpClient? httpClient = null)
|
|
{
|
|
_finlyticLogger = finlyticLogger;
|
|
_settingsService = settingsService;
|
|
_cookieContainer = new CookieContainer();
|
|
|
|
if (httpClient != null)
|
|
{
|
|
_httpClient = httpClient;
|
|
}
|
|
else
|
|
{
|
|
var handler = new HttpClientHandler
|
|
{
|
|
CookieContainer = _cookieContainer,
|
|
UseCookies = true,
|
|
AllowAutoRedirect = true
|
|
};
|
|
|
|
_httpClient = new HttpClient(handler);
|
|
_httpClient.DefaultRequestHeaders.Add("User-Agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Ensures that an active Yahoo session (Cookie + dynamic Crumb token) is initialized.
|
|
/// Uses persistent DB caching and only refreshes when the crumb is invalid or forceRefresh is true.
|
|
/// </summary>
|
|
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
await _authLock.WaitAsync(cancellationToken);
|
|
try
|
|
{
|
|
// 1. Check in-memory crumb
|
|
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb))
|
|
{
|
|
return _crumb;
|
|
}
|
|
|
|
// 2. Check persistent DB cache via SettingsService
|
|
if (!forceRefresh && _settingsService != null)
|
|
{
|
|
try
|
|
{
|
|
var cachedCrumb = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCrumb, cancellationToken);
|
|
var cachedCookies = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCookie, cancellationToken);
|
|
|
|
if (!string.IsNullOrWhiteSpace(cachedCrumb) && !string.IsNullOrWhiteSpace(cachedCookies))
|
|
{
|
|
RestoreCookies(cachedCookies);
|
|
_crumb = cachedCrumb;
|
|
_lastAuthTime = DateTime.UtcNow;
|
|
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Restored cached Yahoo session & crumb from database ({Crumb}).", _crumb);
|
|
return _crumb;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Error restoring cached Yahoo session from DB.");
|
|
}
|
|
}
|
|
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");
|
|
|
|
// 3. Send GET request to fc.yahoo.com to obtain session cookie A3
|
|
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
|
|
{
|
|
initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
|
initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
|
|
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
|
|
}
|
|
|
|
// 4. Send GET request to getcrumb to obtain dynamic crumb token
|
|
string[] crumbUrls = new[]
|
|
{
|
|
"https://query1.finance.yahoo.com/v1/test/getcrumb",
|
|
"https://query2.finance.yahoo.com/v1/test/getcrumb"
|
|
};
|
|
|
|
foreach (var url in crumbUrls)
|
|
{
|
|
try
|
|
{
|
|
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
|
|
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
|
crumbRequest.Headers.Add("Accept", "*/*");
|
|
|
|
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
|
|
if (crumbResponse.IsSuccessStatusCode)
|
|
{
|
|
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
|
|
if (!string.IsNullOrWhiteSpace(crumbText))
|
|
{
|
|
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
|
|
_lastAuthTime = DateTime.UtcNow;
|
|
|
|
// Persist to DB cache via SettingsService
|
|
if (_settingsService != null)
|
|
{
|
|
try
|
|
{
|
|
var serializedCookies = SerializeCookies();
|
|
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken);
|
|
if (!string.IsNullOrWhiteSpace(serializedCookies))
|
|
{
|
|
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
|
|
}
|
|
}
|
|
catch (Exception persistEx)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database.");
|
|
}
|
|
}
|
|
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb);
|
|
return _crumb;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode);
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
// Fallthrough to next endpoint
|
|
}
|
|
}
|
|
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Failed to fetch crumb token from all endpoints.");
|
|
return null;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
_authLock.Release();
|
|
}
|
|
}
|
|
|
|
private string SerializeCookies()
|
|
{
|
|
try
|
|
{
|
|
var cookies = _cookieContainer.GetAllCookies();
|
|
var pairs = new List<string>();
|
|
foreach (Cookie cookie in cookies)
|
|
{
|
|
pairs.Add($"{cookie.Name}={cookie.Value}");
|
|
}
|
|
return string.Join(";", pairs);
|
|
}
|
|
catch
|
|
{
|
|
return string.Empty;
|
|
}
|
|
}
|
|
|
|
private void RestoreCookies(string serializedCookies)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(serializedCookies)) return;
|
|
|
|
try
|
|
{
|
|
var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
|
foreach (var part in parts)
|
|
{
|
|
var eqIdx = part.IndexOf('=');
|
|
if (eqIdx > 0 && eqIdx < part.Length - 1)
|
|
{
|
|
var name = part.Substring(0, eqIdx).Trim();
|
|
var val = part.Substring(eqIdx + 1).Trim();
|
|
_cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com"));
|
|
}
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
// Ignore cookie restore errors
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
|
|
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&quotesCount={quotesCount}&newsCount={newsCount}
|
|
/// Note: Does not require Cookie/Crumb authentication.
|
|
/// </summary>
|
|
public async Task<YahooSearchResponseDto?> SearchAsync(
|
|
string query,
|
|
int quotesCount = 10,
|
|
int newsCount = 0,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(query)) return null;
|
|
|
|
try
|
|
{
|
|
var url =
|
|
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}";
|
|
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode);
|
|
return null;
|
|
}
|
|
|
|
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
|
|
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&modules={modules}
|
|
/// </summary>
|
|
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
|
|
string symbol,
|
|
IEnumerable<string> modules,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(symbol)) return null;
|
|
|
|
var moduleList = string.Join(",", modules);
|
|
return await ExecuteWithRetryAsync(async (crumb) =>
|
|
{
|
|
var url =
|
|
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
|
|
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
|
|
return (
|
|
response.StatusCode == HttpStatusCode.Unauthorized ||
|
|
response.StatusCode == HttpStatusCode.Forbidden, null);
|
|
}
|
|
|
|
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
|
|
return (false, dto);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Convenience method to fetch all standard quoteSummary modules for a given symbol.
|
|
/// </summary>
|
|
public Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(string symbol,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
return GetQuoteSummaryAsync(symbol, StandardQuoteSummaryModules, cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves historical OHLCV chart data for a given symbol.
|
|
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&interval={interval}&crumb={crumb}
|
|
/// </summary>
|
|
public async Task<YahooChartResponseDto?> GetChartAsync(
|
|
string symbol,
|
|
string range = "1y",
|
|
string interval = "1d",
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(symbol)) return null;
|
|
|
|
return await ExecuteWithRetryAsync(async (crumb) =>
|
|
{
|
|
var url =
|
|
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
|
|
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
|
|
return (
|
|
response.StatusCode == HttpStatusCode.Unauthorized ||
|
|
response.StatusCode == HttpStatusCode.Forbidden, null);
|
|
}
|
|
|
|
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
|
|
return (false, dto);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves quick real-time price quotes for one or more symbols.
|
|
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&crumb={crumb}
|
|
/// </summary>
|
|
public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
|
|
IEnumerable<string> symbols,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var symbolList = symbols.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
|
|
if (symbolList.Count == 0) return null;
|
|
|
|
var symbolsParam = string.Join(",", symbolList);
|
|
return await ExecuteWithRetryAsync(async (crumb) =>
|
|
{
|
|
var url =
|
|
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
|
|
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
|
|
return (
|
|
response.StatusCode == HttpStatusCode.Unauthorized ||
|
|
response.StatusCode == HttpStatusCode.Forbidden, null);
|
|
}
|
|
|
|
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
|
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
|
|
return (false, dto);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Convenient helper method to fetch the current live price for a single symbol (e.g., "^VIX").
|
|
/// </summary>
|
|
public async Task<decimal?> GetLivePriceAsync(string symbol, CancellationToken cancellationToken = default)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(symbol)) return null;
|
|
|
|
var quotes = await GetQuotesAsync(new[] { symbol }, cancellationToken);
|
|
var item = quotes?.QuoteResponse?.Result?.FirstOrDefault();
|
|
|
|
if (item?.RegularMarketPrice.HasValue == true && item.RegularMarketPrice.Value > 0)
|
|
{
|
|
return Convert.ToDecimal(item.RegularMarketPrice.Value);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private async Task<T?> ExecuteWithRetryAsync<T>(
|
|
Func<string, Task<(bool isAuthError, T? result)>> action,
|
|
CancellationToken cancellationToken) where T : class
|
|
{
|
|
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
|
|
if (string.IsNullOrEmpty(crumb)) return null;
|
|
|
|
var (isAuthError, result) = await action(crumb);
|
|
if (!isAuthError && result != null)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
if (isAuthError)
|
|
{
|
|
if (_finlyticLogger != null)
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
|
|
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
|
|
if (string.IsNullOrEmpty(crumb)) return null;
|
|
|
|
var (_, retryResult) = await action(crumb);
|
|
return retryResult;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private static JsonSerializerOptions GetJsonOptions()
|
|
{
|
|
return new JsonSerializerOptions
|
|
{
|
|
PropertyNameCaseInsensitive = true,
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
};
|
|
}
|
|
} |