using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.Yahoo; using FinlyticCore.Models.Settings; using FinlyticCore.Services; namespace FinlyticCore.Services.Yahoo; public class YahooFinanceClient { private readonly HttpClient _httpClient; private readonly CookieContainer _cookieContainer; private readonly IFinlyticLogger? _finlyticLogger; private readonly ISettingsService? _settingsService; private readonly SemaphoreSlim _authLock = new(1, 1); private string? _crumb; private DateTime _lastAuthTime = DateTime.MinValue; /// /// Standard modules available for the quoteSummary endpoint. /// public static readonly string[] StandardQuoteSummaryModules = new[] { "assetProfile", "financialData", "defaultKeyStatistics", "summaryDetail", "incomeStatementHistory", "incomeStatementHistoryQuarterly", "balanceSheetHistory", "balanceSheetHistoryQuarterly", "cashflowStatementHistory", "cashflowStatementHistoryQuarterly", "calendarEvents" }; public YahooFinanceClient( IFinlyticLogger? finlyticLogger = null, ISettingsService? settingsService = null, HttpClient? httpClient = null) { _finlyticLogger = finlyticLogger; _settingsService = settingsService; _cookieContainer = new CookieContainer(); if (httpClient != null) { _httpClient = httpClient; } else { var handler = new HttpClientHandler { CookieContainer = _cookieContainer, UseCookies = true, AllowAutoRedirect = true }; _httpClient = new HttpClient(handler); _httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); } } /// /// Ensures that an active Yahoo session (Cookie + dynamic Crumb token) is initialized. /// Uses persistent DB caching and only refreshes when the crumb is invalid or forceRefresh is true. /// public async Task EnsureAuthenticatedAsync(bool forceRefresh = false, CancellationToken cancellationToken = default) { await _authLock.WaitAsync(cancellationToken); try { // 1. Check in-memory crumb if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb)) { return _crumb; } // 2. Check persistent DB cache via SettingsService if (!forceRefresh && _settingsService != null) { try { var cachedCrumb = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCrumb, cancellationToken); var cachedCookies = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCookie, cancellationToken); if (!string.IsNullOrWhiteSpace(cachedCrumb) && !string.IsNullOrWhiteSpace(cachedCookies)) { RestoreCookies(cachedCookies); _crumb = cachedCrumb; _lastAuthTime = DateTime.UtcNow; if (_finlyticLogger != null) await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Restored cached Yahoo session & crumb from database ({Crumb}).", _crumb); return _crumb; } } catch (Exception ex) { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Error restoring cached Yahoo session from DB."); } } if (_finlyticLogger != null) await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)..."); // 3. Send GET request to fc.yahoo.com to obtain session cookie A3 using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com")) { initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"); using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken); } // 4. Send GET request to getcrumb to obtain dynamic crumb token string[] crumbUrls = new[] { "https://query1.finance.yahoo.com/v1/test/getcrumb", "https://query2.finance.yahoo.com/v1/test/getcrumb" }; foreach (var url in crumbUrls) { try { using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url); crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"); crumbRequest.Headers.Add("Accept", "*/*"); using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken); if (crumbResponse.IsSuccessStatusCode) { var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken); if (!string.IsNullOrWhiteSpace(crumbText)) { _crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n'); _lastAuthTime = DateTime.UtcNow; // Persist to DB cache via SettingsService if (_settingsService != null) { try { var serializedCookies = SerializeCookies(); await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken); if (!string.IsNullOrWhiteSpace(serializedCookies)) { await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken); } } catch (Exception persistEx) { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database."); } } if (_finlyticLogger != null) await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb); return _crumb; } } else { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode); } } catch { // Fallthrough to next endpoint } } if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Failed to fetch crumb token from all endpoints."); return null; } catch (Exception ex) { if (_finlyticLogger != null) await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication."); return null; } finally { _authLock.Release(); } } private string SerializeCookies() { try { var cookies = _cookieContainer.GetAllCookies(); var pairs = new List(); foreach (Cookie cookie in cookies) { pairs.Add($"{cookie.Name}={cookie.Value}"); } return string.Join(";", pairs); } catch { return string.Empty; } } private void RestoreCookies(string serializedCookies) { if (string.IsNullOrWhiteSpace(serializedCookies)) return; try { var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); foreach (var part in parts) { var eqIdx = part.IndexOf('='); if (eqIdx > 0 && eqIdx < part.Length - 1) { var name = part.Substring(0, eqIdx).Trim(); var val = part.Substring(eqIdx + 1).Trim(); _cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com")); } } } catch { // Ignore cookie restore errors } } /// /// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API. /// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&quotesCount={quotesCount}&newsCount={newsCount} /// Note: Does not require Cookie/Crumb authentication. /// public async Task SearchAsync( string query, int quotesCount = 10, int newsCount = 0, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(query)) return null; try { var url = $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}"; using var response = await _httpClient.GetAsync(url, cancellationToken); if (!response.IsSuccessStatusCode) { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode); return null; } var json = await response.Content.ReadAsStringAsync(cancellationToken); return JsonSerializer.Deserialize(json, GetJsonOptions()); } catch (Exception ex) { if (_finlyticLogger != null) await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query); return null; } } /// /// Retrieves fundamentals and company metadata using the quoteSummary endpoint. /// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&modules={modules} /// public async Task GetQuoteSummaryAsync( string symbol, IEnumerable modules, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(symbol)) return null; var moduleList = string.Join(",", modules); return await ExecuteWithRetryAsync(async (crumb) => { var url = $"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}"; using var response = await _httpClient.GetAsync(url, cancellationToken); if (!response.IsSuccessStatusCode) { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode); return ( response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden, null); } var json = await response.Content.ReadAsStringAsync(cancellationToken); var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); return (false, dto); }, cancellationToken); } /// /// Convenience method to fetch all standard quoteSummary modules for a given symbol. /// public Task GetFullQuoteSummaryAsync(string symbol, CancellationToken cancellationToken = default) { return GetQuoteSummaryAsync(symbol, StandardQuoteSummaryModules, cancellationToken); } /// /// Retrieves historical OHLCV chart data for a given symbol. /// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&interval={interval}&crumb={crumb} /// public async Task GetChartAsync( string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(symbol)) return null; return await ExecuteWithRetryAsync(async (crumb) => { var url = $"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}"; using var response = await _httpClient.GetAsync(url, cancellationToken); if (!response.IsSuccessStatusCode) { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode); return ( response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden, null); } var json = await response.Content.ReadAsStringAsync(cancellationToken); var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); return (false, dto); }, cancellationToken); } /// /// Retrieves quick real-time price quotes for one or more symbols. /// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&crumb={crumb} /// public async Task GetQuotesAsync( IEnumerable symbols, CancellationToken cancellationToken = default) { var symbolList = symbols.Where(s => !string.IsNullOrWhiteSpace(s)).ToList(); if (symbolList.Count == 0) return null; var symbolsParam = string.Join(",", symbolList); return await ExecuteWithRetryAsync(async (crumb) => { var url = $"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}"; using var response = await _httpClient.GetAsync(url, cancellationToken); if (!response.IsSuccessStatusCode) { if (_finlyticLogger != null) await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode); return ( response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden, null); } var json = await response.Content.ReadAsStringAsync(cancellationToken); var dto = JsonSerializer.Deserialize(json, GetJsonOptions()); return (false, dto); }, cancellationToken); } /// /// Convenient helper method to fetch the current live price for a single symbol (e.g., "^VIX"). /// public async Task GetLivePriceAsync(string symbol, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(symbol)) return null; var quotes = await GetQuotesAsync(new[] { symbol }, cancellationToken); var item = quotes?.QuoteResponse?.Result?.FirstOrDefault(); if (item?.RegularMarketPrice.HasValue == true && item.RegularMarketPrice.Value > 0) { return Convert.ToDecimal(item.RegularMarketPrice.Value); } return null; } private async Task ExecuteWithRetryAsync( Func> action, CancellationToken cancellationToken) where T : class { var crumb = await EnsureAuthenticatedAsync(false, cancellationToken); if (string.IsNullOrEmpty(crumb)) return null; var (isAuthError, result) = await action(crumb); if (!isAuthError && result != null) { return result; } if (isAuthError) { if (_finlyticLogger != null) await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating..."); crumb = await EnsureAuthenticatedAsync(true, cancellationToken); if (string.IsNullOrEmpty(crumb)) return null; var (_, retryResult) = await action(crumb); return retryResult; } return result; } private static JsonSerializerOptions GetJsonOptions() { return new JsonSerializerOptions { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; } }