Finlytic/FinlyticCore/Services/Yahoo/YahooFinanceClient.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using Microsoft.Extensions.Logging;

namespace FinlyticCore.Services.Yahoo;

/// <summary>
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
/// </summary>
public class YahooFinanceClient
{
    private const string DefaultUserAgent =
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";

    private readonly HttpClient _httpClient;
    private readonly CookieContainer _cookieContainer;
    private readonly ILogger<YahooFinanceClient>? _logger;
    private readonly SemaphoreSlim _authLock = new(1, 1);

    private string? _crumb;
    private DateTime _lastAuthTime = DateTime.MinValue;

    /// <summary>
    /// Standard modules available for the quoteSummary endpoint.
    /// </summary>
    public static readonly string[] StandardQuoteSummaryModules = new[]
    {
        "assetProfile",
        "financialData",
        "defaultKeyStatistics",
        "summaryDetail",
        "incomeStatementHistory",
        "incomeStatementHistoryQuarterly",
        "balanceSheetHistory",
        "balanceSheetHistoryQuarterly",
        "cashflowStatementHistory",
        "cashflowStatementHistoryQuarterly",
        "calendarEvents"
    };

    public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
    {
        _logger = logger;
        _cookieContainer = new CookieContainer();

        if (httpClient != null)
        {
            _httpClient = httpClient;
        }
        else
        {
            var handler = new HttpClientHandler
            {
                CookieContainer = _cookieContainer,
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
            };
            _httpClient = new HttpClient(handler);
        }

        if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
        {
            _httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
        }
    }

    /// <summary>
    /// Executes the Cookie (A3) &amp; Crumb token authentication flow.
    /// 1. GET https://fc.yahoo.com (sets session A3 cookie)
    /// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
    /// </summary>
    public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
        CancellationToken cancellationToken = default)
    {
        if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
        {
            return _crumb;
        }

        await _authLock.WaitAsync(cancellationToken);
        try
        {
            if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
                (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
            {
                return _crumb;
            }

            _logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");

            // 1. Send GET request to fc.yahoo.com to obtain session cookie A3
            using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
            {
                using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
                // CookieContainer automatically intercepts and stores 'A3' cookie
            }

            // 2. Send GET request to getcrumb to obtain the dynamic crumb token
            using (var crumbRequest =
                   new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
            {
                using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
                if (!crumbResponse.IsSuccessStatusCode)
                {
                    _logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
                        crumbResponse.StatusCode);
                    return null;
                }

                var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
                _crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
                _lastAuthTime = DateTime.UtcNow;

                _logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
                return _crumb;
            }
        }
        catch (Exception ex)
        {
            _logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
            return null;
        }
        finally
        {
            _authLock.Release();
        }
    }

    /// <summary>
    /// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
    /// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
    /// Note: Does not require Cookie/Crumb authentication.
    /// </summary>
    public async Task<YahooSearchResponseDto?> SearchAsync(
        string query,
        int quotesCount = 10,
        int newsCount = 0,
        CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(query)) return null;

        try
        {
            var url =
                $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}";
            using var response = await _httpClient.GetAsync(url, cancellationToken);

            if (!response.IsSuccessStatusCode)
            {
                _logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
                    response.StatusCode);
                return null;
            }

            var json = await response.Content.ReadAsStringAsync(cancellationToken);
            return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
        }
        catch (Exception ex)
        {
            _logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
            return null;
        }
    }

    /// <summary>
    /// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
    /// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&amp;modules={modules}
    /// </summary>
    public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
        string symbol,
        IEnumerable<string> modules,
        CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(symbol)) return null;

        var moduleList = string.Join(",", modules);
        return await ExecuteWithRetryAsync(async (crumb) =>
        {
            var url =
                $"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
            using var response = await _httpClient.GetAsync(url, cancellationToken);

            if (!response.IsSuccessStatusCode)
            {
                _logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
                    symbol, response.StatusCode);
                return (
                    response.StatusCode == HttpStatusCode.Unauthorized ||
                    response.StatusCode == HttpStatusCode.Forbidden, null);
            }

            var json = await response.Content.ReadAsStringAsync(cancellationToken);
            var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
            return (false, dto);
        }, cancellationToken);
    }

    /// <summary>
    /// Convenience method to fetch all standard quoteSummary modules for a given symbol.
    /// </summary>
    public Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(string symbol,
        CancellationToken cancellationToken = default)
    {
        return GetQuoteSummaryAsync(symbol, StandardQuoteSummaryModules, cancellationToken);
    }

    /// <summary>
    /// Retrieves historical OHLCV chart data for a given symbol.
    /// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&amp;interval={interval}&amp;crumb={crumb}
    /// </summary>
    public async Task<YahooChartResponseDto?> GetChartAsync(
        string symbol,
        string range = "1y",
        string interval = "1d",
        CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(symbol)) return null;

        return await ExecuteWithRetryAsync(async (crumb) =>
        {
            var url =
                $"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
            using var response = await _httpClient.GetAsync(url, cancellationToken);

            if (!response.IsSuccessStatusCode)
            {
                _logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
                    response.StatusCode);
                return (
                    response.StatusCode == HttpStatusCode.Unauthorized ||
                    response.StatusCode == HttpStatusCode.Forbidden, null);
            }

            var json = await response.Content.ReadAsStringAsync(cancellationToken);
            var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
            return (false, dto);
        }, cancellationToken);
    }

    /// <summary>
    /// Retrieves quick real-time price quotes for one or more symbols.
    /// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&amp;crumb={crumb}
    /// </summary>
    public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
        IEnumerable<string> symbols,
        CancellationToken cancellationToken = default)
    {
        var symbolList = symbols.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
        if (symbolList.Count == 0) return null;

        var symbolsParam = string.Join(",", symbolList);
        return await ExecuteWithRetryAsync(async (crumb) =>
        {
            var url =
                $"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
            using var response = await _httpClient.GetAsync(url, cancellationToken);

            if (!response.IsSuccessStatusCode)
            {
                _logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
                return (
                    response.StatusCode == HttpStatusCode.Unauthorized ||
                    response.StatusCode == HttpStatusCode.Forbidden, null);
            }

            var json = await response.Content.ReadAsStringAsync(cancellationToken);
            var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
            return (false, dto);
        }, cancellationToken);
    }

    /// <summary>
    /// Convenient helper method to fetch the current live price for a single symbol (e.g., "^VIX").
    /// </summary>
    public async Task<decimal?> GetLivePriceAsync(string symbol, CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(symbol)) return null;

        var quotes = await GetQuotesAsync(new[] { symbol }, cancellationToken);
        var item = quotes?.QuoteResponse?.Result?.FirstOrDefault();

        if (item?.RegularMarketPrice.HasValue == true && item.RegularMarketPrice.Value > 0)
        {
            return Convert.ToDecimal(item.RegularMarketPrice.Value);
        }

        return null;
    }

    private async Task<T?> ExecuteWithRetryAsync<T>(
        Func<string, Task<(bool isAuthError, T? result)>> action,
        CancellationToken cancellationToken) where T : class
    {
        var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
        if (string.IsNullOrEmpty(crumb)) return null;

        var (isAuthError, result) = await action(crumb);
        if (!isAuthError && result != null)
        {
            return result;
        }

        if (isAuthError)
        {
            _logger?.LogInformation(
                "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
            crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
            if (string.IsNullOrEmpty(crumb)) return null;

            var (_, retryResult) = await action(crumb);
            return retryResult;
        }

        return result;
    }

    private static JsonSerializerOptions GetJsonOptions()
    {
        return new JsonSerializerOptions
        {
            PropertyNameCaseInsensitive = true,
            NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
        };
    }
}