Finlytic/FinlyticTechnicalAnalysis/Services/YahooMarketDataScraper.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services.Yahoo;
using FinlyticTechnicalAnalysis.Entities;
using Microsoft.Extensions.Logging;

namespace FinlyticTechnicalAnalysis.Services;

public record YahooCandlesResult(
    List<MarketCandleEntity> Candles,
    string Currency
);

public interface IYahooMarketDataScraper
{
    /// <summary>
    /// Resolves ticker from ISIN.
    /// </summary>
    Task<string?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default);

    /// <summary>
    /// Fetches historical candles.
    /// </summary>
    Task<List<MarketCandleEntity>> FetchHistoricalCandlesAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default);

    /// <summary>
    /// Fetches historical candles with currency.
    /// </summary>
    Task<YahooCandlesResult> FetchHistoricalCandlesWithCurrencyAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default);

    /// <summary>
    /// Fetches macro ticker.
    /// </summary>
    Task<MacroDataEntity?> FetchMacroTickerAsync(string symbol, CancellationToken cancellationToken = default);
}

public class YahooMarketDataScraper : IYahooMarketDataScraper
{
    private readonly YahooFinanceClient _yahooClient;
    private readonly ILogger<YahooMarketDataScraper> _logger;

    public YahooMarketDataScraper(YahooFinanceClient yahooClient, ILogger<YahooMarketDataScraper> logger)
    {
        _yahooClient = yahooClient;
        _logger = logger;
    }

    /// <summary>
    /// Resolves ticker from ISIN using Yahoo Search API.
    /// </summary>
    public async Task<string?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(isin)) return null;

        var cleanIsin = isin.Trim().ToUpperInvariant();
        if (cleanIsin.Contains('.'))
        {
            return cleanIsin;
        }

        try
        {
            var searchResult = await _yahooClient.SearchAsync(cleanIsin, quotesCount: 10, newsCount: 0, cancellationToken);
            if (searchResult?.Quotes != null && searchResult.Quotes.Count > 0)
            {
                var symbolList = searchResult.Quotes
                    .Select(q => q.Symbol)
                    .Where(s => !string.IsNullOrEmpty(s))
                    .Select(s => s!)
                    .ToList();

                if (symbolList.Count > 0)
                {
                    if (cleanIsin.StartsWith("US", StringComparison.OrdinalIgnoreCase))
                    {
                        var noDotSymbol = symbolList.FirstOrDefault(s => !s.Contains('.'));
                        if (noDotSymbol != null) return noDotSymbol;
                    }
                    return symbolList[0];
                }
            }
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "[{Channel}] Failed to resolve Yahoo ticker for ISIN {Isin}", "TechnicalAnalysisChannel", cleanIsin);
        }

        return null;
    }

    /// <summary>
    /// Fetches historical candles.
    /// </summary>
    public async Task<List<MarketCandleEntity>> FetchHistoricalCandlesAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default)
    {
        var result = await FetchHistoricalCandlesWithCurrencyAsync(symbol, range, interval, cancellationToken);
        return result.Candles;
    }

    /// <summary>
    /// Fetches historical candles with currency metadata using authenticated Crumb/Cookie flow.
    /// </summary>
    public async Task<YahooCandlesResult> FetchHistoricalCandlesWithCurrencyAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default)
    {
        var results = new List<MarketCandleEntity>();
        string detectedCurrency = FallbackCurrencyBySymbol(symbol);

        if (string.IsNullOrWhiteSpace(symbol)) return new YahooCandlesResult(results, detectedCurrency);

        try
        {
            var chartDto = await _yahooClient.GetChartAsync(symbol, range, interval, cancellationToken);
            var resultObj = chartDto?.Chart?.Result?.FirstOrDefault();

            if (resultObj == null)
            {
                _logger.LogWarning("[{Channel}] No chart data returned from Yahoo Client for symbol {Symbol}", "TechnicalAnalysisChannel", symbol);
                return new YahooCandlesResult(results, detectedCurrency);
            }

            // Extract currency metadata
            if (!string.IsNullOrWhiteSpace(resultObj.Meta?.Currency))
            {
                detectedCurrency = resultObj.Meta.Currency.ToUpperInvariant();
            }

            var timestamps = resultObj.Timestamp;
            var quote = resultObj.Indicators?.Quote?.FirstOrDefault();

            if (timestamps == null || quote == null || timestamps.Count == 0)
            {
                return new YahooCandlesResult(results, detectedCurrency);
            }

            var opens = quote.Open ?? [];
            var highs = quote.High ?? [];
            var lows = quote.Low ?? [];
            var closes = quote.Close ?? [];
            var volumes = quote.Volume ?? [];

            for (int i = 0; i < timestamps.Count; i++)
            {
                var dt = DateTimeOffset.FromUnixTimeSeconds(timestamps[i]).UtcDateTime;

                var open = i < opens.Count && opens[i].HasValue ? (decimal)opens[i]!.Value : 0m;
                var high = i < highs.Count && highs[i].HasValue ? (decimal)highs[i]!.Value : open;
                var low = i < lows.Count && lows[i].HasValue ? (decimal)lows[i]!.Value : open;
                var close = i < closes.Count && closes[i].HasValue ? (decimal)closes[i]!.Value : open;
                var vol = i < volumes.Count && volumes[i].HasValue ? (long)volumes[i]!.Value : 0L;

                // Skip invalid or empty weekend/holiday records
                if (close <= 0m && open <= 0m) continue;

                results.Add(new MarketCandleEntity
                {
                    Symbol = symbol.ToUpperInvariant(),
                    Interval = interval,
                    Timestamp = dt,
                    Open = open,
                    High = Math.Max(high, Math.Max(open, close)),
                    Low = Math.Min(low, Math.Min(open, close)),
                    Close = close,
                    Volume = vol
                });
            }

            _logger.LogInformation("[{Channel}] Successfully fetched {Count} candles for {Symbol} ({Range}, {Interval}, Currency: {Currency})", 
                "TechnicalAnalysisChannel", results.Count, symbol, range, interval, detectedCurrency);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "[{Channel}] Error fetching historical candles for {Symbol}", "TechnicalAnalysisChannel", symbol);
        }

        return new YahooCandlesResult(results, detectedCurrency);
    }

    /// <summary>
    /// Fetches macro ticker data (e.g., ^VIX, ^GSPC, DX-Y.NY).
    /// </summary>
    public async Task<MacroDataEntity?> FetchMacroTickerAsync(string symbol, CancellationToken cancellationToken = default)
    {
        var candles = await FetchHistoricalCandlesAsync(symbol, "5d", "1d", cancellationToken);
        if (candles.Count == 0) return null;

        var lastCandle = candles.Last();
        var prevCandle = candles.Count > 1 ? candles[^2] : lastCandle;

        var trendState = lastCandle.Close >= prevCandle.Close ? "Bullish" : "Bearish";
        if (symbol == "^VIX")
        {
            trendState = lastCandle.Close > 25m ? "HighVolatility" : (lastCandle.Close > 18m ? "Moderate" : "LowVolatility");
        }

        return new MacroDataEntity
        {
            Symbol = symbol,
            Value = lastCandle.Close,
            PreviousClose = prevCandle.Close,
            TrendState = trendState,
            LastUpdatedAt = DateTime.UtcNow
        };
    }

    private static string FallbackCurrencyBySymbol(string symbol)
    {
        if (string.IsNullOrWhiteSpace(symbol)) return "EUR";

        if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase) ||
            symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase) ||
            symbol.EndsWith(".VI", StringComparison.OrdinalIgnoreCase) ||
            symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase))
        {
            return "EUR";
        }

        if (!symbol.Contains('.'))
        {
            return "USD";
        }

        return "EUR";
    }
}