Finlytic/FinlyticTechnicals/Services/YahooMarketDataScraper.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.TechnicalAnalysis;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using FinlyticTechnicals.Util;
using Microsoft.Extensions.Configuration;

namespace FinlyticTechnicals.Services;

public record YahooCandlesResult(
    List<CandleDto> Candles,
    string Currency
);

public interface IYahooMarketDataScraper
{
    /// <summary>
    /// Resolves ticker from ISIN using Yahoo Search API.
    /// </summary>
    Task<string?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default);

    /// <summary>
    /// Fetches historical candles with strict UTC timestamps.
    /// </summary>
    Task<List<CandleDto>> FetchHistoricalCandlesAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default);

    /// <summary>
    /// Fetches historical candles with currency metadata.
    /// </summary>
    Task<YahooCandlesResult> FetchHistoricalCandlesWithCurrencyAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default);
}

public class YahooMarketDataScraper : IYahooMarketDataScraper
{
    private readonly YahooFinanceClient _yahooClient;
    private readonly IConfiguration _configuration;
    private readonly IFinlyticLogger<YahooMarketDataScraper> _finlyticLogger;

    public YahooMarketDataScraper(
        YahooFinanceClient yahooClient,
        IConfiguration configuration,
        IFinlyticLogger<YahooMarketDataScraper> finlyticLogger)
    {
        _yahooClient = yahooClient;
        _configuration = configuration;
        _finlyticLogger = finlyticLogger;
    }

    public async Task<string?> ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(isin)) return null;

        var cleanIsin = isin.Trim().ToUpperInvariant();
        if (cleanIsin.Contains('.'))
        {
            return cleanIsin;
        }

        if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase))
        {
            var (cryptoSubtitle, cryptoName) = await FinlyticCore.Util.CryptoSubtitleResolver.ResolveCryptoInfoAsync(
                cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken);

            if (!string.IsNullOrWhiteSpace(cryptoSubtitle))
            {
                var candidates = new[] { $"{cryptoSubtitle}-EUR", $"{cryptoSubtitle}-USD", cryptoSubtitle };
                foreach (var candidate in candidates)
                {
                    try
                    {
                        var res = await FetchHistoricalCandlesWithCurrencyAsync(candidate, "5d", "1d", cancellationToken);
                        if (res.Candles.Count > 0)
                        {
                            await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved Crypto ISIN {Isin} to {Symbol} using Subtitle {Sub}", cleanIsin, candidate, cryptoSubtitle);
                            return candidate;
                        }
                    }
                    catch { }
                }

                return $"{cryptoSubtitle}-EUR";
            }
        }

        try
        {
            var searchResult = await _yahooClient.SearchAsync(cleanIsin, quotesCount: 10, newsCount: 0, cancellationToken);
            if (searchResult?.Quotes != null && searchResult.Quotes.Count > 0)
            {
                var prioritizedSuffixes = new[] { ".DE", ".F", ".SG", ".MU", ".BE", ".DU", ".HM" };

                foreach (var suffix in prioritizedSuffixes)
                {
                    var match = searchResult.Quotes.FirstOrDefault(q =>
                        !string.IsNullOrWhiteSpace(q.Symbol) &&
                        q.Symbol.EndsWith(suffix, StringComparison.OrdinalIgnoreCase));

                    if (match != null)
                    {
                        await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved ISIN {Isin} to German ticker {Symbol}", cleanIsin, match.Symbol);
                        return match.Symbol;
                    }
                }

                var defaultQuote = searchResult.Quotes.FirstOrDefault(q => !string.IsNullOrWhiteSpace(q.Symbol));
                if (defaultQuote != null)
                {
                    await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved ISIN {Isin} to primary ticker {Symbol}", cleanIsin, defaultQuote.Symbol);
                    return defaultQuote.Symbol;
                }
            }
        }
        catch (Exception ex)
        {
            await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[YahooMarketDataScraper] Search failed for ISIN {Isin}", cleanIsin);
        }

        return null;
    }

    public async Task<List<CandleDto>> FetchHistoricalCandlesAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default)
    {
        var result = await FetchHistoricalCandlesWithCurrencyAsync(symbol, range, interval, cancellationToken);
        return result.Candles;
    }

    public async Task<YahooCandlesResult> FetchHistoricalCandlesWithCurrencyAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default)
    {
        var results = new List<CandleDto>();
        string detectedCurrency = FallbackCurrencyBySymbol(symbol);

        if (string.IsNullOrWhiteSpace(symbol)) return new YahooCandlesResult(results, detectedCurrency);

        try
        {
            var chartDto = await _yahooClient.GetChartAsync(symbol, range, interval, cancellationToken);
            var resultObj = chartDto?.Chart?.Result?.FirstOrDefault();

            if (resultObj == null)
            {
                await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] No chart data returned from Yahoo Client for symbol {Symbol}", symbol);
                return new YahooCandlesResult(results, detectedCurrency);
            }

            if (!string.IsNullOrWhiteSpace(resultObj.Meta?.Currency))
            {
                detectedCurrency = resultObj.Meta.Currency.ToUpperInvariant();
            }

            var timestamps = resultObj.Timestamp;
            var quote = resultObj.Indicators?.Quote?.FirstOrDefault();

            if (timestamps == null || quote == null || timestamps.Count == 0)
            {
                return new YahooCandlesResult(results, detectedCurrency);
            }

            var opens = quote.Open ?? [];
            var highs = quote.High ?? [];
            var lows = quote.Low ?? [];
            var closes = quote.Close ?? [];
            var volumes = quote.Volume ?? [];

            for (int i = 0; i < timestamps.Count; i++)
            {
                // Strict UTC timestamp
                var dt = DateTimeOffset.FromUnixTimeSeconds(timestamps[i]).UtcDateTime;

                var open = i < opens.Count && opens[i].HasValue ? (decimal)opens[i]!.Value : 0m;
                var high = i < highs.Count && highs[i].HasValue ? (decimal)highs[i]!.Value : open;
                var low = i < lows.Count && lows[i].HasValue ? (decimal)lows[i]!.Value : open;
                var close = i < closes.Count && closes[i].HasValue ? (decimal)closes[i]!.Value : open;
                var vol = i < volumes.Count && volumes[i].HasValue ? (long)volumes[i]!.Value : 0L;

                if (close <= 0m && open <= 0m) continue;

                results.Add(new CandleDto(
                    Timestamp: dt,
                    Open: open,
                    High: Math.Max(high, Math.Max(open, close)),
                    Low: Math.Min(low, Math.Min(open, close)),
                    Close: close,
                    Volume: vol
                ));
            }

            await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Successfully fetched {Count} candles for {Symbol} ({Range}, {Interval}, Currency: {Currency})", 
                results.Count, symbol, range, interval, detectedCurrency);
        }
        catch (Exception ex)
        {
            await _finlyticLogger.LogErrorAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[YahooMarketDataScraper] Error fetching historical candles for {Symbol}", symbol);
        }

        return new YahooCandlesResult(results, detectedCurrency);
    }

    private static string FallbackCurrencyBySymbol(string symbol)
    {
        if (string.IsNullOrWhiteSpace(symbol)) return "EUR";
        var s = symbol.Trim().ToUpperInvariant();
        if (s.EndsWith(".DE") || s.EndsWith(".F") || s.EndsWith(".PA") || s.EndsWith(".AS") || s.EndsWith(".MI"))
            return "EUR";
        if (s.EndsWith(".L"))
            return "GBp";
        return "USD";
    }
}