using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.TechnicalAnalysis; using FinlyticCore.Services; using FinlyticCore.Services.Yahoo; using FinlyticTechnicals.Util; using Microsoft.Extensions.Configuration; namespace FinlyticTechnicals.Services; public record YahooCandlesResult( List Candles, string Currency ); public interface IYahooMarketDataScraper { /// /// Resolves ticker from ISIN using Yahoo Search API. /// Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// /// Fetches historical candles with strict UTC timestamps. /// Task> FetchHistoricalCandlesAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default); /// /// Fetches historical candles with currency metadata. /// Task FetchHistoricalCandlesWithCurrencyAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default); } public class YahooMarketDataScraper : IYahooMarketDataScraper { private readonly YahooFinanceClient _yahooClient; private readonly IConfiguration _configuration; private readonly IFinlyticLogger _finlyticLogger; public YahooMarketDataScraper( YahooFinanceClient yahooClient, IConfiguration configuration, IFinlyticLogger finlyticLogger) { _yahooClient = yahooClient; _configuration = configuration; _finlyticLogger = finlyticLogger; } public async Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(isin)) return null; var cleanIsin = isin.Trim().ToUpperInvariant(); if (cleanIsin.Contains('.')) { return cleanIsin; } if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase)) { var (cryptoSubtitle, cryptoName) = await FinlyticCore.Util.CryptoSubtitleResolver.ResolveCryptoInfoAsync( cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken); if (!string.IsNullOrWhiteSpace(cryptoSubtitle)) { var candidates = new[] { $"{cryptoSubtitle}-EUR", $"{cryptoSubtitle}-USD", cryptoSubtitle }; foreach (var candidate in candidates) { try { var res = await FetchHistoricalCandlesWithCurrencyAsync(candidate, "5d", "1d", cancellationToken); if (res.Candles.Count > 0) { await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved Crypto ISIN {Isin} to {Symbol} using Subtitle {Sub}", cleanIsin, candidate, cryptoSubtitle); return candidate; } } catch { } } return $"{cryptoSubtitle}-EUR"; } } try { var searchResult = await _yahooClient.SearchAsync(cleanIsin, quotesCount: 10, newsCount: 0, cancellationToken); if (searchResult?.Quotes != null && searchResult.Quotes.Count > 0) { var prioritizedSuffixes = new[] { ".DE", ".F", ".SG", ".MU", ".BE", ".DU", ".HM" }; foreach (var suffix in prioritizedSuffixes) { var match = searchResult.Quotes.FirstOrDefault(q => !string.IsNullOrWhiteSpace(q.Symbol) && q.Symbol.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); if (match != null) { await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved ISIN {Isin} to German ticker {Symbol}", cleanIsin, match.Symbol); return match.Symbol; } } var defaultQuote = searchResult.Quotes.FirstOrDefault(q => !string.IsNullOrWhiteSpace(q.Symbol)); if (defaultQuote != null) { await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved ISIN {Isin} to primary ticker {Symbol}", cleanIsin, defaultQuote.Symbol); return defaultQuote.Symbol; } } } catch (Exception ex) { await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[YahooMarketDataScraper] Search failed for ISIN {Isin}", cleanIsin); } return null; } public async Task> FetchHistoricalCandlesAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default) { var result = await FetchHistoricalCandlesWithCurrencyAsync(symbol, range, interval, cancellationToken); return result.Candles; } public async Task FetchHistoricalCandlesWithCurrencyAsync(string symbol, string range = "1y", string interval = "1d", CancellationToken cancellationToken = default) { var results = new List(); string detectedCurrency = FallbackCurrencyBySymbol(symbol); if (string.IsNullOrWhiteSpace(symbol)) return new YahooCandlesResult(results, detectedCurrency); try { var chartDto = await _yahooClient.GetChartAsync(symbol, range, interval, cancellationToken); var resultObj = chartDto?.Chart?.Result?.FirstOrDefault(); if (resultObj == null) { await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] No chart data returned from Yahoo Client for symbol {Symbol}", symbol); return new YahooCandlesResult(results, detectedCurrency); } if (!string.IsNullOrWhiteSpace(resultObj.Meta?.Currency)) { detectedCurrency = resultObj.Meta.Currency.ToUpperInvariant(); } var timestamps = resultObj.Timestamp; var quote = resultObj.Indicators?.Quote?.FirstOrDefault(); if (timestamps == null || quote == null || timestamps.Count == 0) { return new YahooCandlesResult(results, detectedCurrency); } var opens = quote.Open ?? []; var highs = quote.High ?? []; var lows = quote.Low ?? []; var closes = quote.Close ?? []; var volumes = quote.Volume ?? []; for (int i = 0; i < timestamps.Count; i++) { // Strict UTC timestamp var dt = DateTimeOffset.FromUnixTimeSeconds(timestamps[i]).UtcDateTime; var open = i < opens.Count && opens[i].HasValue ? (decimal)opens[i]!.Value : 0m; var high = i < highs.Count && highs[i].HasValue ? (decimal)highs[i]!.Value : open; var low = i < lows.Count && lows[i].HasValue ? (decimal)lows[i]!.Value : open; var close = i < closes.Count && closes[i].HasValue ? (decimal)closes[i]!.Value : open; var vol = i < volumes.Count && volumes[i].HasValue ? (long)volumes[i]!.Value : 0L; if (close <= 0m && open <= 0m) continue; results.Add(new CandleDto( Timestamp: dt, Open: open, High: Math.Max(high, Math.Max(open, close)), Low: Math.Min(low, Math.Min(open, close)), Close: close, Volume: vol )); } await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Successfully fetched {Count} candles for {Symbol} ({Range}, {Interval}, Currency: {Currency})", results.Count, symbol, range, interval, detectedCurrency); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[YahooMarketDataScraper] Error fetching historical candles for {Symbol}", symbol); } return new YahooCandlesResult(results, detectedCurrency); } private static string FallbackCurrencyBySymbol(string symbol) { if (string.IsNullOrWhiteSpace(symbol)) return "EUR"; var s = symbol.Trim().ToUpperInvariant(); if (s.EndsWith(".DE") || s.EndsWith(".F") || s.EndsWith(".PA") || s.EndsWith(".AS") || s.EndsWith(".MI")) return "EUR"; if (s.EndsWith(".L")) return "GBp"; return "USD"; } }