feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings

2026-08-16 14:05:57 +02:00
parent b0f8d4b78b
commit 2ba54e8057
17 changed files with 1791 additions and 563 deletions
@@ -9,6 +9,8 @@ using System.Threading.Tasks;
 using FinlyticCore.Dtos.Yahoo;
 using FinlyticCore.Models.Settings;
 using FinlyticCore.Services;
+using FinlyticCore.Services.PlaywrightScrapper;
+using Microsoft.Playwright;

 namespace FinlyticCore.Services.Yahoo;

@@ -18,9 +20,11 @@ public class YahooFinanceClient
    private readonly CookieContainer _cookieContainer;
    private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
    private readonly ISettingsService? _settingsService;
+    private readonly IPlaywrightExecutionService? _playwrightService;
    private readonly SemaphoreSlim _authLock = new(1, 1);

    private string? _crumb;
+    private string? _rawCookieHeader;
    private DateTime _lastAuthTime = DateTime.MinValue;

    /// <summary>
@@ -44,10 +48,12 @@ public class YahooFinanceClient
    public YahooFinanceClient(
        IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
        ISettingsService? settingsService = null,
+        IPlaywrightExecutionService? playwrightService = null,
        HttpClient? httpClient = null)
    {
        _finlyticLogger = finlyticLogger;
        _settingsService = settingsService;
+        _playwrightService = playwrightService;
        _cookieContainer = new CookieContainer();

        if (httpClient != null)
@@ -97,6 +103,7 @@ public class YahooFinanceClient
                    {
                        RestoreCookies(cachedCookies);
                        _crumb = cachedCrumb;
+                        _rawCookieHeader = cachedCookies;
                        _lastAuthTime = DateTime.UtcNow;

                        if (_finlyticLogger != null)
@@ -115,70 +122,40 @@ public class YahooFinanceClient
                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");

            // 3. Send GET request to fc.yahoo.com to obtain session cookie A3
-            using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
+            try
            {
+                using var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com");
                initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
                initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
                using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
            }
+            catch { }

            // 4. Send GET request to getcrumb to obtain dynamic crumb token
-            string[] crumbUrls = new[]
+            var crumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
+            if (!string.IsNullOrWhiteSpace(crumb))
            {
-                "https://query1.finance.yahoo.com/v1/test/getcrumb",
-                "https://query2.finance.yahoo.com/v1/test/getcrumb"
-            };
+                _crumb = crumb;
+                _lastAuthTime = DateTime.UtcNow;
+                await PersistSessionAsync(_crumb, cancellationToken);
+                return _crumb;
+            }

-            foreach (var url in crumbUrls)
+            // 5. FALLBACK: Playwright Browser Authentication (Bypasses EU Consent Wall and 429)
+            if (_playwrightService != null)
            {
-                try
+                var (browserCrumb, browserCookies) = await AuthenticateViaPlaywrightAsync(cancellationToken);
+                if (!string.IsNullOrWhiteSpace(browserCrumb))
                {
-                    using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
-                    crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
-                    crumbRequest.Headers.Add("Accept", "*/*");
-
-                    using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
-                    if (crumbResponse.IsSuccessStatusCode)
+                    _crumb = browserCrumb;
+                    _lastAuthTime = DateTime.UtcNow;
+                    if (!string.IsNullOrWhiteSpace(browserCookies))
                    {
-                        var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
-                        if (!string.IsNullOrWhiteSpace(crumbText))
-                        {
-                            _crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
-                            _lastAuthTime = DateTime.UtcNow;
-
-                            // Persist to DB cache via SettingsService
-                            if (_settingsService != null)
-                            {
-                                try
-                                {
-                                    var serializedCookies = SerializeCookies();
-                                    await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken);
-                                    if (!string.IsNullOrWhiteSpace(serializedCookies))
-                                    {
-                                        await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
-                                    }
-                                }
-                                catch (Exception persistEx)
-                                {
-                                    if (_finlyticLogger != null)
-                                        await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database.");
-                                }
-                            }
-
-                            if (_finlyticLogger != null)
-                                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb);
-                            return _crumb;
-                        }
+                        RestoreCookies(browserCookies);
+                        _rawCookieHeader = browserCookies;
                    }
-                    else
-                    {
-                        if (_finlyticLogger != null)
-                            await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode);
-                    }
-                }
-                catch
-                {
-                    // Fallthrough to next endpoint
+                    await PersistSessionAsync(_crumb, cancellationToken);
+                    return _crumb;
                }
            }

@@ -198,13 +175,204 @@ public class YahooFinanceClient
        }
    }

+    private async Task<string?> FetchCrumbWithHttpClientAsync(CancellationToken cancellationToken)
+    {
+        string[] crumbUrls = new[]
+        {
+            "https://query1.finance.yahoo.com/v1/test/getcrumb",
+            "https://query2.finance.yahoo.com/v1/test/getcrumb"
+        };
+
+        foreach (var url in crumbUrls)
+        {
+            try
+            {
+                using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
+                crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
+                crumbRequest.Headers.Add("Accept", "*/*");
+                crumbRequest.Headers.Add("Origin", "https://finance.yahoo.com");
+                crumbRequest.Headers.Add("Referer", "https://finance.yahoo.com/");
+
+                if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
+                {
+                    crumbRequest.Headers.Add("Cookie", _rawCookieHeader);
+                }
+
+                using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
+                var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
+
+                if (crumbResponse.IsSuccessStatusCode && !string.IsNullOrWhiteSpace(crumbText))
+                {
+                    var cleanCrumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
+                    if (!cleanCrumb.Contains("<html", StringComparison.OrdinalIgnoreCase) && cleanCrumb.Length < 100)
+                    {
+                        return cleanCrumb;
+                    }
+                }
+                else
+                {
+                    if (_finlyticLogger != null)
+                        await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, 
+                            "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}. Response Body: {Body}", 
+                            url, crumbResponse.StatusCode, crumbText);
+                }
+            }
+            catch (Exception ex)
+            {
+                if (_finlyticLogger != null)
+                    await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during FetchCrumb on '{Url}'", url);
+            }
+        }
+
+        return null;
+    }
+
+    private async Task<(string? crumb, string? cookieStr)> AuthenticateViaPlaywrightAsync(CancellationToken cancellationToken)
+    {
+        if (_playwrightService == null) return (null, null);
+
+        try
+        {
+            if (_finlyticLogger != null)
+                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Launching Playwright browser to acquire valid EU Yahoo session and Crumb...");
+
+            return await _playwrightService.ExecuteInContextAsync(async context =>
+            {
+                var page = await context.NewPageAsync();
+                try
+                {
+                    await page.GotoAsync("https://finance.yahoo.com/quote/AAPL/", new PageGotoOptions
+                    {
+                        WaitUntil = WaitUntilState.DOMContentLoaded,
+                        Timeout = 30_000
+                    });
+
+                    // Handle EU Consent if redirected
+                    var url = page.Url;
+                    if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) || 
+                        url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
+                    {
+                        var selectors = new[]
+                        {
+                            "button[name='agree']",
+                            "button[value='agree']",
+                            "button.accept-all",
+                            "button.btn.primary",
+                            "button.btn.secondary.accept-all",
+                            "form[action*='consent'] button[type='submit']",
+                            "button:has-text('Alle akzeptieren')",
+                            "button:has-text('Accept all')",
+                            "button:has-text('Akzeptieren')",
+                            "button:has-text('Agree')"
+                        };
+
+                        foreach (var sel in selectors)
+                        {
+                            var btn = page.Locator(sel);
+                            if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
+                            {
+                                await btn.First.ClickAsync();
+                                await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15_000 });
+                                break;
+                            }
+                        }
+                    }
+
+                    // 1. Extract cookies from BrowserContext
+                    var cookies = await context.CookiesAsync();
+                    var pairs = new List<string>();
+                    foreach (var c in cookies)
+                    {
+                        if (c.Domain.Contains("yahoo.com", StringComparison.OrdinalIgnoreCase))
+                        {
+                            pairs.Add($"{c.Name}={c.Value}");
+                        }
+                    }
+                    var extractedCookies = string.Join(";", pairs);
+
+                    // 2. Fetch Crumb from inside the authenticated page
+                    string? extractedCrumb = null;
+                    try
+                    {
+                        extractedCrumb = await page.EvaluateAsync<string>(@"async () => {
+                            try {
+                                const res = await fetch('/v1/test/getcrumb');
+                                if (res.ok) {
+                                    return await res.text();
+                                }
+                            } catch {}
+                            return null;
+                        }");
+                    }
+                    catch { }
+
+                    // 3. If in-page fetch was empty, use the extracted cookies with HttpClient
+                    if (string.IsNullOrWhiteSpace(extractedCrumb) && !string.IsNullOrWhiteSpace(extractedCookies))
+                    {
+                        RestoreCookies(extractedCookies);
+                        _rawCookieHeader = extractedCookies;
+                        extractedCrumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
+                    }
+
+                    if (!string.IsNullOrWhiteSpace(extractedCrumb))
+                    {
+                        extractedCrumb = extractedCrumb.Trim('"', ' ', '\t', '\r', '\n');
+                        if (_finlyticLogger != null)
+                            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Playwright successfully acquired Yahoo Crumb token: {Crumb}", extractedCrumb);
+                    }
+
+                    return (extractedCrumb, extractedCookies);
+                }
+                finally
+                {
+                    await page.CloseAsync();
+                }
+            }, cancellationToken: cancellationToken);
+        }
+        catch (Exception ex)
+        {
+            if (_finlyticLogger != null)
+                await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Playwright-based authentication failed.");
+            return (null, null);
+        }
+    }
+
+    private async Task PersistSessionAsync(string crumb, CancellationToken cancellationToken)
+    {
+        if (_settingsService == null || string.IsNullOrWhiteSpace(crumb)) return;
+
+        try
+        {
+            var serializedCookies = SerializeCookies();
+            await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, crumb, cancellationToken);
+            if (!string.IsNullOrWhiteSpace(serializedCookies))
+            {
+                _rawCookieHeader = serializedCookies;
+                await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
+            }
+
+            if (_finlyticLogger != null)
+                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Persisted valid Yahoo session & crumb to database.");
+        }
+        catch (Exception ex)
+        {
+            if (_finlyticLogger != null)
+                await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Failed to persist Yahoo session to database.");
+        }
+    }
+
    private string SerializeCookies()
    {
+        if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
+        {
+            return _rawCookieHeader;
+        }
+
        try
        {
            var cookies = _cookieContainer.GetAllCookies();
            var pairs = new List<string>();
-            foreach (Cookie cookie in cookies)
+            foreach (System.Net.Cookie cookie in cookies)
            {
                pairs.Add($"{cookie.Name}={cookie.Value}");
            }
@@ -220,9 +388,20 @@ public class YahooFinanceClient
    {
        if (string.IsNullOrWhiteSpace(serializedCookies)) return;

+        _rawCookieHeader = serializedCookies;
+
        try
        {
            var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+            var uris = new[]
+            {
+                new Uri("https://yahoo.com"),
+                new Uri("https://finance.yahoo.com"),
+                new Uri("https://query1.finance.yahoo.com"),
+                new Uri("https://query2.finance.yahoo.com"),
+                new Uri("https://fc.yahoo.com")
+            };
+
            foreach (var part in parts)
            {
                var eqIdx = part.IndexOf('=');
@@ -230,7 +409,14 @@ public class YahooFinanceClient
                {
                    var name = part.Substring(0, eqIdx).Trim();
                    var val = part.Substring(eqIdx + 1).Trim();
-                    _cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com"));
+                    foreach (var uri in uris)
+                    {
+                        try
+                        {
+                            _cookieContainer.Add(uri, new System.Net.Cookie(name, val));
+                        }
+                        catch { }
+                    }
                }
            }
        }
@@ -242,8 +428,6 @@ public class YahooFinanceClient

    /// <summary>
    /// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
-    /// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
-    /// Note: Does not require Cookie/Crumb authentication.
    /// </summary>
    public async Task<YahooSearchResponseDto?> SearchAsync(
        string query,
@@ -253,33 +437,55 @@ public class YahooFinanceClient
    {
        if (string.IsNullOrWhiteSpace(query)) return null;

-        try
+        string[] endpoints = new[]
        {
-            var url =
-                $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}";
-            using var response = await _httpClient.GetAsync(url, cancellationToken);
+            $"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}",
+            $"https://query1.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}"
+        };

-            if (!response.IsSuccessStatusCode)
+        foreach (var url in endpoints)
+        {
+            try
+            {
+                using var request = new HttpRequestMessage(HttpMethod.Get, url);
+                request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
+                request.Headers.Add("Accept", "*/*");
+                if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
+                {
+                    request.Headers.Add("Cookie", _rawCookieHeader);
+                }
+
+                using var response = await _httpClient.SendAsync(request, cancellationToken);
+                var json = await response.Content.ReadAsStringAsync(cancellationToken);
+
+                if (response.IsSuccessStatusCode)
+                {
+                    var dto = JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
+                    if (dto != null)
+                    {
+                        return dto;
+                    }
+                }
+                else
+                {
+                    if (_finlyticLogger != null)
+                        await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, 
+                            "[YahooFinanceClient] Search endpoint '{Url}' for '{Query}' returned status {Status}. Response Body: {Body}", 
+                            url, query, response.StatusCode, json);
+                }
+            }
+            catch (Exception ex)
            {
                if (_finlyticLogger != null)
-                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode);
-                return null;
+                    await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search endpoint '{Url}'", url);
            }
+        }

-            var json = await response.Content.ReadAsStringAsync(cancellationToken);
-            return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
-        }
-        catch (Exception ex)
-        {
-            if (_finlyticLogger != null)
-                await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
-            return null;
-        }
+        return null;
    }

    /// <summary>
    /// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
-    /// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&amp;modules={modules}
    /// </summary>
    public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
        string symbol,
@@ -291,22 +497,49 @@ public class YahooFinanceClient
        var moduleList = string.Join(",", modules);
        return await ExecuteWithRetryAsync(async (crumb) =>
        {
-            var url =
-                $"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
-            using var response = await _httpClient.GetAsync(url, cancellationToken);
-
-            if (!response.IsSuccessStatusCode)
+            string[] baseUrls = new[]
            {
+                "https://query2.finance.yahoo.com/v10/finance/quoteSummary",
+                "https://query1.finance.yahoo.com/v10/finance/quoteSummary"
+            };
+
+            foreach (var baseUrl in baseUrls)
+            {
+                var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
+                using var request = new HttpRequestMessage(HttpMethod.Get, url);
+                request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
+                request.Headers.Add("Accept", "*/*");
+                request.Headers.Add("Origin", "https://finance.yahoo.com");
+                request.Headers.Add("Referer", "https://finance.yahoo.com/");
+
+                if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
+                {
+                    request.Headers.Add("Cookie", _rawCookieHeader);
+                }
+
+                using var response = await _httpClient.SendAsync(request, cancellationToken);
+                var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
+
+                if (response.IsSuccessStatusCode)
+                {
+                    var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(responseContent, GetJsonOptions());
+                    return (false, dto);
+                }
+
                if (_finlyticLogger != null)
-                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
-                return (
-                    response.StatusCode == HttpStatusCode.Unauthorized ||
-                    response.StatusCode == HttpStatusCode.Forbidden, null);
+                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, 
+                        "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}", 
+                        symbol, response.StatusCode, url, responseContent);
+
+                if (response.StatusCode == HttpStatusCode.Unauthorized ||
+                    response.StatusCode == HttpStatusCode.Forbidden ||
+                    response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    return (true, null);
+                }
            }

-            var json = await response.Content.ReadAsStringAsync(cancellationToken);
-            var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
-            return (false, dto);
+            return (false, null);
        }, cancellationToken);
    }

@@ -321,7 +554,6 @@ public class YahooFinanceClient

    /// <summary>
    /// Retrieves historical OHLCV chart data for a given symbol.
-    /// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&amp;interval={interval}&amp;crumb={crumb}
    /// </summary>
    public async Task<YahooChartResponseDto?> GetChartAsync(
        string symbol,
@@ -333,28 +565,54 @@ public class YahooFinanceClient

        return await ExecuteWithRetryAsync(async (crumb) =>
        {
-            var url =
-                $"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
-            using var response = await _httpClient.GetAsync(url, cancellationToken);
-
-            if (!response.IsSuccessStatusCode)
+            string[] baseUrls = new[]
            {
+                "https://query1.finance.yahoo.com/v8/finance/chart",
+                "https://query2.finance.yahoo.com/v8/finance/chart"
+            };
+
+            foreach (var baseUrl in baseUrls)
+            {
+                var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
+                using var request = new HttpRequestMessage(HttpMethod.Get, url);
+                request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
+                request.Headers.Add("Accept", "*/*");
+                request.Headers.Add("Origin", "https://finance.yahoo.com");
+                request.Headers.Add("Referer", "https://finance.yahoo.com/");
+
+                if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
+                {
+                    request.Headers.Add("Cookie", _rawCookieHeader);
+                }
+
+                using var response = await _httpClient.SendAsync(request, cancellationToken);
+                var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
+
+                if (response.IsSuccessStatusCode)
+                {
+                    var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(responseContent, GetJsonOptions());
+                    return (false, dto);
+                }
+
                if (_finlyticLogger != null)
-                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
-                return (
-                    response.StatusCode == HttpStatusCode.Unauthorized ||
-                    response.StatusCode == HttpStatusCode.Forbidden, null);
+                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, 
+                        "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}", 
+                        symbol, response.StatusCode, url, responseContent);
+
+                if (response.StatusCode == HttpStatusCode.Unauthorized ||
+                    response.StatusCode == HttpStatusCode.Forbidden ||
+                    response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    return (true, null);
+                }
            }

-            var json = await response.Content.ReadAsStringAsync(cancellationToken);
-            var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
-            return (false, dto);
+            return (false, null);
        }, cancellationToken);
    }

    /// <summary>
    /// Retrieves quick real-time price quotes for one or more symbols.
-    /// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&amp;crumb={crumb}
    /// </summary>
    public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
        IEnumerable<string> symbols,
@@ -366,22 +624,49 @@ public class YahooFinanceClient
        var symbolsParam = string.Join(",", symbolList);
        return await ExecuteWithRetryAsync(async (crumb) =>
        {
-            var url =
-                $"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
-            using var response = await _httpClient.GetAsync(url, cancellationToken);
-
-            if (!response.IsSuccessStatusCode)
+            string[] baseUrls = new[]
            {
+                "https://query1.finance.yahoo.com/v7/finance/quote",
+                "https://query2.finance.yahoo.com/v7/finance/quote"
+            };
+
+            foreach (var baseUrl in baseUrls)
+            {
+                var url = $"{baseUrl}?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
+                using var request = new HttpRequestMessage(HttpMethod.Get, url);
+                request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
+                request.Headers.Add("Accept", "*/*");
+                request.Headers.Add("Origin", "https://finance.yahoo.com");
+                request.Headers.Add("Referer", "https://finance.yahoo.com/");
+
+                if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
+                {
+                    request.Headers.Add("Cookie", _rawCookieHeader);
+                }
+
+                using var response = await _httpClient.SendAsync(request, cancellationToken);
+                var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
+
+                if (response.IsSuccessStatusCode)
+                {
+                    var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(responseContent, GetJsonOptions());
+                    return (false, dto);
+                }
+
                if (_finlyticLogger != null)
-                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
-                return (
-                    response.StatusCode == HttpStatusCode.Unauthorized ||
-                    response.StatusCode == HttpStatusCode.Forbidden, null);
+                    await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, 
+                        "[YahooFinanceClient] GetQuotes failed with status {Status}. URL: {Url}. Response Body: {Body}", 
+                        response.StatusCode, url, responseContent);
+
+                if (response.StatusCode == HttpStatusCode.Unauthorized ||
+                    response.StatusCode == HttpStatusCode.Forbidden ||
+                    response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    return (true, null);
+                }
            }

-            var json = await response.Content.ReadAsStringAsync(cancellationToken);
-            var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
-            return (false, dto);
+            return (false, null);
        }, cancellationToken);
    }

@@ -408,26 +693,29 @@ public class YahooFinanceClient
        CancellationToken cancellationToken) where T : class
    {
        var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
+        if (!string.IsNullOrEmpty(crumb))
+        {
+            var (isAuthError, result) = await action(crumb);
+            if (!isAuthError && result != null)
+            {
+                return result;
+            }
+
+            if (!isAuthError)
+            {
+                return result;
+            }
+        }
+
+        // Re-authenticate when auth error (401/403/429) or empty crumb occurs
+        if (_finlyticLogger != null)
+            await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error or invalid crumb encountered (401/403/429). Force re-authenticating...");
+
+        crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
        if (string.IsNullOrEmpty(crumb)) return null;

-        var (isAuthError, result) = await action(crumb);
-        if (!isAuthError && result != null)
-        {
-            return result;
-        }
-
-        if (isAuthError)
-        {
-            if (_finlyticLogger != null)
-                await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
-            crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
-            if (string.IsNullOrEmpty(crumb)) return null;
-
-            var (_, retryResult) = await action(crumb);
-            return retryResult;
-        }
-
-        return result;
+        var (_, retryResult) = await action(crumb);
+        return retryResult;
    }

    private static JsonSerializerOptions GetJsonOptions()