refactor: save current workspace state including FinlyticAnalyzer fixes, FinlyticApp trade route alignment, and DTO audit documentation
This commit is contained in:
@@ -0,0 +1,305 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticFundamentals.Entities;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace FinlyticFundamentals.Services;
|
||||
|
||||
public interface IHtmlFallbackScraper
|
||||
{
|
||||
Task<ScrapedFundamentalsData?> ScrapeFallbackAsync(string isin, string ticker, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fallback scraper using Playwright (headless Chromium) to scrape Yahoo Finance pages
|
||||
/// for alternative ticker symbols (e.g., APC.SG) whose data is not available via the API.
|
||||
/// Targets stable data-testid selectors from the rendered Yahoo Finance SPA.
|
||||
/// </summary>
|
||||
public class HtmlFallbackScraper : IHtmlFallbackScraper, IAsyncDisposable
|
||||
{
|
||||
private readonly ILogger<HtmlFallbackScraper> _logger;
|
||||
|
||||
private IPlaywright? _playwright;
|
||||
private IBrowser? _browser;
|
||||
private readonly SemaphoreSlim _browserLock = new(1, 1);
|
||||
|
||||
public HtmlFallbackScraper(ILogger<HtmlFallbackScraper> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ScrapedFundamentalsData?> ScrapeFallbackAsync(
|
||||
string isin,
|
||||
string ticker,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"[YahooFallbackScraper] Executing Playwright Fallback Scrape for ticker '{Ticker}' (ISIN: {Isin})...",
|
||||
ticker, isin);
|
||||
|
||||
try
|
||||
{
|
||||
var browser = await GetOrInitBrowserAsync(cancellationToken);
|
||||
|
||||
var fundamentals = new AssetFundamentalsEntity
|
||||
{
|
||||
Isin = isin,
|
||||
PrimaryTicker = ticker,
|
||||
LastUpdatedAt = DateTime.UtcNow,
|
||||
LastStaticUpdatedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
var tickerData = new TickerFundamentalsEntity
|
||||
{
|
||||
Ticker = ticker,
|
||||
Isin = isin,
|
||||
LastUpdatedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
fundamentals.CompanyName = ticker;
|
||||
// Share context for both pages so we only have to accept cookies once
|
||||
await using (var ctx = await browser.NewContextAsync(BuildContextOptions()))
|
||||
{
|
||||
// ── 1. Key Statistics Page ─────────────────────────────────────
|
||||
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(ticker)}/key-statistics/";
|
||||
_logger.LogInformation("[YahooFallbackScraper] Navigating to stats page: {Url}", statsUrl);
|
||||
|
||||
var statsPage = await ctx.NewPageAsync();
|
||||
try
|
||||
{
|
||||
await statsPage.GotoAsync(statsUrl, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 45_000
|
||||
});
|
||||
|
||||
await HandleConsentAsync(statsPage);
|
||||
|
||||
// Wait for the statistics section to be visible
|
||||
await statsPage.WaitForSelectorAsync(
|
||||
"section[data-testid='qsp-statistics'], section[data-testid='stats-highlight']",
|
||||
new PageWaitForSelectorOptions { Timeout = 20_000 });
|
||||
|
||||
// --- Valuation Measures Table ---
|
||||
var valuationRows = await statsPage.QuerySelectorAllAsync(
|
||||
"section[data-testid='qsp-statistics'] table tbody tr");
|
||||
|
||||
foreach (var row in valuationRows)
|
||||
{
|
||||
var cells = await row.QuerySelectorAllAsync("td");
|
||||
if (cells.Count < 2) continue;
|
||||
|
||||
var label = (await cells[0].InnerTextAsync()).Trim();
|
||||
var value = (await cells[1].InnerTextAsync()).Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value) || value == "N/A" || value == "--") continue;
|
||||
|
||||
switch (NormalizeLabel(label))
|
||||
{
|
||||
case "market cap": tickerData.MarketCapitalization = ParseSuffixNumber(value) ?? 0m; break;
|
||||
case "enterprise value": tickerData.EnterpriseValue = ParseSuffixNumber(value) ?? 0m; break;
|
||||
case "trailing p/e": tickerData.PeRatioTrailing = ParseDecimal(value); break;
|
||||
case "forward p/e": tickerData.PeRatioForward = ParseDecimal(value); break;
|
||||
case "peg ratio (5yr expected)": tickerData.PegRatio = ParseDecimal(value); break;
|
||||
case "price/sales": tickerData.PsRatio = ParseDecimal(value); break;
|
||||
case "price/book": tickerData.PbRatio = ParseDecimal(value); break;
|
||||
case "enterprise value/revenue": tickerData.EvToRevenue = ParseDecimal(value); break;
|
||||
case "enterprise value/ebitda": tickerData.EvToEbitda = ParseDecimal(value); break;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Financial Highlight Cards ---
|
||||
var highlightRows = await statsPage.QuerySelectorAllAsync(
|
||||
"div[data-testid='stats-highlight'] section[data-testid='card-container'] table tr");
|
||||
|
||||
foreach (var row in highlightRows)
|
||||
{
|
||||
var cells = await row.QuerySelectorAllAsync("td");
|
||||
if (cells.Count < 2) continue;
|
||||
|
||||
var label = (await cells[0].InnerTextAsync()).Trim();
|
||||
var value = (await cells[1].InnerTextAsync()).Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value) || value == "N/A" || value == "--") continue;
|
||||
|
||||
switch (NormalizeLabel(label))
|
||||
{
|
||||
case "profit margin": tickerData.NetProfitMargin = ParsePercent(value); break;
|
||||
case "operating margin": tickerData.OperatingMargin = ParsePercent(value); break;
|
||||
case "return on assets": tickerData.ReturnOnAssets = ParsePercent(value); break;
|
||||
case "return on equity": tickerData.ReturnOnEquity = ParsePercent(value); break;
|
||||
case "current ratio": tickerData.CurrentRatio = ParseDecimal(value); break;
|
||||
case "quick ratio": tickerData.QuickRatio = ParseDecimal(value); break;
|
||||
case "total debt/equity": tickerData.DebtToEquity = ParseDecimal(value); break;
|
||||
case "52 week high": tickerData.FiftyTwoWeekHigh = ParseDecimal(value) ?? 0m; break;
|
||||
case "52 week low": tickerData.FiftyTwoWeekLow = ParseDecimal(value) ?? 0m; break;
|
||||
case "forward annual dividend yield":
|
||||
case "trailing annual dividend yield":
|
||||
tickerData.DividendYield ??= ParsePercent(value); break;
|
||||
case "payout ratio": tickerData.PayoutRatio = ParsePercent(value); break;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"[YahooFallbackScraper] Stats parsed for '{Ticker}': MarketCap={MarketCap}, EV={EV}, TrailingPE={PE}, ForwardPE={FPE}",
|
||||
ticker, tickerData.MarketCapitalization, tickerData.EnterpriseValue,
|
||||
tickerData.PeRatioTrailing, tickerData.PeRatioForward);
|
||||
}
|
||||
catch (TimeoutException tex)
|
||||
{
|
||||
_logger.LogWarning(tex,
|
||||
"[YahooFallbackScraper] Timeout waiting for stats page selectors for ticker '{Ticker}'. Page may not have loaded.", ticker);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await statsPage.CloseAsync();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ── 2. Guard: no meaningful data ──────────────────────────────
|
||||
if (tickerData.MarketCapitalization == 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"[YahooFallbackScraper] Playwright scrape for '{Ticker}' produced no meaningful data (MarketCap=0). Returning NULL.",
|
||||
ticker);
|
||||
return null;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"[YahooFallbackScraper] Playwright Fallback Scrape complete for '{Ticker}'. MarketCap={MarketCap}",
|
||||
ticker, tickerData.MarketCapitalization);
|
||||
|
||||
return new ScrapedFundamentalsData(
|
||||
fundamentals,
|
||||
tickerData,
|
||||
new List<CompanyExecutiveEntity>(),
|
||||
new List<FinancialStatementEntity>(),
|
||||
new List<ForwardEstimateEntity>()
|
||||
);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"[YahooFallbackScraper] Error during Playwright Fallback Scrape for ticker '{Ticker}' (ISIN: {Isin})",
|
||||
ticker, isin);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Browser Lifecycle ──────────────────────────────────────────────────
|
||||
|
||||
private async Task<IBrowser> GetOrInitBrowserAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_browser != null) return _browser;
|
||||
|
||||
await _browserLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (_browser != null) return _browser;
|
||||
|
||||
_playwright = await Playwright.CreateAsync();
|
||||
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
|
||||
{
|
||||
Headless = true,
|
||||
Args = new[] { "--no-sandbox", "--disable-dev-shm-usage" }
|
||||
});
|
||||
|
||||
|
||||
_logger.LogInformation("[YahooFallbackScraper] Playwright Chromium browser initialized.");
|
||||
return _browser;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_browserLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static BrowserNewContextOptions BuildContextOptions() => new()
|
||||
{
|
||||
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||
ViewportSize = new ViewportSize { Width = 1280, Height = 900 },
|
||||
Locale = "en-US",
|
||||
ExtraHTTPHeaders = new Dictionary<string, string>
|
||||
{
|
||||
["Accept-Language"] = "en-US,en;q=0.9"
|
||||
}
|
||||
};
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_browser != null) await _browser.CloseAsync();
|
||||
_playwright?.Dispose();
|
||||
}
|
||||
|
||||
// ── Parse Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
private async Task HandleConsentAsync(IPage page)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (page.Url.Contains("consent.yahoo.com"))
|
||||
{
|
||||
_logger.LogInformation("[YahooFallbackScraper] Redirected to consent page. Attempting to accept cookies...");
|
||||
var agreeBtn = page.Locator("button[name='agree'], button.accept-all, button[value='agree']");
|
||||
if (await agreeBtn.CountAsync() > 0)
|
||||
{
|
||||
await agreeBtn.First.ClickAsync();
|
||||
await page.WaitForNavigationAsync(new PageWaitForNavigationOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 20_000 });
|
||||
_logger.LogInformation("[YahooFallbackScraper] Cookie consent accepted. Navigated back to: {Url}", page.Url);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("[YahooFallbackScraper] On consent page but could not find the 'agree' button.");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[YahooFallbackScraper] Error while handling cookie consent.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Lowercases, strips trailing digits, common Yahoo qualifiers like (ttm), (mrq), and extra spaces.</summary>
|
||||
private static string NormalizeLabel(string label)
|
||||
{
|
||||
label = label.ToLowerInvariant();
|
||||
label = Regex.Replace(label, @"\s*\d+\s*$", ""); // trailing superscripts
|
||||
label = label.Replace("(ttm)", "").Replace("(mrq)", "").Replace("(fye)", ""); // remove date qualifiers
|
||||
label = Regex.Replace(label, @"\s+", " "); // collapse spaces
|
||||
return label.Trim();
|
||||
}
|
||||
|
||||
private static decimal? ParseDecimal(string? input)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(input) || input == "N/A" || input == "--" || input == "-") return null;
|
||||
input = Regex.Replace(input, @"[^\d.-]", "");
|
||||
return decimal.TryParse(input, NumberStyles.Any, CultureInfo.InvariantCulture, out var val) ? val : null;
|
||||
}
|
||||
|
||||
private static decimal? ParsePercent(string? input)
|
||||
{
|
||||
var val = ParseDecimal(input);
|
||||
if (!val.HasValue) return null;
|
||||
return val.Value > 1m ? val.Value / 100m : val.Value;
|
||||
}
|
||||
|
||||
private static decimal? ParseSuffixNumber(string? input)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(input) || input == "N/A" || input == "--" || input == "-") return null;
|
||||
input = input.Trim();
|
||||
decimal multiplier = input.EndsWith("T", StringComparison.OrdinalIgnoreCase) ? 1_000_000_000_000m
|
||||
: input.EndsWith("B", StringComparison.OrdinalIgnoreCase) ? 1_000_000_000m
|
||||
: input.EndsWith("M", StringComparison.OrdinalIgnoreCase) ? 1_000_000m
|
||||
: input.EndsWith("K", StringComparison.OrdinalIgnoreCase) ? 1_000m
|
||||
: 1m;
|
||||
var numPart = Regex.Replace(input, @"[^\d.-]", "");
|
||||
var val = ParseDecimal(numPart);
|
||||
return val.HasValue ? val.Value * multiplier : null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user