Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: save current workspace state including FinlyticAnalyzer fixes, FinlyticApp trade route alignment, and DTO audit documentation

Browse Source
This commit is contained in:
Kleidukos
2026-08-12 18:30:42 +02:00
parent a9553e9fbf
commit 3d8af3940b
163 changed files with 3421 additions and 1751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticFundamentals/Services/FundamentalsDbService.cs
+195 -2
View File
@@ -37,6 +37,11 @@ public interface IFundamentalsDbService
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A list of corporate events sorted chronologically.</returns>
Task<List<CorporateEventDto>> GetAllEventsAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Gets corporate events for a specific month.
/// </summary>
Task<List<CorporateEventDto>> GetEventsByMonthAsync(int year, int month, CancellationToken cancellationToken = default);
}
public class FundamentalsDbService : IFundamentalsDbService
@@ -45,17 +50,20 @@ public class FundamentalsDbService : IFundamentalsDbService
private readonly IServiceScopeFactory _scopeFactory;
private readonly IYahooFinanceScraper _scraper;
private readonly IHtmlFallbackScraper _fallbackScraper;
private readonly YahooFinanceClient _yahooClient;
private readonly ILogger<FundamentalsDbService> _logger;
public FundamentalsDbService(
IServiceScopeFactory scopeFactory,
IYahooFinanceScraper scraper,
IHtmlFallbackScraper fallbackScraper,
YahooFinanceClient yahooClient,
ILogger<FundamentalsDbService> logger)
{
_scopeFactory = scopeFactory;
_scraper = scraper;
_fallbackScraper = fallbackScraper;
_yahooClient = yahooClient;
_logger = logger;
}
@@ -197,10 +205,49 @@ public class FundamentalsDbService : IFundamentalsDbService
}
}
if (tickers.Count == 0) return existingEntity;
if (tickers.Count == 0)
{
_logger.LogWarning("[YahooFallbackScraper] No tickers resolved for ISIN {Isin}. Fallback scraper cannot be invoked without a ticker.", isin);
return existingEntity;
}
var primaryTicker = tickers[0];
_logger.LogInformation("[YahooFallbackScraper] Primary ticker resolved: '{Ticker}' for ISIN {Isin}", primaryTicker, isin);
var scraped = await _scraper.ScrapeFundamentalsAsync(isin, primaryTicker, cancellationToken);
bool needsFallback = IsDataIncomplete(scraped, _logger);
_logger.LogInformation("[YahooFallbackScraper] Primary scrape completeness check for '{Ticker}': scrapedIsNull={ScrapedIsNull}, needsFallback={NeedsFallback}",
primaryTicker, scraped == null, needsFallback);
if (needsFallback)
{
_logger.LogInformation("[YahooFallbackScraper] Executing Playwright Fallback Scraper for ticker '{Ticker}' (ISIN: {Isin})...", primaryTicker, isin);
var fallbackData = await _fallbackScraper.ScrapeFallbackAsync(isin, primaryTicker, cancellationToken);
if (fallbackData != null)
{
_logger.LogInformation("[YahooFallbackScraper] Fallback scraper returned data for {Ticker}. MarketCap={MarketCap}, EV={EV}, Sector='{Sector}'",
primaryTicker, fallbackData.TickerData?.MarketCapitalization, fallbackData.TickerData?.EnterpriseValue, fallbackData.Fundamentals?.Sector);
if (scraped == null)
{
_logger.LogInformation("[YahooFallbackScraper] Primary scraped data was null. Using entirely Playwright fallback data for {Ticker}...", primaryTicker);
scraped = fallbackData;
}
else
{
_logger.LogInformation("[YahooFallbackScraper] Merging Playwright fallback data into primary scraped data for {Ticker}...", primaryTicker);
// Merge fallback into scraped
MergeFundamentals(scraped, fallbackData);
}
}
else
{
_logger.LogWarning("[YahooFallbackScraper] Fallback scraper returned NULL for {Ticker}!", primaryTicker);
}
}
// ------------------------------
if (scraped == null) return existingEntity;
var tickerEntities = new List<TickerFundamentalsEntity> { scraped.TickerData };
@@ -234,7 +281,93 @@ public class FundamentalsDbService : IFundamentalsDbService
return await LoadEntityGraphAsync(context, isin, cancellationToken);
}
private static void MergeFundamentals(ScrapedFundamentalsData target, ScrapedFundamentalsData source)
{
var t = target.TickerData;
var s = source.TickerData;
// Kennzahlen & Ratios
if (t.MarketCapitalization == 0 && s.MarketCapitalization > 0) t.MarketCapitalization = s.MarketCapitalization;
if ((t.EnterpriseValue == 0) && s.EnterpriseValue > 0) t.EnterpriseValue = s.EnterpriseValue;
t.PeRatioTrailing ??= s.PeRatioTrailing;
t.PeRatioForward ??= s.PeRatioForward;
t.PegRatio ??= s.PegRatio;
t.PbRatio ??= s.PbRatio;
t.PsRatio ??= s.PsRatio;
t.EvToEbitda ??= s.EvToEbitda;
t.EvToRevenue ??= s.EvToRevenue;
// Margen
t.GrossMargin ??= s.GrossMargin;
t.OperatingMargin ??= s.OperatingMargin;
t.NetProfitMargin ??= s.NetProfitMargin;
t.ReturnOnEquity ??= s.ReturnOnEquity;
t.ReturnOnAssets ??= s.ReturnOnAssets;
// Preise & Dividenden
if (t.FiftyTwoWeekHigh == 0 && s.FiftyTwoWeekHigh > 0) t.FiftyTwoWeekHigh = s.FiftyTwoWeekHigh;
if (t.FiftyTwoWeekLow == 0 && s.FiftyTwoWeekLow > 0) t.FiftyTwoWeekLow = s.FiftyTwoWeekLow;
if ((!t.DividendYield.HasValue || t.DividendYield == 0) && s.DividendYield > 0) t.DividendYield = s.DividendYield;
// Stammdaten
if (string.IsNullOrWhiteSpace(target.Fundamentals.Sector)) target.Fundamentals.Sector = source.Fundamentals.Sector;
if (string.IsNullOrWhiteSpace(target.Fundamentals.Industry)) target.Fundamentals.Industry = source.Fundamentals.Industry;
if (!target.Fundamentals.Employees.HasValue) target.Fundamentals.Employees = source.Fundamentals.Employees;
if (string.IsNullOrWhiteSpace(target.Fundamentals.BusinessSummary)) target.Fundamentals.BusinessSummary = source.Fundamentals.BusinessSummary;
}
private static bool IsDataIncomplete(ScrapedFundamentalsData? data, ILogger logger)
{
if (data == null || data.TickerData == null)
{
logger.LogWarning("[YahooFallbackScraper] IsDataIncomplete -> TRUE (scraped data or TickerData is NULL)");
return true;
}
var td = data.TickerData;
var f = data.Fundamentals;
int missingCriticalFields = 0;
// 1. Absolute Must-Haves (sofortiger Fallback wenn 0)
if (td.MarketCapitalization == 0)
{
logger.LogWarning("[YahooFallbackScraper] IsDataIncomplete -> TRUE (MarketCapitalization is 0)");
return true;
}
if (td.FiftyTwoWeekHigh == 0 || td.FiftyTwoWeekLow == 0)
{
logger.LogWarning("[YahooFallbackScraper] IsDataIncomplete -> TRUE (52WeekHigh={High} or 52WeekLow={Low} is 0)", td.FiftyTwoWeekHigh, td.FiftyTwoWeekLow);
return true;
}
// 2. Bewertung & Ratios (Zähle fehlende Metriken)
// KGV: Trailing ODER Forward muss vorhanden sein, sonst zählt die KGV-Bewertung als fehlend
if ((!td.PeRatioTrailing.HasValue || td.PeRatioTrailing == 0) && (!td.PeRatioForward.HasValue || td.PeRatioForward == 0))
missingCriticalFields++;
if (!td.PbRatio.HasValue || td.PbRatio == 0) missingCriticalFields++;
if (!td.PsRatio.HasValue || td.PsRatio == 0) missingCriticalFields++;
if (td.EnterpriseValue == 0) missingCriticalFields++;
// 3. Margen & Profitabilität
if (!td.GrossMargin.HasValue) missingCriticalFields++;
if (!td.OperatingMargin.HasValue) missingCriticalFields++;
if (!td.NetProfitMargin.HasValue) missingCriticalFields++;
// 4. Stammdaten
if (string.IsNullOrWhiteSpace(f.Sector)) missingCriticalFields++;
if (string.IsNullOrWhiteSpace(f.Industry)) missingCriticalFields++;
// Wenn 2 oder mehr der wichtigen Kennzahlen fehlen, gilt die Quelle als unvollständig
bool isIncomplete = missingCriticalFields >= 2;
logger.LogInformation("[YahooFallbackScraper] IsDataIncomplete total missingCriticalFields={Count} (threshold >= 2 -> isIncomplete={Result})", missingCriticalFields, isIncomplete);
return isIncomplete;
}
#endregion
#region Data Access & Mapping Helpers
@@ -546,9 +679,14 @@ public class FundamentalsDbService : IFundamentalsDbService
using var scope = _scopeFactory.CreateScope();
var context = scope.ServiceProvider.GetRequiredService<FundamentalsDbContext>();
var now = DateTime.UtcNow;
var startOfToday = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0, DateTimeKind.Utc);
var endOfYear = new DateTime(now.Year, 12, 31, 23, 59, 59, DateTimeKind.Utc);
var entities = await context.AssetFundamentals
.AsNoTracking()
.Where(f => f.NextEarningsDate.HasValue || f.ExDividendDate.HasValue)
.Where(f => (f.NextEarningsDate.HasValue && f.NextEarningsDate.Value >= startOfToday && f.NextEarningsDate.Value <= endOfYear)
|| (f.ExDividendDate.HasValue && f.ExDividendDate.Value >= startOfToday && f.ExDividendDate.Value <= endOfYear))
.ToListAsync(cancellationToken);
var events = new List<CorporateEventDto>();
@@ -585,5 +723,60 @@ public class FundamentalsDbService : IFundamentalsDbService
return events.OrderBy(e => e.Date).ToList();
}
/// <inheritdoc />
public async Task<List<CorporateEventDto>> GetEventsByMonthAsync(int year, int month, CancellationToken cancellationToken = default)
{
using var scope = _scopeFactory.CreateScope();
var context = scope.ServiceProvider.GetRequiredService<FundamentalsDbContext>();
var startOfMonth = new DateTime(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
var startOfNextMonth = startOfMonth.AddMonths(1);
_logger.LogInformation("[FundamentalsDbService] Querying events between {Start} and {End}", startOfMonth, startOfNextMonth);
var entities = await context.AssetFundamentals
.AsNoTracking()
.Where(f => (f.NextEarningsDate != null && f.NextEarningsDate >= startOfMonth && f.NextEarningsDate < startOfNextMonth)
|| (f.ExDividendDate != null && f.ExDividendDate >= startOfMonth && f.ExDividendDate < startOfNextMonth))
.ToListAsync(cancellationToken);
_logger.LogInformation("[FundamentalsDbService] Found {Count} entities.", entities.Count);
var events = new List<CorporateEventDto>();
foreach (var entity in entities)
{
var companyName = string.IsNullOrWhiteSpace(entity.CompanyName) ? entity.PrimaryTicker : entity.CompanyName;
if (entity.NextEarningsDate != null && entity.NextEarningsDate >= startOfMonth && entity.NextEarningsDate < startOfNextMonth)
{
events.Add(new CorporateEventDto
{
Isin = entity.Isin,
Ticker = entity.PrimaryTicker,
CompanyName = companyName,
EventType = "Quartalsergebnis",
Date = entity.NextEarningsDate.Value
});
}
if (entity.ExDividendDate != null && entity.ExDividendDate >= startOfMonth && entity.ExDividendDate < startOfNextMonth)
{
events.Add(new CorporateEventDto
{
Isin = entity.Isin,
Ticker = entity.PrimaryTicker,
CompanyName = companyName,
EventType = "Ex-Dividendentag",
Date = entity.ExDividendDate.Value
});
}
}
_logger.LogInformation("[FundamentalsDbService] Returning {Count} total events.", events.Count);
return events.OrderBy(e => e.Date).ToList();
}
#endregion
}
FinlyticFundamentals/Services/HtmlFallbackScraper.cs
+305
View File
@@ -0,0 +1,305 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using FinlyticFundamentals.Entities;
using Microsoft.Extensions.Logging;
using Microsoft.Playwright;
namespace FinlyticFundamentals.Services;
public interface IHtmlFallbackScraper
{
Task<ScrapedFundamentalsData?> ScrapeFallbackAsync(string isin, string ticker, CancellationToken cancellationToken = default);
}
/// <summary>
/// Fallback scraper using Playwright (headless Chromium) to scrape Yahoo Finance pages
/// for alternative ticker symbols (e.g., APC.SG) whose data is not available via the API.
/// Targets stable data-testid selectors from the rendered Yahoo Finance SPA.
/// </summary>
public class HtmlFallbackScraper : IHtmlFallbackScraper, IAsyncDisposable
{
private readonly ILogger<HtmlFallbackScraper> _logger;
private IPlaywright? _playwright;
private IBrowser? _browser;
private readonly SemaphoreSlim _browserLock = new(1, 1);
public HtmlFallbackScraper(ILogger<HtmlFallbackScraper> logger)
{
_logger = logger;
}
public async Task<ScrapedFundamentalsData?> ScrapeFallbackAsync(
string isin,
string ticker,
CancellationToken cancellationToken = default)
{
_logger.LogInformation(
"[YahooFallbackScraper] Executing Playwright Fallback Scrape for ticker '{Ticker}' (ISIN: {Isin})...",
ticker, isin);
try
{
var browser = await GetOrInitBrowserAsync(cancellationToken);
var fundamentals = new AssetFundamentalsEntity
{
Isin = isin,
PrimaryTicker = ticker,
LastUpdatedAt = DateTime.UtcNow,
LastStaticUpdatedAt = DateTime.UtcNow
};
var tickerData = new TickerFundamentalsEntity
{
Ticker = ticker,
Isin = isin,
LastUpdatedAt = DateTime.UtcNow
};
fundamentals.CompanyName = ticker;
// Share context for both pages so we only have to accept cookies once
await using (var ctx = await browser.NewContextAsync(BuildContextOptions()))
{
// ── 1. Key Statistics Page ─────────────────────────────────────
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(ticker)}/key-statistics/";
_logger.LogInformation("[YahooFallbackScraper] Navigating to stats page: {Url}", statsUrl);
var statsPage = await ctx.NewPageAsync();
try
{
await statsPage.GotoAsync(statsUrl, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 45_000
});
await HandleConsentAsync(statsPage);
// Wait for the statistics section to be visible
await statsPage.WaitForSelectorAsync(
"section[data-testid='qsp-statistics'], section[data-testid='stats-highlight']",
new PageWaitForSelectorOptions { Timeout = 20_000 });
// --- Valuation Measures Table ---
var valuationRows = await statsPage.QuerySelectorAllAsync(
"section[data-testid='qsp-statistics'] table tbody tr");
foreach (var row in valuationRows)
{
var cells = await row.QuerySelectorAllAsync("td");
if (cells.Count < 2) continue;
var label = (await cells[0].InnerTextAsync()).Trim();
var value = (await cells[1].InnerTextAsync()).Trim();
if (string.IsNullOrWhiteSpace(value) || value == "N/A" || value == "--") continue;
switch (NormalizeLabel(label))
{
case "market cap": tickerData.MarketCapitalization = ParseSuffixNumber(value) ?? 0m; break;
case "enterprise value": tickerData.EnterpriseValue = ParseSuffixNumber(value) ?? 0m; break;
case "trailing p/e": tickerData.PeRatioTrailing = ParseDecimal(value); break;
case "forward p/e": tickerData.PeRatioForward = ParseDecimal(value); break;
case "peg ratio (5yr expected)": tickerData.PegRatio = ParseDecimal(value); break;
case "price/sales": tickerData.PsRatio = ParseDecimal(value); break;
case "price/book": tickerData.PbRatio = ParseDecimal(value); break;
case "enterprise value/revenue": tickerData.EvToRevenue = ParseDecimal(value); break;
case "enterprise value/ebitda": tickerData.EvToEbitda = ParseDecimal(value); break;
}
}
// --- Financial Highlight Cards ---
var highlightRows = await statsPage.QuerySelectorAllAsync(
"div[data-testid='stats-highlight'] section[data-testid='card-container'] table tr");
foreach (var row in highlightRows)
{
var cells = await row.QuerySelectorAllAsync("td");
if (cells.Count < 2) continue;
var label = (await cells[0].InnerTextAsync()).Trim();
var value = (await cells[1].InnerTextAsync()).Trim();
if (string.IsNullOrWhiteSpace(value) || value == "N/A" || value == "--") continue;
switch (NormalizeLabel(label))
{
case "profit margin": tickerData.NetProfitMargin = ParsePercent(value); break;
case "operating margin": tickerData.OperatingMargin = ParsePercent(value); break;
case "return on assets": tickerData.ReturnOnAssets = ParsePercent(value); break;
case "return on equity": tickerData.ReturnOnEquity = ParsePercent(value); break;
case "current ratio": tickerData.CurrentRatio = ParseDecimal(value); break;
case "quick ratio": tickerData.QuickRatio = ParseDecimal(value); break;
case "total debt/equity": tickerData.DebtToEquity = ParseDecimal(value); break;
case "52 week high": tickerData.FiftyTwoWeekHigh = ParseDecimal(value) ?? 0m; break;
case "52 week low": tickerData.FiftyTwoWeekLow = ParseDecimal(value) ?? 0m; break;
case "forward annual dividend yield":
case "trailing annual dividend yield":
tickerData.DividendYield ??= ParsePercent(value); break;
case "payout ratio": tickerData.PayoutRatio = ParsePercent(value); break;
}
}
_logger.LogInformation(
"[YahooFallbackScraper] Stats parsed for '{Ticker}': MarketCap={MarketCap}, EV={EV}, TrailingPE={PE}, ForwardPE={FPE}",
ticker, tickerData.MarketCapitalization, tickerData.EnterpriseValue,
tickerData.PeRatioTrailing, tickerData.PeRatioForward);
}
catch (TimeoutException tex)
{
_logger.LogWarning(tex,
"[YahooFallbackScraper] Timeout waiting for stats page selectors for ticker '{Ticker}'. Page may not have loaded.", ticker);
}
finally
{
await statsPage.CloseAsync();
}
}
// ── 2. Guard: no meaningful data ──────────────────────────────
if (tickerData.MarketCapitalization == 0)
{
_logger.LogWarning(
"[YahooFallbackScraper] Playwright scrape for '{Ticker}' produced no meaningful data (MarketCap=0). Returning NULL.",
ticker);
return null;
}
_logger.LogInformation(
"[YahooFallbackScraper] Playwright Fallback Scrape complete for '{Ticker}'. MarketCap={MarketCap}",
ticker, tickerData.MarketCapitalization);
return new ScrapedFundamentalsData(
fundamentals,
tickerData,
new List<CompanyExecutiveEntity>(),
new List<FinancialStatementEntity>(),
new List<ForwardEstimateEntity>()
);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[YahooFallbackScraper] Error during Playwright Fallback Scrape for ticker '{Ticker}' (ISIN: {Isin})",
ticker, isin);
return null;
}
}
// ── Browser Lifecycle ──────────────────────────────────────────────────
private async Task<IBrowser> GetOrInitBrowserAsync(CancellationToken cancellationToken = default)
{
if (_browser != null) return _browser;
await _browserLock.WaitAsync(cancellationToken);
try
{
if (_browser != null) return _browser;
_playwright = await Playwright.CreateAsync();
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = true,
Args = new[] { "--no-sandbox", "--disable-dev-shm-usage" }
});
_logger.LogInformation("[YahooFallbackScraper] Playwright Chromium browser initialized.");
return _browser;
}
finally
{
_browserLock.Release();
}
}
private static BrowserNewContextOptions BuildContextOptions() => new()
{
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
ViewportSize = new ViewportSize { Width = 1280, Height = 900 },
Locale = "en-US",
ExtraHTTPHeaders = new Dictionary<string, string>
{
["Accept-Language"] = "en-US,en;q=0.9"
}
};
public async ValueTask DisposeAsync()
{
if (_browser != null) await _browser.CloseAsync();
_playwright?.Dispose();
}
// ── Parse Helpers ─────────────────────────────────────────────────────
private async Task HandleConsentAsync(IPage page)
{
try
{
if (page.Url.Contains("consent.yahoo.com"))
{
_logger.LogInformation("[YahooFallbackScraper] Redirected to consent page. Attempting to accept cookies...");
var agreeBtn = page.Locator("button[name='agree'], button.accept-all, button[value='agree']");
if (await agreeBtn.CountAsync() > 0)
{
await agreeBtn.First.ClickAsync();
await page.WaitForNavigationAsync(new PageWaitForNavigationOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 20_000 });
_logger.LogInformation("[YahooFallbackScraper] Cookie consent accepted. Navigated back to: {Url}", page.Url);
}
else
{
_logger.LogWarning("[YahooFallbackScraper] On consent page but could not find the 'agree' button.");
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[YahooFallbackScraper] Error while handling cookie consent.");
}
}
/// <summary>Lowercases, strips trailing digits, common Yahoo qualifiers like (ttm), (mrq), and extra spaces.</summary>
private static string NormalizeLabel(string label)
{
label = label.ToLowerInvariant();
label = Regex.Replace(label, @"\s*\d+\s*$", ""); // trailing superscripts
label = label.Replace("(ttm)", "").Replace("(mrq)", "").Replace("(fye)", ""); // remove date qualifiers
label = Regex.Replace(label, @"\s+", " "); // collapse spaces
return label.Trim();
}
private static decimal? ParseDecimal(string? input)
{
if (string.IsNullOrWhiteSpace(input) || input == "N/A" || input == "--" || input == "-") return null;
input = Regex.Replace(input, @"[^\d.-]", "");
return decimal.TryParse(input, NumberStyles.Any, CultureInfo.InvariantCulture, out var val) ? val : null;
}
private static decimal? ParsePercent(string? input)
{
var val = ParseDecimal(input);
if (!val.HasValue) return null;
return val.Value > 1m ? val.Value / 100m : val.Value;
}
private static decimal? ParseSuffixNumber(string? input)
{
if (string.IsNullOrWhiteSpace(input) || input == "N/A" || input == "--" || input == "-") return null;
input = input.Trim();
decimal multiplier = input.EndsWith("T", StringComparison.OrdinalIgnoreCase) ? 1_000_000_000_000m
: input.EndsWith("B", StringComparison.OrdinalIgnoreCase) ? 1_000_000_000m
: input.EndsWith("M", StringComparison.OrdinalIgnoreCase) ? 1_000_000m
: input.EndsWith("K", StringComparison.OrdinalIgnoreCase) ? 1_000m
: 1m;
var numPart = Regex.Replace(input, @"[^\d.-]", "");
var val = ParseDecimal(numPart);
return val.HasValue ? val.Value * multiplier : null;
}
}