feat(core): update DTOs, Trade Republic client, Yahoo scrapers, and dynamic settings

This commit is contained in:
2026-08-14 23:55:02 +02:00
parent 3d8af3940b
commit 4f733bf8c3
511 changed files with 1990 additions and 1080548 deletions
+419
View File
@@ -0,0 +1,419 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services.Yahoo;
/// <summary>
/// Thread-sicherer Client für den Zugriff auf die internen Yahoo Finance APIs.
/// Verwaltet automatisch den erforderlichen Cookie- (A3) und Crumb-Token-Authentifizierungs-Flow.
/// </summary>
public interface IYahooFinanceClient
{
/// <summary>
/// Stellt sicher, dass die aktuelle Session über ein gültiges Cookie und einen Crumb-Token verfügt.
/// </summary>
/// <param name="forceRefresh">Erzwingt das Erneuern des Authentifizierungs-Tokens, selbst wenn die Frist noch nicht abgelaufen ist.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Crumb-Token oder <c>null</c>, wenn die Authentifizierung fehlgeschlagen ist.</returns>
Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>
/// Sucht nach Tickern, Namen, ISINs oder Firmen über die Yahoo Finance Such-API.
/// erfordert keine Cookie/Crumb-Authentifizierung.
/// </summary>
/// <param name="query">Der Suchbegriff (z. B. "Apple", "US0378331005", "AAPL").</param>
/// <param name="quotesCount">Die maximale Anzahl an Treffern für Wertpapiere/Aktien.</param>
/// <param name="newsCount">Die maximale Anzahl an News-Treffern.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Suchergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooSearchResponseDto?> SearchAsync(
string query,
int quotesCount = 10,
int newsCount = 0,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft Fundamentaldaten und Unternehmens-Metadaten für ein bestimmtes Symbol über den quoteSummary-Endpunkt ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL", "MSFT").</param>
/// <param name="modules">Die abzufragenden Yahoo-Module (z. B. "assetProfile", "financialData").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Die Abfrageergebnisse als DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
IEnumerable<string> modules,
CancellationToken cancellationToken = default);
/// <summary>
/// Hilfsmethode zum Abrufen aller vordefinierten Standard-Module für ein Tickersymbol.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das vollständige QuoteSummary-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(
string symbol,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft historische Chart- und Kursdaten (OHLCV) für ein Symbol ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="range">Der Abfragezeitraum (z. B. "1d", "1m", "1y", "5y").</param>
/// <param name="interval">Das Intervall der Datenpunkte (z. B. "1m", "5m", "1d", "1wk").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Chart-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
string range = "1y",
string interval = "1d",
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft schnelle Realtime-Preise für eine Liste von Tickersymbolen ab.
/// </summary>
/// <param name="symbols">Eine Liste von Tickersymbolen (z. B. <c>["AAPL", "MSFT", "^GSPC"]</c>).</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Quote-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
CancellationToken cancellationToken = default);
/// <summary>
/// Bequeme Hilfsmethode, um den aktuellen regulären Marktpreis für ein einzelnes Tickersymbol abzufragen.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "^VIX", "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Preis als <see cref="decimal"/> oder <c>null</c>, wenn kein Preis ermittelt werden konnte.</returns>
Task<decimal?> GetLivePriceAsync(
string symbol,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
/// </summary>
public class YahooFinanceClient
{
private const string DefaultUserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
private readonly HttpClient _httpClient;
private readonly CookieContainer _cookieContainer;
private readonly ILogger<YahooFinanceClient>? _logger;
private readonly SemaphoreSlim _authLock = new(1, 1);
private string? _crumb;
private DateTime _lastAuthTime = DateTime.MinValue;
/// <summary>
/// Standard modules available for the quoteSummary endpoint.
/// </summary>
public static readonly string[] StandardQuoteSummaryModules = new[]
{
"assetProfile",
"financialData",
"defaultKeyStatistics",
"summaryDetail",
"incomeStatementHistory",
"incomeStatementHistoryQuarterly",
"balanceSheetHistory",
"balanceSheetHistoryQuarterly",
"cashflowStatementHistory",
"cashflowStatementHistoryQuarterly",
"calendarEvents"
};
public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
{
_logger = logger;
_cookieContainer = new CookieContainer();
if (httpClient != null)
{
_httpClient = httpClient;
}
else
{
var handler = new HttpClientHandler
{
CookieContainer = _cookieContainer,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
_httpClient = new HttpClient(handler);
}
if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
{
_httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
}
}
/// <summary>
/// Executes the Cookie (A3) &amp; Crumb token authentication flow.
/// 1. GET https://fc.yahoo.com (sets session A3 cookie)
/// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
/// </summary>
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
CancellationToken cancellationToken = default)
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
await _authLock.WaitAsync(cancellationToken);
try
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
(DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
_logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");
// 1. Send GET request to fc.yahoo.com to obtain session cookie A3
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
{
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
// CookieContainer automatically intercepts and stores 'A3' cookie
}
// 2. Send GET request to getcrumb to obtain the dynamic crumb token
using (var crumbRequest =
new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
{
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (!crumbResponse.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
crumbResponse.StatusCode);
return null;
}
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
_logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
return _crumb;
}
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
return null;
}
finally
{
_authLock.Release();
}
}
/// <summary>
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
/// Note: Does not require Cookie/Crumb authentication.
/// </summary>
public async Task<YahooSearchResponseDto?> SearchAsync(
string query,
int quotesCount = 10,
int newsCount = 0,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(query)) return null;
try
{
var url =
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
response.StatusCode);
return null;
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
return null;
}
}
/// <summary>
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&amp;modules={modules}
/// </summary>
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
IEnumerable<string> modules,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbol)) return null;
var moduleList = string.Join(",", modules);
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken);
}
/// <summary>
/// Convenience method to fetch all standard quoteSummary modules for a given symbol.
/// </summary>
public Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(string symbol,
CancellationToken cancellationToken = default)
{
return GetQuoteSummaryAsync(symbol, StandardQuoteSummaryModules, cancellationToken);
}
/// <summary>
/// Retrieves historical OHLCV chart data for a given symbol.
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&amp;interval={interval}&amp;crumb={crumb}
/// </summary>
public async Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
string range = "1y",
string interval = "1d",
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbol)) return null;
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken);
}
/// <summary>
/// Retrieves quick real-time price quotes for one or more symbols.
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&amp;crumb={crumb}
/// </summary>
public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
CancellationToken cancellationToken = default)
{
var symbolList = symbols.Where(s => !string.IsNullOrWhiteSpace(s)).ToList();
if (symbolList.Count == 0) return null;
var symbolsParam = string.Join(",", symbolList);
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
return (false, dto);
}, cancellationToken);
}
/// <summary>
/// Convenient helper method to fetch the current live price for a single symbol (e.g., "^VIX").
/// </summary>
public async Task<decimal?> GetLivePriceAsync(string symbol, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbol)) return null;
var quotes = await GetQuotesAsync(new[] { symbol }, cancellationToken);
var item = quotes?.QuoteResponse?.Result?.FirstOrDefault();
if (item?.RegularMarketPrice.HasValue == true && item.RegularMarketPrice.Value > 0)
{
return Convert.ToDecimal(item.RegularMarketPrice.Value);
}
return null;
}
private async Task<T?> ExecuteWithRetryAsync<T>(
Func<string, Task<(bool isAuthError, T? result)>> action,
CancellationToken cancellationToken) where T : class
{
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (isAuthError, result) = await action(crumb);
if (!isAuthError && result != null)
{
return result;
}
if (isAuthError)
{
_logger?.LogInformation(
"[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (_, retryResult) = await action(crumb);
return retryResult;
}
return result;
}
private static JsonSerializerOptions GetJsonOptions()
{
return new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
};
}
}
@@ -0,0 +1,519 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Playwright;
namespace FinlyticCore.Clients;
public interface IYahooFinanceHtmlClient
{
Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
string isinOrSymbol,
CancellationToken cancellationToken = default);
}
public interface IYahooFinanceHtmlClient<TDbContext> : IYahooFinanceHtmlClient
where TDbContext : DbContext
{
}
public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHtmlClient<TDbContext>
where TDbContext : DbContext
{
private readonly IPlaywrightExecutionService _playwrightService;
private readonly IFinlyticLogger<TContextClass, TDbContext> _finlyticLogger;
private readonly string _serviceName;
public YahooFinanceHtmlClient(
IPlaywrightExecutionService playwrightService,
IFinlyticLogger<TContextClass, TDbContext> finlyticLogger)
{
_playwrightService = playwrightService;
_finlyticLogger = finlyticLogger;
_serviceName = typeof(TContextClass).Name;
}
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
string isinOrSymbol,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;
var symbol = isinOrSymbol.Trim().ToUpperInvariant();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'...");
return await _playwrightService.ExecuteInContextAsync(async context =>
{
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var analysisData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
ProfileExtractionResult? profileResult = null;
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/";
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken);
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken);
try
{
await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask);
keyStatsData = await statsTask;
profileResult = await profileTask;
financialsData = await financialsTask;
analysisData = await analysisTask;
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}.");
}
var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}");
return BuildModulesDto(
keyStatsData,
profileDict,
financialsData,
analysisData,
officers,
profileResult?.Sector,
profileResult?.Industry,
profileResult?.Employees,
profileResult?.Description);
}, PlaywrightBrowserFactory.GetDefaultContextOptions(), cancellationToken);
}
private async Task<Dictionary<string, string>> ScrapePagePairsAsync(
IBrowserContext context,
string url,
CancellationToken cancellationToken)
{
var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var page = await context.NewPageAsync();
try
{
await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 20_000
});
await HandleConsentAsync(page);
var extracted = await page.EvaluateAsync<Dictionary<string, string>>(@"() => {
const results = {};
const cleanKey = (str) => {
return str.toLowerCase()
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
.replace(/\s*\d+\s*$/, '')
.replace(/\s+/g, ' ')
.trim();
};
document.querySelectorAll('table tr').forEach(tr => {
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
if (cells.length >= 2 && cells[0] && cells[1]) {
const key = cleanKey(cells[0]);
const val = cells[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-') {
results[key] = val;
}
}
});
return results;
}");
if (extracted != null)
{
foreach (var (k, v) in extracted)
{
targetDict[k] = v;
}
}
}
catch (Exception ex)
{
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}");
}
finally
{
await page.CloseAsync();
}
return targetDict;
}
private async Task<ProfileExtractionResult> ScrapeProfilePageAsync(
IBrowserContext context,
string url,
CancellationToken cancellationToken)
{
var profileDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var companyOfficers = new List<YahooCompanyOfficerDto>();
string? sector = null;
string? industry = null;
int? fullTimeEmployees = null;
string? description = null;
var page = await context.NewPageAsync();
try
{
await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 20_000
});
await HandleConsentAsync(page);
var metaInfo = await page.EvaluateAsync<ProfileMetaJsResult>(@"() => {
let sector = null, industry = null, employees = null, description = null;
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary');
if (descEl) description = descEl.innerText.trim();
const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
if (profileSec) {
const text = profileSec.innerText;
const sectorMatch = text.match(/Sector\(s\)\s*:?\s*([^\n\r]+)/i) || text.match(/Sector\s*:?\s*([^\n\r]+)/i);
if (sectorMatch) sector = sectorMatch[1].trim();
const indMatch = text.match(/Industry\s*:?\s*([^\n\r]+)/i);
if (indMatch) industry = indMatch[1].trim();
const empMatch = text.match(/Full Time Employees\s*:?\s*([\d,]+)/i);
if (empMatch) {
const cleanNum = empMatch[1].replace(/,/g, '');
employees = parseInt(cleanNum, 10);
}
}
const officers = [];
const officerRows = document.querySelectorAll('section[data-testid=""asset-profile""] table tr, table.officers tr, main table tr');
officerRows.forEach((tr, index) => {
if (index === 0) return;
const tds = Array.from(tr.querySelectorAll('td')).map(td => td.innerText.trim());
if (tds.length >= 2) {
officers.push({
name: tds[0] || null,
title: tds[1] || null,
pay: tds[2] || null,
exercised: tds[3] || null,
yearBorn: tds[4] ? parseInt(tds[4], 10) : null
});
}
});
return { sector, industry, employees, description, officers };
}");
if (metaInfo != null)
{
sector = metaInfo.Sector;
industry = metaInfo.Industry;
fullTimeEmployees = metaInfo.Employees;
description = metaInfo.Description;
if (metaInfo.Officers != null)
{
foreach (var off in metaInfo.Officers)
{
if (!string.IsNullOrWhiteSpace(off.Name))
{
companyOfficers.Add(new YahooCompanyOfficerDto(
Name: off.Name,
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
Title: off.Title,
YearBorn: off.YearBorn,
FiscalYear: null,
TotalPay: ParseYahooValue(off.Pay),
ExercisedValue: ParseYahooValue(off.Exercised),
UnexercisedValue: null
));
}
}
}
}
}
catch (Exception ex)
{
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}");
}
finally
{
await page.CloseAsync();
}
return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
}
private static async Task HandleConsentAsync(IPage page)
{
try
{
if (page.Url.Contains("consent.yahoo.com"))
{
var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button");
if (await consentBtn.CountAsync() > 0)
{
await consentBtn.First.ClickAsync();
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 });
}
}
}
catch { /* Fallback */ }
}
private YahooQuoteSummaryModulesDto BuildModulesDto(
Dictionary<string, string> keyStatsData,
Dictionary<string, string> profileData,
Dictionary<string, string> financialsData,
Dictionary<string, string> analysisData,
List<YahooCompanyOfficerDto> companyOfficers,
string? sector,
string? industry,
int? fullTimeEmployees,
string? description)
{
var allStats = new Dictionary<string, string>(keyStatsData, StringComparer.OrdinalIgnoreCase);
foreach (var (k, v) in profileData) allStats[k] = v;
foreach (var (k, v) in financialsData) allStats[k] = v;
foreach (var (k, v) in analysisData) allStats[k] = v;
var assetProfile = new YahooAssetProfileDto(
Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
Industry: industry ?? GetString(allStats, "industry"),
IndustryKey: null, IndustryDisp: null,
Sector: sector ?? GetString(allStats, "sector"),
SectorKey: null, SectorDisp: null,
LongBusinessSummary: description,
FullTimeEmployees: fullTimeEmployees,
CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
AuditRisk: null, BoardRisk: null, CompensationRisk: null, ShareHolderRightsRisk: null, OverallRisk: null,
GovernanceEpochDate: null, CompensationAsOfEpochDate: null
);
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
PriceToBook: GetVal(allStats, "price/book", "price / book"),
EnterpriseValue: GetVal(allStats, "enterprise value"),
ForwardPE: GetVal(allStats, "forward p/e"),
ProfitMargins: GetVal(allStats, "profit margin"),
FloatShares: GetVal(allStats, "float"),
SharesOutstanding: GetVal(allStats, "shares outstanding"),
SharesShort: GetVal(allStats, "shares short"),
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
SharesShortPreviousMonthDate: null, DateShortInterest: null,
SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"),
HeldPercentInsiders: GetVal(allStats, "% held by insiders"),
HeldPercentInstitutions: GetVal(allStats, "% held by institutions"),
ShortRatio: GetVal(allStats, "short ratio"),
ShortPercentOfFloat: GetVal(allStats, "short % of float"),
Beta: GetVal(allStats, "beta (5y monthly)", "beta"),
Category: null,
BookValue: GetVal(allStats, "book value per share", "book value"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"),
LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"),
NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"),
MostRecentQuarter: GetVal(allStats, "most recent quarter"),
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"),
NetIncomeToCommon: GetVal(allStats, "net income avi to common"),
TrailingEps: GetVal(allStats, "diluted eps"),
ForwardEps: GetVal(allStats, "forward eps"),
PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"),
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"),
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"),
FiftyTwoWeekChange: GetVal(allStats, "52-week change"),
SandP52WeekChange: GetVal(allStats, "s&p500 52-week change")
);
var financialData = new YahooFinancialDataDto(
CurrentPrice: GetVal(allStats, "current price", "price"),
TargetHighPrice: GetVal(allStats, "target high", "high target"),
TargetLowPrice: GetVal(allStats, "target low", "low target"),
TargetMeanPrice: GetVal(allStats, "target mean", "target est"),
TargetMedianPrice: GetVal(allStats, "target median"),
RecommendationMean: GetVal(allStats, "recommendation mean"),
RecommendationKey: GetString(allStats, "recommendation key"),
NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"),
TotalCash: GetVal(allStats, "total cash"),
TotalCashPerShare: GetVal(allStats, "total cash per share"),
Ebitda: GetVal(allStats, "ebitda"),
TotalDebt: GetVal(allStats, "total debt"),
QuickRatio: GetVal(allStats, "quick ratio"),
CurrentRatio: GetVal(allStats, "current ratio"),
TotalRevenue: GetVal(allStats, "revenue", "total revenue"),
DebtToEquity: GetVal(allStats, "total debt/equity"),
RevenuePerShare: GetVal(allStats, "revenue per share"),
ReturnOnAssets: GetVal(allStats, "return on assets"),
ReturnOnEquity: GetVal(allStats, "return on equity"),
GrossProfits: GetVal(allStats, "gross profit"),
FreeCashflow: GetVal(allStats, "levered free cash flow"),
OperatingCashflow: GetVal(allStats, "operating cash flow"),
RevenueGrowth: GetVal(allStats, "quarterly revenue growth"),
GrossMargins: GetVal(allStats, "gross margin"),
EbitdaMargins: GetVal(allStats, "ebitda margin"),
OperatingMargins: GetVal(allStats, "operating margin"),
ProfitMargins: GetVal(allStats, "profit margin"),
FinancialCurrency: "USD"
);
var summaryDetail = new YahooSummaryDetailDto(
MaxAge: 86400, PriceHint: null,
PreviousClose: GetVal(allStats, "previous close"),
Open: GetVal(allStats, "open"),
DayLow: GetVal(allStats, "day low"),
DayHigh: GetVal(allStats, "day high"),
RegularMarketPreviousClose: GetVal(allStats, "previous close"),
RegularMarketOpen: GetVal(allStats, "open"),
RegularMarketDayLow: GetVal(allStats, "day low"),
RegularMarketDayHigh: GetVal(allStats, "day high"),
DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"),
DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"),
ExDividendDate: GetVal(allStats, "ex-dividend date"),
PayoutRatio: GetVal(allStats, "payout ratio"),
FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"),
Beta: GetVal(allStats, "beta"),
TrailingPE: GetVal(allStats, "trailing p/e"),
ForwardPE: GetVal(allStats, "forward p/e"),
Volume: GetVal(allStats, "volume"),
RegularMarketVolume: GetVal(allStats, "volume"),
AverageVolume: GetVal(allStats, "avg. volume", "average volume"),
AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"),
AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"),
Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"),
BidSize: null, AskSize: null,
MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"),
FiftyTwoWeekLow: GetVal(allStats, "52 week low"),
FiftyTwoWeekHigh: GetVal(allStats, "52 week high"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"),
Currency: "USD"
);
return new YahooQuoteSummaryModulesDto(
QuoteType: null,
AssetProfile: assetProfile,
FinancialData: financialData,
DefaultKeyStatistics: defaultKeyStatistics,
SummaryDetail: summaryDetail,
IncomeStatementHistory: null,
IncomeStatementHistoryQuarterly: null,
BalanceSheetHistory: null,
BalanceSheetHistoryQuarterly: null,
CashflowStatementHistory: null,
CashflowStatementHistoryQuarterly: null,
CalendarEvents: null
);
}
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] keys)
{
foreach (var k in keys)
{
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
return ParseYahooValue(val);
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match.Value))
return ParseYahooValue(match.Value);
}
return null;
}
private static string? GetString(Dictionary<string, string> dict, params string[] keys)
{
foreach (var k in keys)
{
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
return val.Trim();
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match.Value))
return match.Value.Trim();
}
return null;
}
/// <summary>
/// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln.
/// </summary>
public static YahooValueDto? ParseYahooValue(string? text)
{
if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-")
return null;
var trimmed = text.Trim();
bool isPercent = trimmed.EndsWith("%");
double multiplier = 1.0;
if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0;
else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0;
else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0;
else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0;
// Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt
var numPart = Regex.Replace(trimmed, @"[^\d.-]", "");
if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal))
{
double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier);
return new YahooValueDto
{
Raw = finalVal,
Fmt = trimmed,
LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture)
};
}
return null;
}
private record ProfileExtractionResult(
Dictionary<string, string> ProfileDict,
List<YahooCompanyOfficerDto> Officers,
string? Sector,
string? Industry,
int? Employees,
string? Description
);
private class ProfileMetaJsResult
{
public string? Sector { get; set; }
public string? Industry { get; set; }
public int? Employees { get; set; }
public string? Description { get; set; }
public List<OfficerJsResult>? Officers { get; set; }
}
private class OfficerJsResult
{
public string? Name { get; set; }
public string? Title { get; set; }
public string? Pay { get; set; }
public string? Exercised { get; set; }
public int? YearBorn { get; set; }
}
}