636 lines
31 KiB
C#
636 lines
31 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Globalization;
|
|
using System.Linq;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FinlyticCore.Dtos.Yahoo;
|
|
using FinlyticCore.Models.Settings;
|
|
using FinlyticCore.Services;
|
|
using FinlyticCore.Services.PlaywrightScrapper;
|
|
using Microsoft.Playwright;
|
|
|
|
namespace FinlyticCore.Clients;
|
|
|
|
public interface IYahooFinanceHtmlClient
|
|
{
|
|
Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
|
|
string isinOrSymbol,
|
|
bool includeProfile = true,
|
|
CancellationToken cancellationToken = default);
|
|
}
|
|
|
|
public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
|
{
|
|
private readonly IPlaywrightExecutionService _playwrightService;
|
|
private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
|
|
private const string _serviceName = "YahooFinanceHtmlClient";
|
|
|
|
public YahooFinanceHtmlClient(
|
|
IPlaywrightExecutionService playwrightService,
|
|
IFinlyticLogger<YahooFinanceHtmlClient> finlyticLogger)
|
|
{
|
|
_playwrightService = playwrightService;
|
|
_finlyticLogger = finlyticLogger;
|
|
}
|
|
|
|
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
|
|
string isinOrSymbol,
|
|
bool includeProfile = true,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;
|
|
|
|
var symbol = isinOrSymbol.Trim().ToUpperInvariant();
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Starting parallel fast Playwright HTML DOM scrape for symbol '{symbol}' (IncludeProfile: {includeProfile})...");
|
|
|
|
return await _playwrightService.ExecuteInContextAsync(async context =>
|
|
{
|
|
var summaryUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/";
|
|
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
|
|
var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
|
|
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
|
|
|
|
// 1. Initial Page: Authenticate session and pass Cookie Consent once for the entire context
|
|
var initialPage = await context.NewPageAsync();
|
|
var summaryData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
try
|
|
{
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [1/2] Loading Summary & passing Consent: {summaryUrl}");
|
|
await initialPage.GotoAsync(summaryUrl, new PageGotoOptions
|
|
{
|
|
WaitUntil = WaitUntilState.DOMContentLoaded,
|
|
Timeout = 15_000
|
|
});
|
|
|
|
await HandleConsentAsync(initialPage);
|
|
await WaitForContentAsync(initialPage);
|
|
summaryData = await ExtractKeyValuePairsFromPageAsync(initialPage, summaryUrl);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping initial Summary page for {symbol}.");
|
|
}
|
|
finally
|
|
{
|
|
await initialPage.CloseAsync();
|
|
}
|
|
|
|
// 2. Parallel Sub-Pages (Stats, Financials, and conditionally Profile)
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [2/2] Fetching Sub-Pages in parallel (IncludeProfile: {includeProfile})...");
|
|
|
|
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
|
|
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
|
|
Task<ProfileExtractionResult?> profileTask = includeProfile
|
|
? ScrapeProfilePageAsync(context, profileUrl, cancellationToken)!
|
|
: Task.FromResult<ProfileExtractionResult?>(null);
|
|
|
|
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
ProfileExtractionResult? profileResult = null;
|
|
|
|
try
|
|
{
|
|
await Task.WhenAll(statsTask, financialsTask, profileTask);
|
|
|
|
keyStatsData = await statsTask;
|
|
financialsData = await financialsTask;
|
|
profileResult = await profileTask;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error during parallel sub-page scrape for {symbol}.");
|
|
}
|
|
|
|
// Merge summary data into key stats
|
|
foreach (var (k, v) in summaryData)
|
|
{
|
|
if (!keyStatsData.ContainsKey(k))
|
|
{
|
|
keyStatsData[k] = v;
|
|
}
|
|
}
|
|
|
|
var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();
|
|
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
|
$"[{_serviceName}] Fast HTML DOM scrape complete for '{symbol}'. Summary: {summaryData.Count}, Stats: {keyStatsData.Count}, Financials: {financialsData.Count}, Profile: {profileDict.Count}, Officers: {officers.Count}");
|
|
|
|
return BuildModulesDto(
|
|
keyStatsData,
|
|
profileDict,
|
|
financialsData,
|
|
new Dictionary<string, string>(),
|
|
officers,
|
|
profileResult?.Sector,
|
|
profileResult?.Industry,
|
|
profileResult?.Employees,
|
|
profileResult?.Description);
|
|
|
|
}, PlaywrightBrowserFactory.GetDefaultContextOptions(), cancellationToken);
|
|
}
|
|
|
|
private async Task<Dictionary<string, string>> ScrapePagePairsAsync(
|
|
IBrowserContext context,
|
|
string url,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
var page = await context.NewPageAsync();
|
|
|
|
try
|
|
{
|
|
await page.GotoAsync(url, new PageGotoOptions
|
|
{
|
|
WaitUntil = WaitUntilState.DOMContentLoaded,
|
|
Timeout = 10_000
|
|
});
|
|
|
|
await WaitForContentAsync(page);
|
|
targetDict = await ExtractKeyValuePairsFromPageAsync(page, url);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping URL {url}");
|
|
}
|
|
finally
|
|
{
|
|
await page.CloseAsync();
|
|
}
|
|
|
|
return targetDict;
|
|
}
|
|
|
|
private async Task<ProfileExtractionResult?> ScrapeProfilePageAsync(
|
|
IBrowserContext context,
|
|
string url,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var profileDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
var companyOfficers = new List<YahooCompanyOfficerDto>();
|
|
string? sector = null;
|
|
string? industry = null;
|
|
int? fullTimeEmployees = null;
|
|
string? description = null;
|
|
|
|
var page = await context.NewPageAsync();
|
|
|
|
try
|
|
{
|
|
await page.GotoAsync(url, new PageGotoOptions
|
|
{
|
|
WaitUntil = WaitUntilState.DOMContentLoaded,
|
|
Timeout = 10_000
|
|
});
|
|
|
|
await WaitForContentAsync(page);
|
|
|
|
var jsonStr = await page.EvaluateAsync<string>(@"() => {
|
|
let sector = null, industry = null, employees = null, description = null;
|
|
|
|
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary, section[data-testid=""asset-profile""] p');
|
|
if (descEl) description = descEl.innerText.trim();
|
|
|
|
const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
|
|
if (profileSec) {
|
|
const text = profileSec.innerText;
|
|
const sectorMatch = text.match(/Sector\(s\)\s*:?\s*([^\n\r]+)/i) || text.match(/Sector\s*:?\s*([^\n\r]+)/i);
|
|
if (sectorMatch) sector = sectorMatch[1].trim();
|
|
|
|
const indMatch = text.match(/Industry\s*:?\s*([^\n\r]+)/i);
|
|
if (indMatch) industry = indMatch[1].trim();
|
|
|
|
const empMatch = text.match(/Full Time Employees\s*:?\s*([\d,]+)/i);
|
|
if (empMatch) {
|
|
const cleanNum = empMatch[1].replace(/,/g, '');
|
|
employees = parseInt(cleanNum, 10);
|
|
}
|
|
}
|
|
|
|
const officers = [];
|
|
const officerRows = document.querySelectorAll('section[data-testid=""asset-profile""] table tr, table.officers tr, main table tr');
|
|
officerRows.forEach((tr, index) => {
|
|
if (index === 0) return;
|
|
const tds = Array.from(tr.querySelectorAll('td')).map(td => td.innerText.trim());
|
|
if (tds.length >= 2) {
|
|
officers.push({
|
|
name: tds[0] || null,
|
|
title: tds[1] || null,
|
|
pay: tds[2] || null,
|
|
exercised: tds[3] || null,
|
|
yearBorn: tds[4] ? parseInt(tds[4], 10) : null
|
|
});
|
|
}
|
|
});
|
|
|
|
return JSON.stringify({ sector, industry, employees, description, officers });
|
|
}");
|
|
|
|
if (!string.IsNullOrWhiteSpace(jsonStr))
|
|
{
|
|
var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
|
|
var metaInfo = JsonSerializer.Deserialize<ProfileMetaJsResult>(jsonStr, options);
|
|
|
|
if (metaInfo != null)
|
|
{
|
|
sector = metaInfo.Sector;
|
|
industry = metaInfo.Industry;
|
|
fullTimeEmployees = metaInfo.Employees;
|
|
description = metaInfo.Description;
|
|
|
|
if (metaInfo.Officers != null)
|
|
{
|
|
foreach (var off in metaInfo.Officers)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(off.Name))
|
|
{
|
|
companyOfficers.Add(new YahooCompanyOfficerDto(
|
|
Name: off.Name,
|
|
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
|
|
Title: off.Title,
|
|
YearBorn: off.YearBorn,
|
|
FiscalYear: null,
|
|
TotalPay: ParseYahooValue(off.Pay),
|
|
ExercisedValue: ParseYahooValue(off.Exercised),
|
|
UnexercisedValue: null
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
|
$"[{_serviceName}] HTML Profile extracted -> Sector: '{sector}', Industry: '{industry}', Employees: {fullTimeEmployees}, Officers: {companyOfficers.Count}, Desc length: {description?.Length ?? 0}");
|
|
}
|
|
}
|
|
|
|
profileDict = await ExtractKeyValuePairsFromPageAsync(page, url);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping Profile data from {url}");
|
|
}
|
|
finally
|
|
{
|
|
await page.CloseAsync();
|
|
}
|
|
|
|
return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
|
|
}
|
|
|
|
private static async Task WaitForContentAsync(IPage page)
|
|
{
|
|
try
|
|
{
|
|
await page.WaitForSelectorAsync("table tr, ul li, div[data-testid], section, main", new PageWaitForSelectorOptions
|
|
{
|
|
State = WaitForSelectorState.Attached,
|
|
Timeout = 2_500
|
|
});
|
|
}
|
|
catch { }
|
|
}
|
|
|
|
private async Task<Dictionary<string, string>> ExtractKeyValuePairsFromPageAsync(IPage page, string pageUrl)
|
|
{
|
|
var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
try
|
|
{
|
|
var jsonStr = await page.EvaluateAsync<string>(@"() => {
|
|
const results = {};
|
|
|
|
const cleanKey = (str) => {
|
|
return str.toLowerCase()
|
|
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
|
|
.replace(/\s*\d+\s*$/, '')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
};
|
|
|
|
// 1. Standard HTML Table Rows (Financials, Balance Sheet, Key Stats)
|
|
document.querySelectorAll('table tr').forEach(tr => {
|
|
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
|
|
if (cells.length >= 2 && cells[0] && cells[1]) {
|
|
const key = cleanKey(cells[0]);
|
|
const val = cells[1].replace(/\s+/g, ' ').trim();
|
|
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
|
|
results[key] = val;
|
|
}
|
|
}
|
|
});
|
|
|
|
// 2. Modern Yahoo Finance List / Div pairs (Summary Table, Quote Statistics, Flex containers)
|
|
document.querySelectorAll('li, div[class*=""container""], div[class*=""row""], section div, div[data-testid]').forEach(el => {
|
|
const children = Array.from(el.querySelectorAll(':scope > span, :scope > div, :scope > p')).map(s => s.innerText.trim()).filter(Boolean);
|
|
if (children.length === 2) {
|
|
const key = cleanKey(children[0]);
|
|
const val = children[1].replace(/\s+/g, ' ').trim();
|
|
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
|
|
if (!results[key]) {
|
|
results[key] = val;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
return JSON.stringify(results);
|
|
}");
|
|
|
|
if (!string.IsNullOrWhiteSpace(jsonStr))
|
|
{
|
|
var parsed = JsonSerializer.Deserialize<Dictionary<string, string>>(jsonStr);
|
|
if (parsed != null)
|
|
{
|
|
foreach (var (k, v) in parsed)
|
|
{
|
|
targetDict[k] = v;
|
|
}
|
|
}
|
|
}
|
|
|
|
var sampleKeys = targetDict.Keys.Take(6).ToList();
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
|
$"[{_serviceName}] Extracted {targetDict.Count} items from HTML of '{pageUrl}'. Sample keys: [{string.Join(", ", sampleKeys)}]");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error extracting key-value pairs from HTML of '{pageUrl}'");
|
|
}
|
|
|
|
return targetDict;
|
|
}
|
|
|
|
private async Task HandleConsentAsync(IPage page)
|
|
{
|
|
try
|
|
{
|
|
var url = page.Url;
|
|
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
|
|
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
|
$"[{_serviceName}] Detected EU Cookie Consent redirect: '{url}'. Searching for Accept/Reject buttons...");
|
|
|
|
var selectors = new[]
|
|
{
|
|
"button[name='agree']",
|
|
"button[value='agree']",
|
|
"button.accept-all",
|
|
"button.btn.primary",
|
|
"button.btn.secondary.accept-all",
|
|
"button[name='reject']",
|
|
"button[value='reject']",
|
|
"button.reject-all",
|
|
"form[action*='consent'] button[type='submit']",
|
|
"button:has-text('Alle akzeptieren')",
|
|
"button:has-text('Accept all')",
|
|
"button:has-text('Alle ablehnen')",
|
|
"button:has-text('Reject all')",
|
|
"button:has-text('Akzeptieren')",
|
|
"button:has-text('Ablehnen')",
|
|
"button:has-text('Agree')",
|
|
"button:has-text('I agree')"
|
|
};
|
|
|
|
foreach (var sel in selectors)
|
|
{
|
|
var btn = page.Locator(sel);
|
|
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
|
|
{
|
|
var btnText = await btn.First.InnerTextAsync();
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
|
$"[{_serviceName}] Found consent button '{btnText.Trim()}' with selector '{sel}'. Clicking...");
|
|
|
|
// Fast click and wait for DOMContentLoaded on finance.yahoo.com
|
|
await btn.First.ClickAsync();
|
|
try
|
|
{
|
|
await page.WaitForURLAsync(u => u.Contains("finance.yahoo.com", StringComparison.OrdinalIgnoreCase),
|
|
new PageWaitForURLOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 5_000 });
|
|
}
|
|
catch { }
|
|
|
|
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
|
$"[{_serviceName}] Successfully passed consent wall. Current URL: '{page.Url}'");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex,
|
|
$"[{_serviceName}] Exception in HandleConsentAsync.");
|
|
}
|
|
}
|
|
|
|
private YahooQuoteSummaryModulesDto BuildModulesDto(
|
|
Dictionary<string, string> keyStatsData,
|
|
Dictionary<string, string> profileData,
|
|
Dictionary<string, string> financialsData,
|
|
Dictionary<string, string> analysisData,
|
|
List<YahooCompanyOfficerDto> companyOfficers,
|
|
string? sector,
|
|
string? industry,
|
|
int? fullTimeEmployees,
|
|
string? description)
|
|
{
|
|
var allStats = new Dictionary<string, string>(keyStatsData, StringComparer.OrdinalIgnoreCase);
|
|
foreach (var (k, v) in profileData) allStats[k] = v;
|
|
foreach (var (k, v) in financialsData) allStats[k] = v;
|
|
foreach (var (k, v) in analysisData) allStats[k] = v;
|
|
|
|
var assetProfile = new YahooAssetProfileDto(
|
|
Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
|
|
Industry: industry, IndustryKey: null, IndustryDisp: null,
|
|
Sector: sector, SectorKey: null, SectorDisp: null,
|
|
LongBusinessSummary: description,
|
|
FullTimeEmployees: fullTimeEmployees,
|
|
CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
|
|
AuditRisk: null, BoardRisk: null, CompensationRisk: null, ShareHolderRightsRisk: null, OverallRisk: null,
|
|
GovernanceEpochDate: null, CompensationAsOfEpochDate: null
|
|
);
|
|
|
|
var financialData = new YahooFinancialDataDto(
|
|
CurrentPrice: GetVal(allStats, "previous close", "current price", "price", "regular market price"),
|
|
TargetHighPrice: GetVal(allStats, "target high price", "target high"),
|
|
TargetLowPrice: GetVal(allStats, "target low price", "target low"),
|
|
TargetMeanPrice: GetVal(allStats, "1y target est", "target mean price", "target est"),
|
|
TargetMedianPrice: GetVal(allStats, "target median price"),
|
|
RecommendationMean: GetVal(allStats, "recommendation mean"),
|
|
RecommendationKey: allStats.GetValueOrDefault("recommendation") ?? allStats.GetValueOrDefault("recommendation key"),
|
|
NumberOfAnalystOpinions: GetVal(allStats, "number of analyst opinions", "analyst opinions"),
|
|
TotalCash: GetVal(allStats, "total cash", "total cash (mrq)"),
|
|
TotalCashPerShare: GetVal(allStats, "total cash per share", "total cash per share (mrq)"),
|
|
Ebitda: GetVal(allStats, "ebitda"),
|
|
TotalDebt: GetVal(allStats, "total debt", "total debt (mrq)"),
|
|
QuickRatio: GetVal(allStats, "quick ratio"),
|
|
CurrentRatio: GetVal(allStats, "current ratio", "current ratio (mrq)"),
|
|
TotalRevenue: GetVal(allStats, "total revenue", "revenue", "revenue (ttm)"),
|
|
DebtToEquity: GetVal(allStats, "total debt/equity", "total debt/equity (mrq)", "debt to equity"),
|
|
RevenuePerShare: GetVal(allStats, "revenue per share", "revenue per share (ttm)"),
|
|
ReturnOnAssets: GetVal(allStats, "return on assets", "return on assets (ttm)"),
|
|
ReturnOnEquity: GetVal(allStats, "return on equity", "return on equity (ttm)"),
|
|
GrossProfits: GetVal(allStats, "gross profit", "gross profit (ttm)", "gross profits"),
|
|
FreeCashflow: GetVal(allStats, "levered free cash flow", "levered free cash flow (ttm)", "free cash flow"),
|
|
OperatingCashflow: GetVal(allStats, "operating cash flow", "operating cash flow (ttm)"),
|
|
RevenueGrowth: GetVal(allStats, "quarterly revenue growth", "quarterly revenue growth (yoy)", "revenue growth"),
|
|
GrossMargins: GetVal(allStats, "gross margin", "gross margins"),
|
|
EbitdaMargins: GetVal(allStats, "ebitda margin", "ebitda margins"),
|
|
OperatingMargins: GetVal(allStats, "operating margin", "operating margin (ttm)", "operating margins"),
|
|
ProfitMargins: GetVal(allStats, "profit margin", "profit margins"),
|
|
FinancialCurrency: null
|
|
);
|
|
|
|
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
|
|
PriceToBook: GetVal(allStats, "price/book", "price to book", "kbv", "kurs-buchwert-verhältnis"),
|
|
EnterpriseValue: GetVal(allStats, "enterprise value", "unternehmenswert"),
|
|
ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
|
|
ProfitMargins: GetVal(allStats, "profit margin", "gewinnmarge"),
|
|
FloatShares: GetVal(allStats, "float", "streubesitz"),
|
|
SharesOutstanding: GetVal(allStats, "shares outstanding", "ausstehende aktien"),
|
|
SharesShort: GetVal(allStats, "shares short", "leerverkaufte aktien"),
|
|
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
|
|
SharesShortPreviousMonthDate: null,
|
|
DateShortInterest: null,
|
|
SharesPercentSharesOut: GetVal(allStats, "shares % of shares outstanding", "short % of shares outstanding"),
|
|
HeldPercentInsiders: GetVal(allStats, "% held by insiders", "insider anteil"),
|
|
HeldPercentInstitutions: GetVal(allStats, "% held by institutions", "institutioneller anteil"),
|
|
ShortRatio: GetVal(allStats, "short ratio"),
|
|
ShortPercentOfFloat: GetVal(allStats, "short % of float", "short percent of float"),
|
|
Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
|
|
Category: null,
|
|
BookValue: GetVal(allStats, "book value per share", "book value per share (mrq)", "book value", "buchwert"),
|
|
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price to sales", "kuv"),
|
|
LastFiscalYearEnd: null,
|
|
NextFiscalYearEnd: null,
|
|
MostRecentQuarter: null,
|
|
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth", "quarterly earnings growth (yoy)", "earnings growth"),
|
|
NetIncomeToCommon: GetVal(allStats, "net income avi to common", "net income avi to common (ttm)", "net income avail. to common", "net income"),
|
|
TrailingEps: GetVal(allStats, "diluted eps", "diluted eps (ttm)", "trailing eps", "eps (ttm)", "gewinn je aktie"),
|
|
ForwardEps: GetVal(allStats, "forward eps"),
|
|
PegRatio: GetVal(allStats, "peg ratio (5 yr expected)", "peg ratio (5yr expected)", "peg ratio", "peg-verhältnis"),
|
|
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue", "ev/revenue"),
|
|
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda", "ev/ebitda"),
|
|
FiftyTwoWeekChange: GetVal(allStats, "52 week change", "52-week change", "52-wochen-änderung"),
|
|
SandP52WeekChange: GetVal(allStats, "s&p 500 52-week change", "s&p500 52-week change", "s&p 500 52 week change")
|
|
);
|
|
|
|
var summaryDetail = new YahooSummaryDetailDto(
|
|
MaxAge: null,
|
|
PriceHint: null,
|
|
PreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
|
|
Open: GetVal(allStats, "open", "eröffnung"),
|
|
DayLow: null,
|
|
DayHigh: null,
|
|
RegularMarketPreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
|
|
RegularMarketOpen: GetVal(allStats, "open", "eröffnung"),
|
|
RegularMarketDayLow: null,
|
|
RegularMarketDayHigh: null,
|
|
DividendRate: GetVal(allStats, "forward annual dividend rate", "trailing annual dividend rate", "forward dividend & yield", "dividend rate", "dividende"),
|
|
DividendYield: GetVal(allStats, "forward annual dividend yield", "trailing annual dividend yield", "dividend yield", "dividendenrendite"),
|
|
ExDividendDate: null,
|
|
PayoutRatio: GetVal(allStats, "payout ratio", "ausschüttungsquote"),
|
|
FiveYearAvgDividendYield: GetVal(allStats, "5 year average dividend yield"),
|
|
Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
|
|
TrailingPE: GetVal(allStats, "pe ratio (ttm)", "trailing p/e", "p/e ratio", "pe", "trailing pe", "kgv (ttm)", "kgv"),
|
|
ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
|
|
Volume: GetVal(allStats, "volume", "volumen"),
|
|
RegularMarketVolume: GetVal(allStats, "volume", "volumen"),
|
|
AverageVolume: GetVal(allStats, "avg vol (3 month)", "avg. volume", "average volume", "durchschnittsvolumen"),
|
|
AverageVolume10days: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
|
|
AverageDailyVolume10Day: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
|
|
Bid: GetVal(allStats, "bid", "geld"),
|
|
Ask: GetVal(allStats, "ask", "brief"),
|
|
BidSize: null,
|
|
AskSize: null,
|
|
MarketCap: GetVal(allStats, "market cap", "market capitalization", "market cap (intraday)", "marktkapitalisierung"),
|
|
FiftyTwoWeekLow: GetVal(allStats, "52 week low", "52-week low", "52 wochen tief"),
|
|
FiftyTwoWeekHigh: GetVal(allStats, "52 week high", "52-week high", "52 wochen hoch"),
|
|
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "kuv"),
|
|
Currency: null
|
|
);
|
|
|
|
return new YahooQuoteSummaryModulesDto(
|
|
QuoteType: null,
|
|
AssetProfile: assetProfile,
|
|
FinancialData: financialData,
|
|
DefaultKeyStatistics: defaultKeyStatistics,
|
|
SummaryDetail: summaryDetail,
|
|
IncomeStatementHistory: null,
|
|
IncomeStatementHistoryQuarterly: null,
|
|
BalanceSheetHistory: null,
|
|
BalanceSheetHistoryQuarterly: null,
|
|
CashflowStatementHistory: null,
|
|
CashflowStatementHistoryQuarterly: null,
|
|
CalendarEvents: null
|
|
);
|
|
}
|
|
|
|
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] candidateKeys)
|
|
{
|
|
foreach (var candidate in candidateKeys)
|
|
{
|
|
if (dict.TryGetValue(candidate, out var valStr) && !string.IsNullOrWhiteSpace(valStr))
|
|
{
|
|
var parsed = ParseYahooValue(valStr);
|
|
if (parsed != null) return parsed;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private static YahooValueDto? ParseYahooValue(string? raw)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(raw) || raw == "N/A" || raw == "--" || raw == "-") return null;
|
|
|
|
var clean = raw.Trim().Replace(" ", "").Replace("$", "").Replace("€", "").Replace("£", "");
|
|
var isPercent = clean.EndsWith("%");
|
|
if (isPercent) clean = clean.TrimEnd('%');
|
|
|
|
double multiplier = 1.0;
|
|
if (clean.EndsWith("T", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000_000.0; clean = clean[..^1]; }
|
|
else if (clean.EndsWith("B", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000.0; clean = clean[..^1]; }
|
|
else if (clean.EndsWith("M", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000.0; clean = clean[..^1]; }
|
|
else if (clean.EndsWith("K", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000.0; clean = clean[..^1]; }
|
|
|
|
if (double.TryParse(clean, NumberStyles.Any, CultureInfo.InvariantCulture, out var num))
|
|
{
|
|
var finalRaw = num * multiplier;
|
|
if (isPercent) finalRaw /= 100.0;
|
|
return new YahooValueDto { Raw = finalRaw, Fmt = raw };
|
|
}
|
|
|
|
return new YahooValueDto { Raw = null, Fmt = raw };
|
|
}
|
|
|
|
private class ProfileMetaJsResult
|
|
{
|
|
public string? Sector { get; set; }
|
|
public string? Industry { get; set; }
|
|
public int? Employees { get; set; }
|
|
public string? Description { get; set; }
|
|
public List<JsOfficer>? Officers { get; set; }
|
|
}
|
|
|
|
private class JsOfficer
|
|
{
|
|
public string? Name { get; set; }
|
|
public string? Title { get; set; }
|
|
public string? Pay { get; set; }
|
|
public string? Exercised { get; set; }
|
|
public int? YearBorn { get; set; }
|
|
}
|
|
|
|
private record ProfileExtractionResult(
|
|
Dictionary<string, string> ProfileDict,
|
|
List<YahooCompanyOfficerDto> Officers,
|
|
string? Sector,
|
|
string? Industry,
|
|
int? Employees,
|
|
string? Description);
|
|
} |