feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings
This commit is contained in:
@@ -9,6 +9,8 @@ using System.Threading.Tasks;
|
||||
using FinlyticCore.Dtos.Yahoo;
|
||||
using FinlyticCore.Models.Settings;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticCore.Services.PlaywrightScrapper;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace FinlyticCore.Services.Yahoo;
|
||||
|
||||
@@ -18,9 +20,11 @@ public class YahooFinanceClient
|
||||
private readonly CookieContainer _cookieContainer;
|
||||
private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
|
||||
private readonly ISettingsService? _settingsService;
|
||||
private readonly IPlaywrightExecutionService? _playwrightService;
|
||||
private readonly SemaphoreSlim _authLock = new(1, 1);
|
||||
|
||||
private string? _crumb;
|
||||
private string? _rawCookieHeader;
|
||||
private DateTime _lastAuthTime = DateTime.MinValue;
|
||||
|
||||
/// <summary>
|
||||
@@ -44,10 +48,12 @@ public class YahooFinanceClient
|
||||
public YahooFinanceClient(
|
||||
IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
|
||||
ISettingsService? settingsService = null,
|
||||
IPlaywrightExecutionService? playwrightService = null,
|
||||
HttpClient? httpClient = null)
|
||||
{
|
||||
_finlyticLogger = finlyticLogger;
|
||||
_settingsService = settingsService;
|
||||
_playwrightService = playwrightService;
|
||||
_cookieContainer = new CookieContainer();
|
||||
|
||||
if (httpClient != null)
|
||||
@@ -97,6 +103,7 @@ public class YahooFinanceClient
|
||||
{
|
||||
RestoreCookies(cachedCookies);
|
||||
_crumb = cachedCrumb;
|
||||
_rawCookieHeader = cachedCookies;
|
||||
_lastAuthTime = DateTime.UtcNow;
|
||||
|
||||
if (_finlyticLogger != null)
|
||||
@@ -115,70 +122,40 @@ public class YahooFinanceClient
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");
|
||||
|
||||
// 3. Send GET request to fc.yahoo.com to obtain session cookie A3
|
||||
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
|
||||
try
|
||||
{
|
||||
using var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com");
|
||||
initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
|
||||
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
|
||||
}
|
||||
catch { }
|
||||
|
||||
// 4. Send GET request to getcrumb to obtain dynamic crumb token
|
||||
string[] crumbUrls = new[]
|
||||
var crumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(crumb))
|
||||
{
|
||||
"https://query1.finance.yahoo.com/v1/test/getcrumb",
|
||||
"https://query2.finance.yahoo.com/v1/test/getcrumb"
|
||||
};
|
||||
_crumb = crumb;
|
||||
_lastAuthTime = DateTime.UtcNow;
|
||||
await PersistSessionAsync(_crumb, cancellationToken);
|
||||
return _crumb;
|
||||
}
|
||||
|
||||
foreach (var url in crumbUrls)
|
||||
// 5. FALLBACK: Playwright Browser Authentication (Bypasses EU Consent Wall and 429)
|
||||
if (_playwrightService != null)
|
||||
{
|
||||
try
|
||||
var (browserCrumb, browserCookies) = await AuthenticateViaPlaywrightAsync(cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(browserCrumb))
|
||||
{
|
||||
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
crumbRequest.Headers.Add("Accept", "*/*");
|
||||
|
||||
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
|
||||
if (crumbResponse.IsSuccessStatusCode)
|
||||
_crumb = browserCrumb;
|
||||
_lastAuthTime = DateTime.UtcNow;
|
||||
if (!string.IsNullOrWhiteSpace(browserCookies))
|
||||
{
|
||||
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(crumbText))
|
||||
{
|
||||
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
|
||||
_lastAuthTime = DateTime.UtcNow;
|
||||
|
||||
// Persist to DB cache via SettingsService
|
||||
if (_settingsService != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
var serializedCookies = SerializeCookies();
|
||||
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(serializedCookies))
|
||||
{
|
||||
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (Exception persistEx)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database.");
|
||||
}
|
||||
}
|
||||
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb);
|
||||
return _crumb;
|
||||
}
|
||||
RestoreCookies(browserCookies);
|
||||
_rawCookieHeader = browserCookies;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Fallthrough to next endpoint
|
||||
await PersistSessionAsync(_crumb, cancellationToken);
|
||||
return _crumb;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,13 +175,204 @@ public class YahooFinanceClient
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string?> FetchCrumbWithHttpClientAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
string[] crumbUrls = new[]
|
||||
{
|
||||
"https://query1.finance.yahoo.com/v1/test/getcrumb",
|
||||
"https://query2.finance.yahoo.com/v1/test/getcrumb"
|
||||
};
|
||||
|
||||
foreach (var url in crumbUrls)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
crumbRequest.Headers.Add("Accept", "*/*");
|
||||
crumbRequest.Headers.Add("Origin", "https://finance.yahoo.com");
|
||||
crumbRequest.Headers.Add("Referer", "https://finance.yahoo.com/");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
|
||||
{
|
||||
crumbRequest.Headers.Add("Cookie", _rawCookieHeader);
|
||||
}
|
||||
|
||||
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
|
||||
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
|
||||
|
||||
if (crumbResponse.IsSuccessStatusCode && !string.IsNullOrWhiteSpace(crumbText))
|
||||
{
|
||||
var cleanCrumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
|
||||
if (!cleanCrumb.Contains("<html", StringComparison.OrdinalIgnoreCase) && cleanCrumb.Length < 100)
|
||||
{
|
||||
return cleanCrumb;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
|
||||
"[YahooFinanceClient] Endpoint '{Url}' returned status {Status}. Response Body: {Body}",
|
||||
url, crumbResponse.StatusCode, crumbText);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during FetchCrumb on '{Url}'", url);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<(string? crumb, string? cookieStr)> AuthenticateViaPlaywrightAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_playwrightService == null) return (null, null);
|
||||
|
||||
try
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Launching Playwright browser to acquire valid EU Yahoo session and Crumb...");
|
||||
|
||||
return await _playwrightService.ExecuteInContextAsync(async context =>
|
||||
{
|
||||
var page = await context.NewPageAsync();
|
||||
try
|
||||
{
|
||||
await page.GotoAsync("https://finance.yahoo.com/quote/AAPL/", new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 30_000
|
||||
});
|
||||
|
||||
// Handle EU Consent if redirected
|
||||
var url = page.Url;
|
||||
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
|
||||
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var selectors = new[]
|
||||
{
|
||||
"button[name='agree']",
|
||||
"button[value='agree']",
|
||||
"button.accept-all",
|
||||
"button.btn.primary",
|
||||
"button.btn.secondary.accept-all",
|
||||
"form[action*='consent'] button[type='submit']",
|
||||
"button:has-text('Alle akzeptieren')",
|
||||
"button:has-text('Accept all')",
|
||||
"button:has-text('Akzeptieren')",
|
||||
"button:has-text('Agree')"
|
||||
};
|
||||
|
||||
foreach (var sel in selectors)
|
||||
{
|
||||
var btn = page.Locator(sel);
|
||||
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
|
||||
{
|
||||
await btn.First.ClickAsync();
|
||||
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15_000 });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Extract cookies from BrowserContext
|
||||
var cookies = await context.CookiesAsync();
|
||||
var pairs = new List<string>();
|
||||
foreach (var c in cookies)
|
||||
{
|
||||
if (c.Domain.Contains("yahoo.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
pairs.Add($"{c.Name}={c.Value}");
|
||||
}
|
||||
}
|
||||
var extractedCookies = string.Join(";", pairs);
|
||||
|
||||
// 2. Fetch Crumb from inside the authenticated page
|
||||
string? extractedCrumb = null;
|
||||
try
|
||||
{
|
||||
extractedCrumb = await page.EvaluateAsync<string>(@"async () => {
|
||||
try {
|
||||
const res = await fetch('/v1/test/getcrumb');
|
||||
if (res.ok) {
|
||||
return await res.text();
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}");
|
||||
}
|
||||
catch { }
|
||||
|
||||
// 3. If in-page fetch was empty, use the extracted cookies with HttpClient
|
||||
if (string.IsNullOrWhiteSpace(extractedCrumb) && !string.IsNullOrWhiteSpace(extractedCookies))
|
||||
{
|
||||
RestoreCookies(extractedCookies);
|
||||
_rawCookieHeader = extractedCookies;
|
||||
extractedCrumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(extractedCrumb))
|
||||
{
|
||||
extractedCrumb = extractedCrumb.Trim('"', ' ', '\t', '\r', '\n');
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Playwright successfully acquired Yahoo Crumb token: {Crumb}", extractedCrumb);
|
||||
}
|
||||
|
||||
return (extractedCrumb, extractedCookies);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await page.CloseAsync();
|
||||
}
|
||||
}, cancellationToken: cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Playwright-based authentication failed.");
|
||||
return (null, null);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PersistSessionAsync(string crumb, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_settingsService == null || string.IsNullOrWhiteSpace(crumb)) return;
|
||||
|
||||
try
|
||||
{
|
||||
var serializedCookies = SerializeCookies();
|
||||
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, crumb, cancellationToken);
|
||||
if (!string.IsNullOrWhiteSpace(serializedCookies))
|
||||
{
|
||||
_rawCookieHeader = serializedCookies;
|
||||
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
|
||||
}
|
||||
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Persisted valid Yahoo session & crumb to database.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Failed to persist Yahoo session to database.");
|
||||
}
|
||||
}
|
||||
|
||||
private string SerializeCookies()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
|
||||
{
|
||||
return _rawCookieHeader;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var cookies = _cookieContainer.GetAllCookies();
|
||||
var pairs = new List<string>();
|
||||
foreach (Cookie cookie in cookies)
|
||||
foreach (System.Net.Cookie cookie in cookies)
|
||||
{
|
||||
pairs.Add($"{cookie.Name}={cookie.Value}");
|
||||
}
|
||||
@@ -220,9 +388,20 @@ public class YahooFinanceClient
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(serializedCookies)) return;
|
||||
|
||||
_rawCookieHeader = serializedCookies;
|
||||
|
||||
try
|
||||
{
|
||||
var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
var uris = new[]
|
||||
{
|
||||
new Uri("https://yahoo.com"),
|
||||
new Uri("https://finance.yahoo.com"),
|
||||
new Uri("https://query1.finance.yahoo.com"),
|
||||
new Uri("https://query2.finance.yahoo.com"),
|
||||
new Uri("https://fc.yahoo.com")
|
||||
};
|
||||
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var eqIdx = part.IndexOf('=');
|
||||
@@ -230,7 +409,14 @@ public class YahooFinanceClient
|
||||
{
|
||||
var name = part.Substring(0, eqIdx).Trim();
|
||||
var val = part.Substring(eqIdx + 1).Trim();
|
||||
_cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com"));
|
||||
foreach (var uri in uris)
|
||||
{
|
||||
try
|
||||
{
|
||||
_cookieContainer.Add(uri, new System.Net.Cookie(name, val));
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -242,8 +428,6 @@ public class YahooFinanceClient
|
||||
|
||||
/// <summary>
|
||||
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
|
||||
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&quotesCount={quotesCount}&newsCount={newsCount}
|
||||
/// Note: Does not require Cookie/Crumb authentication.
|
||||
/// </summary>
|
||||
public async Task<YahooSearchResponseDto?> SearchAsync(
|
||||
string query,
|
||||
@@ -253,33 +437,55 @@ public class YahooFinanceClient
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query)) return null;
|
||||
|
||||
try
|
||||
string[] endpoints = new[]
|
||||
{
|
||||
var url =
|
||||
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}",
|
||||
$"https://query1.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}"esCount={quotesCount}&newsCount={newsCount}"
|
||||
};
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
foreach (var url in endpoints)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
request.Headers.Add("Accept", "*/*");
|
||||
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
|
||||
{
|
||||
request.Headers.Add("Cookie", _rawCookieHeader);
|
||||
}
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken);
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
|
||||
if (dto != null)
|
||||
{
|
||||
return dto;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
|
||||
"[YahooFinanceClient] Search endpoint '{Url}' for '{Query}' returned status {Status}. Response Body: {Body}",
|
||||
url, query, response.StatusCode, json);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode);
|
||||
return null;
|
||||
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search endpoint '{Url}'", url);
|
||||
}
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
|
||||
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&modules={modules}
|
||||
/// </summary>
|
||||
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
|
||||
string symbol,
|
||||
@@ -291,22 +497,49 @@ public class YahooFinanceClient
|
||||
var moduleList = string.Join(",", modules);
|
||||
return await ExecuteWithRetryAsync(async (crumb) =>
|
||||
{
|
||||
var url =
|
||||
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
string[] baseUrls = new[]
|
||||
{
|
||||
"https://query2.finance.yahoo.com/v10/finance/quoteSummary",
|
||||
"https://query1.finance.yahoo.com/v10/finance/quoteSummary"
|
||||
};
|
||||
|
||||
foreach (var baseUrl in baseUrls)
|
||||
{
|
||||
var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
request.Headers.Add("Accept", "*/*");
|
||||
request.Headers.Add("Origin", "https://finance.yahoo.com");
|
||||
request.Headers.Add("Referer", "https://finance.yahoo.com/");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
|
||||
{
|
||||
request.Headers.Add("Cookie", _rawCookieHeader);
|
||||
}
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken);
|
||||
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(responseContent, GetJsonOptions());
|
||||
return (false, dto);
|
||||
}
|
||||
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
|
||||
return (
|
||||
response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden, null);
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
|
||||
"[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}",
|
||||
symbol, response.StatusCode, url, responseContent);
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden ||
|
||||
response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
return (true, null);
|
||||
}
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
|
||||
return (false, dto);
|
||||
return (false, null);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -321,7 +554,6 @@ public class YahooFinanceClient
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves historical OHLCV chart data for a given symbol.
|
||||
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&interval={interval}&crumb={crumb}
|
||||
/// </summary>
|
||||
public async Task<YahooChartResponseDto?> GetChartAsync(
|
||||
string symbol,
|
||||
@@ -333,28 +565,54 @@ public class YahooFinanceClient
|
||||
|
||||
return await ExecuteWithRetryAsync(async (crumb) =>
|
||||
{
|
||||
var url =
|
||||
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
string[] baseUrls = new[]
|
||||
{
|
||||
"https://query1.finance.yahoo.com/v8/finance/chart",
|
||||
"https://query2.finance.yahoo.com/v8/finance/chart"
|
||||
};
|
||||
|
||||
foreach (var baseUrl in baseUrls)
|
||||
{
|
||||
var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
request.Headers.Add("Accept", "*/*");
|
||||
request.Headers.Add("Origin", "https://finance.yahoo.com");
|
||||
request.Headers.Add("Referer", "https://finance.yahoo.com/");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
|
||||
{
|
||||
request.Headers.Add("Cookie", _rawCookieHeader);
|
||||
}
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken);
|
||||
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(responseContent, GetJsonOptions());
|
||||
return (false, dto);
|
||||
}
|
||||
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
|
||||
return (
|
||||
response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden, null);
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
|
||||
"[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}",
|
||||
symbol, response.StatusCode, url, responseContent);
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden ||
|
||||
response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
return (true, null);
|
||||
}
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
|
||||
return (false, dto);
|
||||
return (false, null);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves quick real-time price quotes for one or more symbols.
|
||||
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&crumb={crumb}
|
||||
/// </summary>
|
||||
public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
|
||||
IEnumerable<string> symbols,
|
||||
@@ -366,22 +624,49 @@ public class YahooFinanceClient
|
||||
var symbolsParam = string.Join(",", symbolList);
|
||||
return await ExecuteWithRetryAsync(async (crumb) =>
|
||||
{
|
||||
var url =
|
||||
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
|
||||
using var response = await _httpClient.GetAsync(url, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
string[] baseUrls = new[]
|
||||
{
|
||||
"https://query1.finance.yahoo.com/v7/finance/quote",
|
||||
"https://query2.finance.yahoo.com/v7/finance/quote"
|
||||
};
|
||||
|
||||
foreach (var baseUrl in baseUrls)
|
||||
{
|
||||
var url = $"{baseUrl}?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
|
||||
request.Headers.Add("Accept", "*/*");
|
||||
request.Headers.Add("Origin", "https://finance.yahoo.com");
|
||||
request.Headers.Add("Referer", "https://finance.yahoo.com/");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
|
||||
{
|
||||
request.Headers.Add("Cookie", _rawCookieHeader);
|
||||
}
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken);
|
||||
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(responseContent, GetJsonOptions());
|
||||
return (false, dto);
|
||||
}
|
||||
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
|
||||
return (
|
||||
response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden, null);
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
|
||||
"[YahooFinanceClient] GetQuotes failed with status {Status}. URL: {Url}. Response Body: {Body}",
|
||||
response.StatusCode, url, responseContent);
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.Unauthorized ||
|
||||
response.StatusCode == HttpStatusCode.Forbidden ||
|
||||
response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
return (true, null);
|
||||
}
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
|
||||
return (false, dto);
|
||||
return (false, null);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -408,26 +693,29 @@ public class YahooFinanceClient
|
||||
CancellationToken cancellationToken) where T : class
|
||||
{
|
||||
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
|
||||
if (!string.IsNullOrEmpty(crumb))
|
||||
{
|
||||
var (isAuthError, result) = await action(crumb);
|
||||
if (!isAuthError && result != null)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
if (!isAuthError)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-authenticate when auth error (401/403/429) or empty crumb occurs
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error or invalid crumb encountered (401/403/429). Force re-authenticating...");
|
||||
|
||||
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
|
||||
if (string.IsNullOrEmpty(crumb)) return null;
|
||||
|
||||
var (isAuthError, result) = await action(crumb);
|
||||
if (!isAuthError && result != null)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
if (isAuthError)
|
||||
{
|
||||
if (_finlyticLogger != null)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
|
||||
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
|
||||
if (string.IsNullOrEmpty(crumb)) return null;
|
||||
|
||||
var (_, retryResult) = await action(crumb);
|
||||
return retryResult;
|
||||
}
|
||||
|
||||
return result;
|
||||
var (_, retryResult) = await action(crumb);
|
||||
return retryResult;
|
||||
}
|
||||
|
||||
private static JsonSerializerOptions GetJsonOptions()
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
@@ -17,14 +18,15 @@ public interface IYahooFinanceHtmlClient
|
||||
{
|
||||
Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
|
||||
string isinOrSymbol,
|
||||
bool includeProfile = true,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
{
|
||||
private const string _serviceName = nameof(YahooFinanceHtmlClient);
|
||||
private readonly IPlaywrightExecutionService _playwrightService;
|
||||
private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
|
||||
private const string _serviceName = "YahooFinanceHtmlClient";
|
||||
|
||||
public YahooFinanceHtmlClient(
|
||||
IPlaywrightExecutionService playwrightService,
|
||||
@@ -36,55 +38,93 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
|
||||
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
|
||||
string isinOrSymbol,
|
||||
bool includeProfile = true,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;
|
||||
|
||||
var symbol = isinOrSymbol.Trim().ToUpperInvariant();
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'...");
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Starting parallel fast Playwright HTML DOM scrape for symbol '{symbol}' (IncludeProfile: {includeProfile})...");
|
||||
|
||||
return await _playwrightService.ExecuteInContextAsync(async context =>
|
||||
{
|
||||
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var analysisData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
ProfileExtractionResult? profileResult = null;
|
||||
|
||||
var summaryUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/";
|
||||
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
|
||||
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
|
||||
var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
|
||||
var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/";
|
||||
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
|
||||
|
||||
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
|
||||
var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken);
|
||||
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
|
||||
var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken);
|
||||
// 1. Initial Page: Authenticate session and pass Cookie Consent once for the entire context
|
||||
var initialPage = await context.NewPageAsync();
|
||||
var summaryData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask);
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [1/2] Loading Summary & passing Consent: {summaryUrl}");
|
||||
await initialPage.GotoAsync(summaryUrl, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 15_000
|
||||
});
|
||||
|
||||
keyStatsData = await statsTask;
|
||||
profileResult = await profileTask;
|
||||
financialsData = await financialsTask;
|
||||
analysisData = await analysisTask;
|
||||
await HandleConsentAsync(initialPage);
|
||||
await WaitForContentAsync(initialPage);
|
||||
summaryData = await ExtractKeyValuePairsFromPageAsync(initialPage, summaryUrl);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}.");
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping initial Summary page for {symbol}.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
await initialPage.CloseAsync();
|
||||
}
|
||||
|
||||
// 2. Parallel Sub-Pages (Stats, Financials, and conditionally Profile)
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [2/2] Fetching Sub-Pages in parallel (IncludeProfile: {includeProfile})...");
|
||||
|
||||
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
|
||||
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
|
||||
Task<ProfileExtractionResult?> profileTask = includeProfile
|
||||
? ScrapeProfilePageAsync(context, profileUrl, cancellationToken)!
|
||||
: Task.FromResult<ProfileExtractionResult?>(null);
|
||||
|
||||
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
ProfileExtractionResult? profileResult = null;
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(statsTask, financialsTask, profileTask);
|
||||
|
||||
keyStatsData = await statsTask;
|
||||
financialsData = await financialsTask;
|
||||
profileResult = await profileTask;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error during parallel sub-page scrape for {symbol}.");
|
||||
}
|
||||
|
||||
// Merge summary data into key stats
|
||||
foreach (var (k, v) in summaryData)
|
||||
{
|
||||
if (!keyStatsData.ContainsKey(k))
|
||||
{
|
||||
keyStatsData[k] = v;
|
||||
}
|
||||
}
|
||||
|
||||
var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}");
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
||||
$"[{_serviceName}] Fast HTML DOM scrape complete for '{symbol}'. Summary: {summaryData.Count}, Stats: {keyStatsData.Count}, Financials: {financialsData.Count}, Profile: {profileDict.Count}, Officers: {officers.Count}");
|
||||
|
||||
return BuildModulesDto(
|
||||
keyStatsData,
|
||||
profileDict,
|
||||
financialsData,
|
||||
analysisData,
|
||||
new Dictionary<string, string>(),
|
||||
officers,
|
||||
profileResult?.Sector,
|
||||
profileResult?.Industry,
|
||||
@@ -107,47 +147,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
await page.GotoAsync(url, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 20_000
|
||||
Timeout = 10_000
|
||||
});
|
||||
|
||||
await HandleConsentAsync(page);
|
||||
|
||||
var extracted = await page.EvaluateAsync<Dictionary<string, string>>(@"() => {
|
||||
const results = {};
|
||||
|
||||
const cleanKey = (str) => {
|
||||
return str.toLowerCase()
|
||||
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
|
||||
.replace(/\s*\d+\s*$/, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
};
|
||||
|
||||
document.querySelectorAll('table tr').forEach(tr => {
|
||||
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
|
||||
if (cells.length >= 2 && cells[0] && cells[1]) {
|
||||
const key = cleanKey(cells[0]);
|
||||
const val = cells[1].replace(/\s+/g, ' ').trim();
|
||||
if (key && val && val !== 'N/A' && val !== '--' && val !== '-') {
|
||||
results[key] = val;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
}");
|
||||
|
||||
if (extracted != null)
|
||||
{
|
||||
foreach (var (k, v) in extracted)
|
||||
{
|
||||
targetDict[k] = v;
|
||||
}
|
||||
}
|
||||
await WaitForContentAsync(page);
|
||||
targetDict = await ExtractKeyValuePairsFromPageAsync(page, url);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}");
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping URL {url}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -157,7 +165,7 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
return targetDict;
|
||||
}
|
||||
|
||||
private async Task<ProfileExtractionResult> ScrapeProfilePageAsync(
|
||||
private async Task<ProfileExtractionResult?> ScrapeProfilePageAsync(
|
||||
IBrowserContext context,
|
||||
string url,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -176,15 +184,15 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
await page.GotoAsync(url, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 20_000
|
||||
Timeout = 10_000
|
||||
});
|
||||
|
||||
await HandleConsentAsync(page);
|
||||
await WaitForContentAsync(page);
|
||||
|
||||
var metaInfo = await page.EvaluateAsync<ProfileMetaJsResult>(@"() => {
|
||||
var jsonStr = await page.EvaluateAsync<string>(@"() => {
|
||||
let sector = null, industry = null, employees = null, description = null;
|
||||
|
||||
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary');
|
||||
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary, section[data-testid=""asset-profile""] p');
|
||||
if (descEl) description = descEl.innerText.trim();
|
||||
|
||||
const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
|
||||
@@ -219,40 +227,51 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
}
|
||||
});
|
||||
|
||||
return { sector, industry, employees, description, officers };
|
||||
return JSON.stringify({ sector, industry, employees, description, officers });
|
||||
}");
|
||||
|
||||
if (metaInfo != null)
|
||||
if (!string.IsNullOrWhiteSpace(jsonStr))
|
||||
{
|
||||
sector = metaInfo.Sector;
|
||||
industry = metaInfo.Industry;
|
||||
fullTimeEmployees = metaInfo.Employees;
|
||||
description = metaInfo.Description;
|
||||
var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
|
||||
var metaInfo = JsonSerializer.Deserialize<ProfileMetaJsResult>(jsonStr, options);
|
||||
|
||||
if (metaInfo.Officers != null)
|
||||
if (metaInfo != null)
|
||||
{
|
||||
foreach (var off in metaInfo.Officers)
|
||||
sector = metaInfo.Sector;
|
||||
industry = metaInfo.Industry;
|
||||
fullTimeEmployees = metaInfo.Employees;
|
||||
description = metaInfo.Description;
|
||||
|
||||
if (metaInfo.Officers != null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(off.Name))
|
||||
foreach (var off in metaInfo.Officers)
|
||||
{
|
||||
companyOfficers.Add(new YahooCompanyOfficerDto(
|
||||
Name: off.Name,
|
||||
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
|
||||
Title: off.Title,
|
||||
YearBorn: off.YearBorn,
|
||||
FiscalYear: null,
|
||||
TotalPay: ParseYahooValue(off.Pay),
|
||||
ExercisedValue: ParseYahooValue(off.Exercised),
|
||||
UnexercisedValue: null
|
||||
));
|
||||
if (!string.IsNullOrWhiteSpace(off.Name))
|
||||
{
|
||||
companyOfficers.Add(new YahooCompanyOfficerDto(
|
||||
Name: off.Name,
|
||||
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
|
||||
Title: off.Title,
|
||||
YearBorn: off.YearBorn,
|
||||
FiscalYear: null,
|
||||
TotalPay: ParseYahooValue(off.Pay),
|
||||
ExercisedValue: ParseYahooValue(off.Exercised),
|
||||
UnexercisedValue: null
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
||||
$"[{_serviceName}] HTML Profile extracted -> Sector: '{sector}', Industry: '{industry}', Employees: {fullTimeEmployees}, Officers: {companyOfficers.Count}, Desc length: {description?.Length ?? 0}");
|
||||
}
|
||||
}
|
||||
|
||||
profileDict = await ExtractKeyValuePairsFromPageAsync(page, url);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}");
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping Profile data from {url}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -262,21 +281,151 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
|
||||
}
|
||||
|
||||
private static async Task HandleConsentAsync(IPage page)
|
||||
private static async Task WaitForContentAsync(IPage page)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (page.Url.Contains("consent.yahoo.com"))
|
||||
await page.WaitForSelectorAsync("table tr, ul li, div[data-testid], section, main", new PageWaitForSelectorOptions
|
||||
{
|
||||
var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button");
|
||||
if (await consentBtn.CountAsync() > 0)
|
||||
State = WaitForSelectorState.Attached,
|
||||
Timeout = 2_500
|
||||
});
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
private async Task<Dictionary<string, string>> ExtractKeyValuePairsFromPageAsync(IPage page, string pageUrl)
|
||||
{
|
||||
var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
try
|
||||
{
|
||||
var jsonStr = await page.EvaluateAsync<string>(@"() => {
|
||||
const results = {};
|
||||
|
||||
const cleanKey = (str) => {
|
||||
return str.toLowerCase()
|
||||
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
|
||||
.replace(/\s*\d+\s*$/, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
};
|
||||
|
||||
// 1. Standard HTML Table Rows (Financials, Balance Sheet, Key Stats)
|
||||
document.querySelectorAll('table tr').forEach(tr => {
|
||||
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
|
||||
if (cells.length >= 2 && cells[0] && cells[1]) {
|
||||
const key = cleanKey(cells[0]);
|
||||
const val = cells[1].replace(/\s+/g, ' ').trim();
|
||||
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
|
||||
results[key] = val;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 2. Modern Yahoo Finance List / Div pairs (Summary Table, Quote Statistics, Flex containers)
|
||||
document.querySelectorAll('li, div[class*=""container""], div[class*=""row""], section div, div[data-testid]').forEach(el => {
|
||||
const children = Array.from(el.querySelectorAll(':scope > span, :scope > div, :scope > p')).map(s => s.innerText.trim()).filter(Boolean);
|
||||
if (children.length === 2) {
|
||||
const key = cleanKey(children[0]);
|
||||
const val = children[1].replace(/\s+/g, ' ').trim();
|
||||
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
|
||||
if (!results[key]) {
|
||||
results[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return JSON.stringify(results);
|
||||
}");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(jsonStr))
|
||||
{
|
||||
var parsed = JsonSerializer.Deserialize<Dictionary<string, string>>(jsonStr);
|
||||
if (parsed != null)
|
||||
{
|
||||
await consentBtn.First.ClickAsync();
|
||||
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 });
|
||||
foreach (var (k, v) in parsed)
|
||||
{
|
||||
targetDict[k] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sampleKeys = targetDict.Keys.Take(6).ToList();
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
||||
$"[{_serviceName}] Extracted {targetDict.Count} items from HTML of '{pageUrl}'. Sample keys: [{string.Join(", ", sampleKeys)}]");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error extracting key-value pairs from HTML of '{pageUrl}'");
|
||||
}
|
||||
|
||||
return targetDict;
|
||||
}
|
||||
|
||||
private async Task HandleConsentAsync(IPage page)
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = page.Url;
|
||||
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
|
||||
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
||||
$"[{_serviceName}] Detected EU Cookie Consent redirect: '{url}'. Searching for Accept/Reject buttons...");
|
||||
|
||||
var selectors = new[]
|
||||
{
|
||||
"button[name='agree']",
|
||||
"button[value='agree']",
|
||||
"button.accept-all",
|
||||
"button.btn.primary",
|
||||
"button.btn.secondary.accept-all",
|
||||
"button[name='reject']",
|
||||
"button[value='reject']",
|
||||
"button.reject-all",
|
||||
"form[action*='consent'] button[type='submit']",
|
||||
"button:has-text('Alle akzeptieren')",
|
||||
"button:has-text('Accept all')",
|
||||
"button:has-text('Alle ablehnen')",
|
||||
"button:has-text('Reject all')",
|
||||
"button:has-text('Akzeptieren')",
|
||||
"button:has-text('Ablehnen')",
|
||||
"button:has-text('Agree')",
|
||||
"button:has-text('I agree')"
|
||||
};
|
||||
|
||||
foreach (var sel in selectors)
|
||||
{
|
||||
var btn = page.Locator(sel);
|
||||
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
|
||||
{
|
||||
var btnText = await btn.First.InnerTextAsync();
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
||||
$"[{_serviceName}] Found consent button '{btnText.Trim()}' with selector '{sel}'. Clicking...");
|
||||
|
||||
// Fast click and wait for DOMContentLoaded on finance.yahoo.com
|
||||
await btn.First.ClickAsync();
|
||||
try
|
||||
{
|
||||
await page.WaitForURLAsync(u => u.Contains("finance.yahoo.com", StringComparison.OrdinalIgnoreCase),
|
||||
new PageWaitForURLOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 5_000 });
|
||||
}
|
||||
catch { }
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
|
||||
$"[{_serviceName}] Successfully passed consent wall. Current URL: '{page.Url}'");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { /* Fallback */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex,
|
||||
$"[{_serviceName}] Exception in HandleConsentAsync.");
|
||||
}
|
||||
}
|
||||
|
||||
private YahooQuoteSummaryModulesDto BuildModulesDto(
|
||||
@@ -297,10 +446,8 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
|
||||
var assetProfile = new YahooAssetProfileDto(
|
||||
Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
|
||||
Industry: industry ?? GetString(allStats, "industry"),
|
||||
IndustryKey: null, IndustryDisp: null,
|
||||
Sector: sector ?? GetString(allStats, "sector"),
|
||||
SectorKey: null, SectorDisp: null,
|
||||
Industry: industry, IndustryKey: null, IndustryDisp: null,
|
||||
Sector: sector, SectorKey: null, SectorDisp: null,
|
||||
LongBusinessSummary: description,
|
||||
FullTimeEmployees: fullTimeEmployees,
|
||||
CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
|
||||
@@ -308,100 +455,104 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
GovernanceEpochDate: null, CompensationAsOfEpochDate: null
|
||||
);
|
||||
|
||||
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
|
||||
PriceToBook: GetVal(allStats, "price/book", "price / book"),
|
||||
EnterpriseValue: GetVal(allStats, "enterprise value"),
|
||||
ForwardPE: GetVal(allStats, "forward p/e"),
|
||||
ProfitMargins: GetVal(allStats, "profit margin"),
|
||||
FloatShares: GetVal(allStats, "float"),
|
||||
SharesOutstanding: GetVal(allStats, "shares outstanding"),
|
||||
SharesShort: GetVal(allStats, "shares short"),
|
||||
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
|
||||
SharesShortPreviousMonthDate: null, DateShortInterest: null,
|
||||
SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"),
|
||||
HeldPercentInsiders: GetVal(allStats, "% held by insiders"),
|
||||
HeldPercentInstitutions: GetVal(allStats, "% held by institutions"),
|
||||
ShortRatio: GetVal(allStats, "short ratio"),
|
||||
ShortPercentOfFloat: GetVal(allStats, "short % of float"),
|
||||
Beta: GetVal(allStats, "beta (5y monthly)", "beta"),
|
||||
Category: null,
|
||||
BookValue: GetVal(allStats, "book value per share", "book value"),
|
||||
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"),
|
||||
LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"),
|
||||
NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"),
|
||||
MostRecentQuarter: GetVal(allStats, "most recent quarter"),
|
||||
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"),
|
||||
NetIncomeToCommon: GetVal(allStats, "net income avi to common"),
|
||||
TrailingEps: GetVal(allStats, "diluted eps"),
|
||||
ForwardEps: GetVal(allStats, "forward eps"),
|
||||
PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"),
|
||||
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"),
|
||||
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"),
|
||||
FiftyTwoWeekChange: GetVal(allStats, "52-week change"),
|
||||
SandP52WeekChange: GetVal(allStats, "s&p500 52-week change")
|
||||
var financialData = new YahooFinancialDataDto(
|
||||
CurrentPrice: GetVal(allStats, "previous close", "current price", "price", "regular market price"),
|
||||
TargetHighPrice: GetVal(allStats, "target high price", "target high"),
|
||||
TargetLowPrice: GetVal(allStats, "target low price", "target low"),
|
||||
TargetMeanPrice: GetVal(allStats, "1y target est", "target mean price", "target est"),
|
||||
TargetMedianPrice: GetVal(allStats, "target median price"),
|
||||
RecommendationMean: GetVal(allStats, "recommendation mean"),
|
||||
RecommendationKey: allStats.GetValueOrDefault("recommendation") ?? allStats.GetValueOrDefault("recommendation key"),
|
||||
NumberOfAnalystOpinions: GetVal(allStats, "number of analyst opinions", "analyst opinions"),
|
||||
TotalCash: GetVal(allStats, "total cash", "total cash (mrq)"),
|
||||
TotalCashPerShare: GetVal(allStats, "total cash per share", "total cash per share (mrq)"),
|
||||
Ebitda: GetVal(allStats, "ebitda"),
|
||||
TotalDebt: GetVal(allStats, "total debt", "total debt (mrq)"),
|
||||
QuickRatio: GetVal(allStats, "quick ratio"),
|
||||
CurrentRatio: GetVal(allStats, "current ratio", "current ratio (mrq)"),
|
||||
TotalRevenue: GetVal(allStats, "total revenue", "revenue", "revenue (ttm)"),
|
||||
DebtToEquity: GetVal(allStats, "total debt/equity", "total debt/equity (mrq)", "debt to equity"),
|
||||
RevenuePerShare: GetVal(allStats, "revenue per share", "revenue per share (ttm)"),
|
||||
ReturnOnAssets: GetVal(allStats, "return on assets", "return on assets (ttm)"),
|
||||
ReturnOnEquity: GetVal(allStats, "return on equity", "return on equity (ttm)"),
|
||||
GrossProfits: GetVal(allStats, "gross profit", "gross profit (ttm)", "gross profits"),
|
||||
FreeCashflow: GetVal(allStats, "levered free cash flow", "levered free cash flow (ttm)", "free cash flow"),
|
||||
OperatingCashflow: GetVal(allStats, "operating cash flow", "operating cash flow (ttm)"),
|
||||
RevenueGrowth: GetVal(allStats, "quarterly revenue growth", "quarterly revenue growth (yoy)", "revenue growth"),
|
||||
GrossMargins: GetVal(allStats, "gross margin", "gross margins"),
|
||||
EbitdaMargins: GetVal(allStats, "ebitda margin", "ebitda margins"),
|
||||
OperatingMargins: GetVal(allStats, "operating margin", "operating margin (ttm)", "operating margins"),
|
||||
ProfitMargins: GetVal(allStats, "profit margin", "profit margins"),
|
||||
FinancialCurrency: null
|
||||
);
|
||||
|
||||
var financialData = new YahooFinancialDataDto(
|
||||
CurrentPrice: GetVal(allStats, "current price", "price"),
|
||||
TargetHighPrice: GetVal(allStats, "target high", "high target"),
|
||||
TargetLowPrice: GetVal(allStats, "target low", "low target"),
|
||||
TargetMeanPrice: GetVal(allStats, "target mean", "target est"),
|
||||
TargetMedianPrice: GetVal(allStats, "target median"),
|
||||
RecommendationMean: GetVal(allStats, "recommendation mean"),
|
||||
RecommendationKey: GetString(allStats, "recommendation key"),
|
||||
NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"),
|
||||
TotalCash: GetVal(allStats, "total cash"),
|
||||
TotalCashPerShare: GetVal(allStats, "total cash per share"),
|
||||
Ebitda: GetVal(allStats, "ebitda"),
|
||||
TotalDebt: GetVal(allStats, "total debt"),
|
||||
QuickRatio: GetVal(allStats, "quick ratio"),
|
||||
CurrentRatio: GetVal(allStats, "current ratio"),
|
||||
TotalRevenue: GetVal(allStats, "revenue", "total revenue"),
|
||||
DebtToEquity: GetVal(allStats, "total debt/equity"),
|
||||
RevenuePerShare: GetVal(allStats, "revenue per share"),
|
||||
ReturnOnAssets: GetVal(allStats, "return on assets"),
|
||||
ReturnOnEquity: GetVal(allStats, "return on equity"),
|
||||
GrossProfits: GetVal(allStats, "gross profit"),
|
||||
FreeCashflow: GetVal(allStats, "levered free cash flow"),
|
||||
OperatingCashflow: GetVal(allStats, "operating cash flow"),
|
||||
RevenueGrowth: GetVal(allStats, "quarterly revenue growth"),
|
||||
GrossMargins: GetVal(allStats, "gross margin"),
|
||||
EbitdaMargins: GetVal(allStats, "ebitda margin"),
|
||||
OperatingMargins: GetVal(allStats, "operating margin"),
|
||||
ProfitMargins: GetVal(allStats, "profit margin"),
|
||||
FinancialCurrency: "USD"
|
||||
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
|
||||
PriceToBook: GetVal(allStats, "price/book", "price to book", "kbv", "kurs-buchwert-verhältnis"),
|
||||
EnterpriseValue: GetVal(allStats, "enterprise value", "unternehmenswert"),
|
||||
ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
|
||||
ProfitMargins: GetVal(allStats, "profit margin", "gewinnmarge"),
|
||||
FloatShares: GetVal(allStats, "float", "streubesitz"),
|
||||
SharesOutstanding: GetVal(allStats, "shares outstanding", "ausstehende aktien"),
|
||||
SharesShort: GetVal(allStats, "shares short", "leerverkaufte aktien"),
|
||||
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
|
||||
SharesShortPreviousMonthDate: null,
|
||||
DateShortInterest: null,
|
||||
SharesPercentSharesOut: GetVal(allStats, "shares % of shares outstanding", "short % of shares outstanding"),
|
||||
HeldPercentInsiders: GetVal(allStats, "% held by insiders", "insider anteil"),
|
||||
HeldPercentInstitutions: GetVal(allStats, "% held by institutions", "institutioneller anteil"),
|
||||
ShortRatio: GetVal(allStats, "short ratio"),
|
||||
ShortPercentOfFloat: GetVal(allStats, "short % of float", "short percent of float"),
|
||||
Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
|
||||
Category: null,
|
||||
BookValue: GetVal(allStats, "book value per share", "book value per share (mrq)", "book value", "buchwert"),
|
||||
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price to sales", "kuv"),
|
||||
LastFiscalYearEnd: null,
|
||||
NextFiscalYearEnd: null,
|
||||
MostRecentQuarter: null,
|
||||
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth", "quarterly earnings growth (yoy)", "earnings growth"),
|
||||
NetIncomeToCommon: GetVal(allStats, "net income avi to common", "net income avi to common (ttm)", "net income avail. to common", "net income"),
|
||||
TrailingEps: GetVal(allStats, "diluted eps", "diluted eps (ttm)", "trailing eps", "eps (ttm)", "gewinn je aktie"),
|
||||
ForwardEps: GetVal(allStats, "forward eps"),
|
||||
PegRatio: GetVal(allStats, "peg ratio (5 yr expected)", "peg ratio (5yr expected)", "peg ratio", "peg-verhältnis"),
|
||||
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue", "ev/revenue"),
|
||||
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda", "ev/ebitda"),
|
||||
FiftyTwoWeekChange: GetVal(allStats, "52 week change", "52-week change", "52-wochen-änderung"),
|
||||
SandP52WeekChange: GetVal(allStats, "s&p 500 52-week change", "s&p500 52-week change", "s&p 500 52 week change")
|
||||
);
|
||||
|
||||
var summaryDetail = new YahooSummaryDetailDto(
|
||||
MaxAge: 86400, PriceHint: null,
|
||||
PreviousClose: GetVal(allStats, "previous close"),
|
||||
Open: GetVal(allStats, "open"),
|
||||
DayLow: GetVal(allStats, "day low"),
|
||||
DayHigh: GetVal(allStats, "day high"),
|
||||
RegularMarketPreviousClose: GetVal(allStats, "previous close"),
|
||||
RegularMarketOpen: GetVal(allStats, "open"),
|
||||
RegularMarketDayLow: GetVal(allStats, "day low"),
|
||||
RegularMarketDayHigh: GetVal(allStats, "day high"),
|
||||
DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"),
|
||||
DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"),
|
||||
ExDividendDate: GetVal(allStats, "ex-dividend date"),
|
||||
PayoutRatio: GetVal(allStats, "payout ratio"),
|
||||
FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"),
|
||||
Beta: GetVal(allStats, "beta"),
|
||||
TrailingPE: GetVal(allStats, "trailing p/e"),
|
||||
ForwardPE: GetVal(allStats, "forward p/e"),
|
||||
Volume: GetVal(allStats, "volume"),
|
||||
RegularMarketVolume: GetVal(allStats, "volume"),
|
||||
AverageVolume: GetVal(allStats, "avg. volume", "average volume"),
|
||||
AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"),
|
||||
AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"),
|
||||
Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"),
|
||||
BidSize: null, AskSize: null,
|
||||
MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"),
|
||||
FiftyTwoWeekLow: GetVal(allStats, "52 week low"),
|
||||
FiftyTwoWeekHigh: GetVal(allStats, "52 week high"),
|
||||
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"),
|
||||
Currency: "USD"
|
||||
MaxAge: null,
|
||||
PriceHint: null,
|
||||
PreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
|
||||
Open: GetVal(allStats, "open", "eröffnung"),
|
||||
DayLow: null,
|
||||
DayHigh: null,
|
||||
RegularMarketPreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
|
||||
RegularMarketOpen: GetVal(allStats, "open", "eröffnung"),
|
||||
RegularMarketDayLow: null,
|
||||
RegularMarketDayHigh: null,
|
||||
DividendRate: GetVal(allStats, "forward annual dividend rate", "trailing annual dividend rate", "forward dividend & yield", "dividend rate", "dividende"),
|
||||
DividendYield: GetVal(allStats, "forward annual dividend yield", "trailing annual dividend yield", "dividend yield", "dividendenrendite"),
|
||||
ExDividendDate: null,
|
||||
PayoutRatio: GetVal(allStats, "payout ratio", "ausschüttungsquote"),
|
||||
FiveYearAvgDividendYield: GetVal(allStats, "5 year average dividend yield"),
|
||||
Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
|
||||
TrailingPE: GetVal(allStats, "pe ratio (ttm)", "trailing p/e", "p/e ratio", "pe", "trailing pe", "kgv (ttm)", "kgv"),
|
||||
ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
|
||||
Volume: GetVal(allStats, "volume", "volumen"),
|
||||
RegularMarketVolume: GetVal(allStats, "volume", "volumen"),
|
||||
AverageVolume: GetVal(allStats, "avg vol (3 month)", "avg. volume", "average volume", "durchschnittsvolumen"),
|
||||
AverageVolume10days: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
|
||||
AverageDailyVolume10Day: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
|
||||
Bid: GetVal(allStats, "bid", "geld"),
|
||||
Ask: GetVal(allStats, "ask", "brief"),
|
||||
BidSize: null,
|
||||
AskSize: null,
|
||||
MarketCap: GetVal(allStats, "market cap", "market capitalization", "market cap (intraday)", "marktkapitalisierung"),
|
||||
FiftyTwoWeekLow: GetVal(allStats, "52 week low", "52-week low", "52 wochen tief"),
|
||||
FiftyTwoWeekHigh: GetVal(allStats, "52 week high", "52-week high", "52 wochen hoch"),
|
||||
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "kuv"),
|
||||
Currency: null
|
||||
);
|
||||
|
||||
return new YahooQuoteSummaryModulesDto(
|
||||
@@ -420,66 +571,59 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
);
|
||||
}
|
||||
|
||||
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] keys)
|
||||
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] candidateKeys)
|
||||
{
|
||||
foreach (var k in keys)
|
||||
foreach (var candidate in candidateKeys)
|
||||
{
|
||||
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
|
||||
return ParseYahooValue(val);
|
||||
|
||||
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
|
||||
if (!string.IsNullOrWhiteSpace(match.Value))
|
||||
return ParseYahooValue(match.Value);
|
||||
if (dict.TryGetValue(candidate, out var valStr) && !string.IsNullOrWhiteSpace(valStr))
|
||||
{
|
||||
var parsed = ParseYahooValue(valStr);
|
||||
if (parsed != null) return parsed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? GetString(Dictionary<string, string> dict, params string[] keys)
|
||||
private static YahooValueDto? ParseYahooValue(string? raw)
|
||||
{
|
||||
foreach (var k in keys)
|
||||
{
|
||||
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
|
||||
return val.Trim();
|
||||
if (string.IsNullOrWhiteSpace(raw) || raw == "N/A" || raw == "--" || raw == "-") return null;
|
||||
|
||||
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
|
||||
if (!string.IsNullOrWhiteSpace(match.Value))
|
||||
return match.Value.Trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln.
|
||||
/// </summary>
|
||||
public static YahooValueDto? ParseYahooValue(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-")
|
||||
return null;
|
||||
|
||||
var trimmed = text.Trim();
|
||||
bool isPercent = trimmed.EndsWith("%");
|
||||
var clean = raw.Trim().Replace(" ", "").Replace("$", "").Replace("€", "").Replace("£", "");
|
||||
var isPercent = clean.EndsWith("%");
|
||||
if (isPercent) clean = clean.TrimEnd('%');
|
||||
|
||||
double multiplier = 1.0;
|
||||
if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0;
|
||||
else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0;
|
||||
else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0;
|
||||
else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0;
|
||||
if (clean.EndsWith("T", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000_000.0; clean = clean[..^1]; }
|
||||
else if (clean.EndsWith("B", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000.0; clean = clean[..^1]; }
|
||||
else if (clean.EndsWith("M", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000.0; clean = clean[..^1]; }
|
||||
else if (clean.EndsWith("K", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000.0; clean = clean[..^1]; }
|
||||
|
||||
// Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt
|
||||
var numPart = Regex.Replace(trimmed, @"[^\d.-]", "");
|
||||
|
||||
if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal))
|
||||
if (double.TryParse(clean, NumberStyles.Any, CultureInfo.InvariantCulture, out var num))
|
||||
{
|
||||
double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier);
|
||||
return new YahooValueDto
|
||||
{
|
||||
Raw = finalVal,
|
||||
Fmt = trimmed,
|
||||
LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture)
|
||||
};
|
||||
var finalRaw = num * multiplier;
|
||||
if (isPercent) finalRaw /= 100.0;
|
||||
return new YahooValueDto { Raw = finalRaw, Fmt = raw };
|
||||
}
|
||||
|
||||
return null;
|
||||
return new YahooValueDto { Raw = null, Fmt = raw };
|
||||
}
|
||||
|
||||
private class ProfileMetaJsResult
|
||||
{
|
||||
public string? Sector { get; set; }
|
||||
public string? Industry { get; set; }
|
||||
public int? Employees { get; set; }
|
||||
public string? Description { get; set; }
|
||||
public List<JsOfficer>? Officers { get; set; }
|
||||
}
|
||||
|
||||
private class JsOfficer
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
public string? Title { get; set; }
|
||||
public string? Pay { get; set; }
|
||||
public string? Exercised { get; set; }
|
||||
public int? YearBorn { get; set; }
|
||||
}
|
||||
|
||||
private record ProfileExtractionResult(
|
||||
@@ -488,24 +632,5 @@ public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
|
||||
string? Sector,
|
||||
string? Industry,
|
||||
int? Employees,
|
||||
string? Description
|
||||
);
|
||||
|
||||
private class ProfileMetaJsResult
|
||||
{
|
||||
public string? Sector { get; set; }
|
||||
public string? Industry { get; set; }
|
||||
public int? Employees { get; set; }
|
||||
public string? Description { get; set; }
|
||||
public List<OfficerJsResult>? Officers { get; set; }
|
||||
}
|
||||
|
||||
private class OfficerJsResult
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
public string? Title { get; set; }
|
||||
public string? Pay { get; set; }
|
||||
public string? Exercised { get; set; }
|
||||
public int? YearBorn { get; set; }
|
||||
}
|
||||
string? Description);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace FinlyticCore.Dtos.Fundamentals;
|
||||
|
||||
@@ -18,7 +18,7 @@ public record AssetHeaderDto
|
||||
public string Description { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("primaryTicker")]
|
||||
public TickerInfoDto PrimaryTicker { get; init; }
|
||||
public TickerInfoDto PrimaryTicker { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("availableTickers")]
|
||||
public List<TickerInfoDto> AvailableTickers { get; init; } = [];
|
||||
|
||||
@@ -16,7 +16,7 @@ public record CorporateEventDto
|
||||
public string? Isin { get; init; }
|
||||
|
||||
[JsonPropertyName("ticker")]
|
||||
public TickerInfoDto Ticker { get; init; }
|
||||
public TickerInfoDto Ticker { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("companyName")]
|
||||
public string? CompanyName { get; init; }
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace FinlyticCore.Dtos.Fundamentals;
|
||||
public record FundamentalDataDto
|
||||
{
|
||||
[JsonPropertyName("ticker")]
|
||||
public TickerInfoDto Ticker { get; init; }
|
||||
public TickerInfoDto Ticker { get; init; } = new();
|
||||
|
||||
// --- Valuation & Multiples ---
|
||||
[JsonPropertyName("marketCap")]
|
||||
|
||||
@@ -32,6 +32,7 @@ public interface IYahooFinanceScraper
|
||||
Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
|
||||
string symbolOrIsin,
|
||||
bool forceHtmlScrape = false,
|
||||
bool includeProfile = true,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -98,7 +99,11 @@ public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
|
||||
$"[{_serviceName}] Crypto search failed for {cryptoSubtitle}");
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel,
|
||||
$"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}");
|
||||
@@ -131,7 +136,7 @@ public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind
|
||||
// 2. Falls Ticker gefunden, mit Unternehmensname noch mehr internationale Exchangeticker suchen (z.B. APC.DE)
|
||||
if (validQuotes.Count > 0)
|
||||
{
|
||||
var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName;
|
||||
@@ -155,16 +160,24 @@ public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
$"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'");
|
||||
}
|
||||
|
||||
return symbols
|
||||
var result = symbols
|
||||
.OrderBy(s => s.priority)
|
||||
.Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange })
|
||||
.Select(s => new TickerInfoDto
|
||||
{
|
||||
Ticker = s.symbol,
|
||||
Exchange = !string.IsNullOrWhiteSpace(s.exchange) ? s.exchange : "Unknown"
|
||||
})
|
||||
.DistinctBy(s => s.Ticker, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
|
||||
string symbolOrIsin,
|
||||
bool forceHtmlScrape = false,
|
||||
bool includeProfile = true,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null;
|
||||
@@ -220,9 +233,9 @@ public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
try
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
|
||||
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}'...");
|
||||
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}' (IncludeProfile: {includeProfile})...");
|
||||
|
||||
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken);
|
||||
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, includeProfile, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -246,16 +259,20 @@ public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
modules.DefaultKeyStatistics != null;
|
||||
}
|
||||
|
||||
private static YahooQuoteSummaryModulesDto MergeModules(
|
||||
YahooQuoteSummaryModulesDto primary,
|
||||
YahooQuoteSummaryModulesDto secondary)
|
||||
public static YahooQuoteSummaryModulesDto? MergeModules(
|
||||
YahooQuoteSummaryModulesDto? primary,
|
||||
YahooQuoteSummaryModulesDto? secondary)
|
||||
{
|
||||
if (primary == null && secondary == null) return null;
|
||||
if (primary == null) return secondary;
|
||||
if (secondary == null) return primary;
|
||||
|
||||
return new YahooQuoteSummaryModulesDto(
|
||||
QuoteType: primary.QuoteType ?? secondary.QuoteType,
|
||||
AssetProfile: primary.AssetProfile ?? secondary.AssetProfile,
|
||||
FinancialData: primary.FinancialData ?? secondary.FinancialData,
|
||||
DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics,
|
||||
SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail,
|
||||
FinancialData: MergeFinancialData(primary.FinancialData, secondary.FinancialData),
|
||||
DefaultKeyStatistics: MergeDefaultKeyStatistics(primary.DefaultKeyStatistics, secondary.DefaultKeyStatistics),
|
||||
SummaryDetail: MergeSummaryDetail(primary.SummaryDetail, secondary.SummaryDetail),
|
||||
IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory,
|
||||
IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly,
|
||||
BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory,
|
||||
@@ -266,6 +283,124 @@ public class YahooFinanceScraper : IYahooFinanceScraper
|
||||
);
|
||||
}
|
||||
|
||||
private static YahooFinancialDataDto? MergeFinancialData(YahooFinancialDataDto? a, YahooFinancialDataDto? b)
|
||||
{
|
||||
if (a == null) return b;
|
||||
if (b == null) return a;
|
||||
|
||||
return new YahooFinancialDataDto(
|
||||
CurrentPrice: a.CurrentPrice ?? b.CurrentPrice,
|
||||
TargetHighPrice: a.TargetHighPrice ?? b.TargetHighPrice,
|
||||
TargetLowPrice: a.TargetLowPrice ?? b.TargetLowPrice,
|
||||
TargetMeanPrice: a.TargetMeanPrice ?? b.TargetMeanPrice,
|
||||
TargetMedianPrice: a.TargetMedianPrice ?? b.TargetMedianPrice,
|
||||
RecommendationMean: a.RecommendationMean ?? b.RecommendationMean,
|
||||
RecommendationKey: !string.IsNullOrWhiteSpace(a.RecommendationKey) && a.RecommendationKey != "none" ? a.RecommendationKey : b.RecommendationKey,
|
||||
NumberOfAnalystOpinions: a.NumberOfAnalystOpinions ?? b.NumberOfAnalystOpinions,
|
||||
TotalCash: a.TotalCash ?? b.TotalCash,
|
||||
TotalCashPerShare: a.TotalCashPerShare ?? b.TotalCashPerShare,
|
||||
Ebitda: a.Ebitda ?? b.Ebitda,
|
||||
TotalDebt: a.TotalDebt ?? b.TotalDebt,
|
||||
QuickRatio: a.QuickRatio ?? b.QuickRatio,
|
||||
CurrentRatio: a.CurrentRatio ?? b.CurrentRatio,
|
||||
TotalRevenue: a.TotalRevenue ?? b.TotalRevenue,
|
||||
DebtToEquity: a.DebtToEquity ?? b.DebtToEquity,
|
||||
RevenuePerShare: a.RevenuePerShare ?? b.RevenuePerShare,
|
||||
ReturnOnAssets: a.ReturnOnAssets ?? b.ReturnOnAssets,
|
||||
ReturnOnEquity: a.ReturnOnEquity ?? b.ReturnOnEquity,
|
||||
GrossProfits: a.GrossProfits ?? b.GrossProfits,
|
||||
FreeCashflow: a.FreeCashflow ?? b.FreeCashflow,
|
||||
OperatingCashflow: a.OperatingCashflow ?? b.OperatingCashflow,
|
||||
RevenueGrowth: a.RevenueGrowth ?? b.RevenueGrowth,
|
||||
GrossMargins: a.GrossMargins ?? b.GrossMargins,
|
||||
EbitdaMargins: a.EbitdaMargins ?? b.EbitdaMargins,
|
||||
OperatingMargins: a.OperatingMargins ?? b.OperatingMargins,
|
||||
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
|
||||
FinancialCurrency: a.FinancialCurrency ?? b.FinancialCurrency
|
||||
);
|
||||
}
|
||||
|
||||
private static YahooDefaultKeyStatisticsDto? MergeDefaultKeyStatistics(YahooDefaultKeyStatisticsDto? a, YahooDefaultKeyStatisticsDto? b)
|
||||
{
|
||||
if (a == null) return b;
|
||||
if (b == null) return a;
|
||||
|
||||
return new YahooDefaultKeyStatisticsDto(
|
||||
PriceToBook: a.PriceToBook ?? b.PriceToBook,
|
||||
EnterpriseValue: a.EnterpriseValue ?? b.EnterpriseValue,
|
||||
ForwardPE: a.ForwardPE ?? b.ForwardPE,
|
||||
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
|
||||
FloatShares: a.FloatShares ?? b.FloatShares,
|
||||
SharesOutstanding: a.SharesOutstanding ?? b.SharesOutstanding,
|
||||
SharesShort: a.SharesShort ?? b.SharesShort,
|
||||
SharesShortPriorMonth: a.SharesShortPriorMonth ?? b.SharesShortPriorMonth,
|
||||
SharesShortPreviousMonthDate: a.SharesShortPreviousMonthDate ?? b.SharesShortPreviousMonthDate,
|
||||
DateShortInterest: a.DateShortInterest ?? b.DateShortInterest,
|
||||
SharesPercentSharesOut: a.SharesPercentSharesOut ?? b.SharesPercentSharesOut,
|
||||
HeldPercentInsiders: a.HeldPercentInsiders ?? b.HeldPercentInsiders,
|
||||
HeldPercentInstitutions: a.HeldPercentInstitutions ?? b.HeldPercentInstitutions,
|
||||
ShortRatio: a.ShortRatio ?? b.ShortRatio,
|
||||
ShortPercentOfFloat: a.ShortPercentOfFloat ?? b.ShortPercentOfFloat,
|
||||
Beta: a.Beta ?? b.Beta,
|
||||
Category: a.Category ?? b.Category,
|
||||
BookValue: a.BookValue ?? b.BookValue,
|
||||
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
|
||||
LastFiscalYearEnd: a.LastFiscalYearEnd ?? b.LastFiscalYearEnd,
|
||||
NextFiscalYearEnd: a.NextFiscalYearEnd ?? b.NextFiscalYearEnd,
|
||||
MostRecentQuarter: a.MostRecentQuarter ?? b.MostRecentQuarter,
|
||||
EarningsQuarterlyGrowth: a.EarningsQuarterlyGrowth ?? b.EarningsQuarterlyGrowth,
|
||||
NetIncomeToCommon: a.NetIncomeToCommon ?? b.NetIncomeToCommon,
|
||||
TrailingEps: a.TrailingEps ?? b.TrailingEps,
|
||||
ForwardEps: a.ForwardEps ?? b.ForwardEps,
|
||||
PegRatio: a.PegRatio ?? b.PegRatio,
|
||||
EnterpriseToRevenue: a.EnterpriseToRevenue ?? b.EnterpriseToRevenue,
|
||||
EnterpriseToEbitda: a.EnterpriseToEbitda ?? b.EnterpriseToEbitda,
|
||||
FiftyTwoWeekChange: a.FiftyTwoWeekChange ?? b.FiftyTwoWeekChange,
|
||||
SandP52WeekChange: a.SandP52WeekChange ?? b.SandP52WeekChange
|
||||
);
|
||||
}
|
||||
|
||||
private static YahooSummaryDetailDto? MergeSummaryDetail(YahooSummaryDetailDto? a, YahooSummaryDetailDto? b)
|
||||
{
|
||||
if (a == null) return b;
|
||||
if (b == null) return a;
|
||||
|
||||
return new YahooSummaryDetailDto(
|
||||
MaxAge: a.MaxAge ?? b.MaxAge,
|
||||
PriceHint: a.PriceHint ?? b.PriceHint,
|
||||
PreviousClose: a.PreviousClose ?? b.PreviousClose,
|
||||
Open: a.Open ?? b.Open,
|
||||
DayLow: a.DayLow ?? b.DayLow,
|
||||
DayHigh: a.DayHigh ?? b.DayHigh,
|
||||
RegularMarketPreviousClose: a.RegularMarketPreviousClose ?? b.RegularMarketPreviousClose,
|
||||
RegularMarketOpen: a.RegularMarketOpen ?? b.RegularMarketOpen,
|
||||
RegularMarketDayLow: a.RegularMarketDayLow ?? b.RegularMarketDayLow,
|
||||
RegularMarketDayHigh: a.RegularMarketDayHigh ?? b.RegularMarketDayHigh,
|
||||
DividendRate: a.DividendRate ?? b.DividendRate,
|
||||
DividendYield: a.DividendYield ?? b.DividendYield,
|
||||
ExDividendDate: a.ExDividendDate ?? b.ExDividendDate,
|
||||
PayoutRatio: a.PayoutRatio ?? b.PayoutRatio,
|
||||
FiveYearAvgDividendYield: a.FiveYearAvgDividendYield ?? b.FiveYearAvgDividendYield,
|
||||
Beta: a.Beta ?? b.Beta,
|
||||
TrailingPE: a.TrailingPE ?? b.TrailingPE,
|
||||
ForwardPE: a.ForwardPE ?? b.ForwardPE,
|
||||
Volume: a.Volume ?? b.Volume,
|
||||
RegularMarketVolume: a.RegularMarketVolume ?? b.RegularMarketVolume,
|
||||
AverageVolume: a.AverageVolume ?? b.AverageVolume,
|
||||
AverageVolume10days: a.AverageVolume10days ?? b.AverageVolume10days,
|
||||
AverageDailyVolume10Day: a.AverageDailyVolume10Day ?? b.AverageDailyVolume10Day,
|
||||
Bid: a.Bid ?? b.Bid,
|
||||
Ask: a.Ask ?? b.Ask,
|
||||
BidSize: a.BidSize ?? b.BidSize,
|
||||
AskSize: a.AskSize ?? b.AskSize,
|
||||
MarketCap: a.MarketCap ?? b.MarketCap,
|
||||
FiftyTwoWeekLow: a.FiftyTwoWeekLow ?? b.FiftyTwoWeekLow,
|
||||
FiftyTwoWeekHigh: a.FiftyTwoWeekHigh ?? b.FiftyTwoWeekHigh,
|
||||
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
|
||||
Currency: a.Currency ?? b.Currency
|
||||
);
|
||||
}
|
||||
|
||||
private static bool IsIsin(string value)
|
||||
{
|
||||
return value.Length == 12 &&
|
||||
|
||||
Reference in New Issue
Block a user