feat(core): dynamic settings service, IFinlyticLogger, log broadcaster, and persistent Yahoo auth

This commit is contained in:
2026-08-15 21:29:38 +02:00
parent 34fa774cbf
commit 3dbee36ca0
15 changed files with 920 additions and 392 deletions
+166 -144
View File
@@ -7,109 +7,17 @@ using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using Microsoft.Extensions.Logging;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
namespace FinlyticCore.Services.Yahoo;
/// <summary>
/// Thread-sicherer Client für den Zugriff auf die internen Yahoo Finance APIs.
/// Verwaltet automatisch den erforderlichen Cookie- (A3) und Crumb-Token-Authentifizierungs-Flow.
/// </summary>
public interface IYahooFinanceClient
{
/// <summary>
/// Stellt sicher, dass die aktuelle Session über ein gültiges Cookie und einen Crumb-Token verfügt.
/// </summary>
/// <param name="forceRefresh">Erzwingt das Erneuern des Authentifizierungs-Tokens, selbst wenn die Frist noch nicht abgelaufen ist.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Crumb-Token oder <c>null</c>, wenn die Authentifizierung fehlgeschlagen ist.</returns>
Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>
/// Sucht nach Tickern, Namen, ISINs oder Firmen über die Yahoo Finance Such-API.
/// erfordert keine Cookie/Crumb-Authentifizierung.
/// </summary>
/// <param name="query">Der Suchbegriff (z. B. "Apple", "US0378331005", "AAPL").</param>
/// <param name="quotesCount">Die maximale Anzahl an Treffern für Wertpapiere/Aktien.</param>
/// <param name="newsCount">Die maximale Anzahl an News-Treffern.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Suchergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooSearchResponseDto?> SearchAsync(
string query,
int quotesCount = 10,
int newsCount = 0,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft Fundamentaldaten und Unternehmens-Metadaten für ein bestimmtes Symbol über den quoteSummary-Endpunkt ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL", "MSFT").</param>
/// <param name="modules">Die abzufragenden Yahoo-Module (z. B. "assetProfile", "financialData").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Die Abfrageergebnisse als DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
IEnumerable<string> modules,
CancellationToken cancellationToken = default);
/// <summary>
/// Hilfsmethode zum Abrufen aller vordefinierten Standard-Module für ein Tickersymbol.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das vollständige QuoteSummary-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(
string symbol,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft historische Chart- und Kursdaten (OHLCV) für ein Symbol ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="range">Der Abfragezeitraum (z. B. "1d", "1m", "1y", "5y").</param>
/// <param name="interval">Das Intervall der Datenpunkte (z. B. "1m", "5m", "1d", "1wk").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Chart-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
string range = "1y",
string interval = "1d",
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft schnelle Realtime-Preise für eine Liste von Tickersymbolen ab.
/// </summary>
/// <param name="symbols">Eine Liste von Tickersymbolen (z. B. <c>["AAPL", "MSFT", "^GSPC"]</c>).</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Quote-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
CancellationToken cancellationToken = default);
/// <summary>
/// Bequeme Hilfsmethode, um den aktuellen regulären Marktpreis für ein einzelnes Tickersymbol abzufragen.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "^VIX", "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Preis als <see cref="decimal"/> oder <c>null</c>, wenn kein Preis ermittelt werden konnte.</returns>
Task<decimal?> GetLivePriceAsync(
string symbol,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
/// </summary>
public class YahooFinanceClient
{
private const string DefaultUserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
private readonly HttpClient _httpClient;
private readonly CookieContainer _cookieContainer;
private readonly ILogger<YahooFinanceClient>? _logger;
private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
private readonly ISettingsService? _settingsService;
private readonly SemaphoreSlim _authLock = new(1, 1);
private string? _crumb;
@@ -133,9 +41,13 @@ public class YahooFinanceClient
"calendarEvents"
};
public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
public YahooFinanceClient(
IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
ISettingsService? settingsService = null,
HttpClient? httpClient = null)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_settingsService = settingsService;
_cookieContainer = new CookieContainer();
if (httpClient != null)
@@ -147,71 +59,137 @@ public class YahooFinanceClient
var handler = new HttpClientHandler
{
CookieContainer = _cookieContainer,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
UseCookies = true,
AllowAutoRedirect = true
};
_httpClient = new HttpClient(handler);
}
if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
{
_httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
_httpClient = new HttpClient(handler);
_httpClient.DefaultRequestHeaders.Add("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
}
}
/// <summary>
/// Executes the Cookie (A3) &amp; Crumb token authentication flow.
/// 1. GET https://fc.yahoo.com (sets session A3 cookie)
/// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
/// Ensures that an active Yahoo session (Cookie + dynamic Crumb token) is initialized.
/// Uses persistent DB caching and only refreshes when the crumb is invalid or forceRefresh is true.
/// </summary>
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
CancellationToken cancellationToken = default)
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
await _authLock.WaitAsync(cancellationToken);
try
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
(DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
// 1. Check in-memory crumb
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb))
{
return _crumb;
}
_logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");
// 2. Check persistent DB cache via SettingsService
if (!forceRefresh && _settingsService != null)
{
try
{
var cachedCrumb = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCrumb, cancellationToken);
var cachedCookies = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCookie, cancellationToken);
// 1. Send GET request to fc.yahoo.com to obtain session cookie A3
if (!string.IsNullOrWhiteSpace(cachedCrumb) && !string.IsNullOrWhiteSpace(cachedCookies))
{
RestoreCookies(cachedCookies);
_crumb = cachedCrumb;
_lastAuthTime = DateTime.UtcNow;
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Restored cached Yahoo session & crumb from database ({Crumb}).", _crumb);
return _crumb;
}
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Error restoring cached Yahoo session from DB.");
}
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");
// 3. Send GET request to fc.yahoo.com to obtain session cookie A3
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
{
initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
// CookieContainer automatically intercepts and stores 'A3' cookie
}
// 2. Send GET request to getcrumb to obtain the dynamic crumb token
using (var crumbRequest =
new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
// 4. Send GET request to getcrumb to obtain dynamic crumb token
string[] crumbUrls = new[]
{
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (!crumbResponse.IsSuccessStatusCode)
"https://query1.finance.yahoo.com/v1/test/getcrumb",
"https://query2.finance.yahoo.com/v1/test/getcrumb"
};
foreach (var url in crumbUrls)
{
try
{
_logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
crumbResponse.StatusCode);
return null;
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
crumbRequest.Headers.Add("Accept", "*/*");
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (crumbResponse.IsSuccessStatusCode)
{
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
if (!string.IsNullOrWhiteSpace(crumbText))
{
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
// Persist to DB cache via SettingsService
if (_settingsService != null)
{
try
{
var serializedCookies = SerializeCookies();
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, _crumb, cancellationToken);
if (!string.IsNullOrWhiteSpace(serializedCookies))
{
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
}
}
catch (Exception persistEx)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, persistEx, "[YahooFinanceClient] Failed to persist new Yahoo session to database.");
}
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Acquired fresh Crumb token successfully and persisted: {Crumb}", _crumb);
return _crumb;
}
}
else
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Endpoint '{Url}' returned status {Status}", url, crumbResponse.StatusCode);
}
}
catch
{
// Fallthrough to next endpoint
}
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
_lastAuthTime = DateTime.UtcNow;
_logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
return _crumb;
}
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Failed to fetch crumb token from all endpoints.");
return null;
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
if (_finlyticLogger != null)
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
return null;
}
finally
@@ -220,6 +198,48 @@ public class YahooFinanceClient
}
}
private string SerializeCookies()
{
try
{
var cookies = _cookieContainer.GetAllCookies();
var pairs = new List<string>();
foreach (Cookie cookie in cookies)
{
pairs.Add($"{cookie.Name}={cookie.Value}");
}
return string.Join(";", pairs);
}
catch
{
return string.Empty;
}
}
private void RestoreCookies(string serializedCookies)
{
if (string.IsNullOrWhiteSpace(serializedCookies)) return;
try
{
var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
foreach (var part in parts)
{
var eqIdx = part.IndexOf('=');
if (eqIdx > 0 && eqIdx < part.Length - 1)
{
var name = part.Substring(0, eqIdx).Trim();
var val = part.Substring(eqIdx + 1).Trim();
_cookieContainer.Add(new Cookie(name, val, "/", ".yahoo.com"));
}
}
}
catch
{
// Ignore cookie restore errors
}
}
/// <summary>
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
@@ -241,8 +261,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query, response.StatusCode);
return null;
}
@@ -251,7 +271,8 @@ public class YahooFinanceClient
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
if (_finlyticLogger != null)
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
return null;
}
}
@@ -276,8 +297,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
symbol, response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
@@ -318,8 +339,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
@@ -351,7 +372,8 @@ public class YahooFinanceClient
if (!response.IsSuccessStatusCode)
{
_logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
@@ -396,8 +418,8 @@ public class YahooFinanceClient
if (isAuthError)
{
_logger?.LogInformation(
"[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
@@ -413,7 +435,7 @@ public class YahooFinanceClient
return new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
}
}
+4 -12
View File
@@ -9,7 +9,6 @@ using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Playwright;
namespace FinlyticCore.Clients;
@@ -21,25 +20,18 @@ public interface IYahooFinanceHtmlClient
CancellationToken cancellationToken = default);
}
public interface IYahooFinanceHtmlClient<TDbContext> : IYahooFinanceHtmlClient
where TDbContext : DbContext
{
}
public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHtmlClient<TDbContext>
where TDbContext : DbContext
public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
{
private const string _serviceName = nameof(YahooFinanceHtmlClient);
private readonly IPlaywrightExecutionService _playwrightService;
private readonly IFinlyticLogger<TContextClass, TDbContext> _finlyticLogger;
private readonly string _serviceName;
private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
public YahooFinanceHtmlClient(
IPlaywrightExecutionService playwrightService,
IFinlyticLogger<TContextClass, TDbContext> finlyticLogger)
IFinlyticLogger<YahooFinanceHtmlClient> finlyticLogger)
{
_playwrightService = playwrightService;
_finlyticLogger = finlyticLogger;
_serviceName = typeof(TContextClass).Name;
}
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
@@ -0,0 +1,15 @@
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Entities.Settings;
using Microsoft.EntityFrameworkCore;
namespace FinlyticCore.Database;
/// <summary>
/// Einheitliches Interface für DbContexts, die dynamische Einstellungen verwalten.
/// </summary>
public interface ISettingsDbContext
{
DbSet<SettingEntity> DynamicSettings { get; }
Task<int> SaveChangesAsync(CancellationToken cancellationToken = default);
}
@@ -0,0 +1,13 @@
using System;
using System.Text.Json.Serialization;
namespace FinlyticCore.Dtos.Logging;
public record LogMessageDto(
[property: JsonPropertyName("timestamp")] DateTime Timestamp,
[property: JsonPropertyName("serviceName")] string ServiceName,
[property: JsonPropertyName("channel")] string Channel,
[property: JsonPropertyName("level")] string Level,
[property: JsonPropertyName("message")] string Message,
[property: JsonPropertyName("exception")] string? Exception = null
);
@@ -0,0 +1,14 @@
using System;
namespace FinlyticCore.Dtos.Settings;
/// <summary>
/// Repräsentiert eine dynamische Einstellung für das Web-UI und MQTT-RPC.
/// </summary>
public record DynamicSettingDto(
string Key,
object? Value,
string Type,
string Description = "",
DateTime? UpdatedAt = null
);
+2
View File
@@ -9,6 +9,8 @@
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Abstractions" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.9" />
<PackageReference Include="Microsoft.Playwright" Version="1.49.0" />
<PackageReference Include="MQTTnet" Version="5.1.0.1559" />
@@ -5,16 +5,27 @@ namespace FinlyticCore.Models.Settings;
/// </summary>
public static class CoreSettingKeys
{
// --- Logging-Kanäle ---
// --- Globale Logging-Kanäle ---
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HtmlScrapperChannel = new("Logging.Channel.HtmlScrapper", true);
public static readonly SettingKey<bool> YahooClientChannel = new("Logging.Channel.YahooClient", true);
public static readonly SettingKey<bool> FundamentalsChannel = new("Logging.Channel.Fundamentals", true);
public static readonly SettingKey<bool> TradeRepublicChannel = new("Logging.Channel.TradeRepublic", true);
public static readonly SettingKey<bool> PlaywrightChannel = new("Logging.Channel.Playwright", true);
public static readonly SettingKey<bool> SettingsChannel = new("Logging.Channel.Settings", true);
// --- Scraper & Feature-Toggles ---
public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true);
public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true);
public static readonly SettingKey<int> ScraperTimeoutSeconds = new("Scraper.TimeoutSeconds", 30);
public static readonly SettingKey<int> ScraperMaxRetries = new("Scraper.MaxRetries", 2);
// --- Trade Republic WebSocket Config ---
public static readonly SettingKey<int> TradeRepublicWsReconnectIntervalSeconds = new("TradeRepublic.WsReconnectIntervalSeconds", 5);
public static readonly SettingKey<int> TradeRepublicWsTimeoutSeconds = new("TradeRepublic.WsTimeoutSeconds", 15);
// --- Yahoo Auth Persistence ---
public static readonly SettingKey<string> YahooAuthCrumb = new("Yahoo.Auth.Crumb", "");
public static readonly SettingKey<string> YahooAuthCookie = new("Yahoo.Auth.Cookie", "");
}
@@ -1,17 +1,42 @@
using System;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Logging;
using FinlyticCore.Models.Settings;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services;
/// <summary>
/// Bietet kanalbasierte, dynamisch steuerbare Logging-Funktionalitäten über den <see cref="ISettingsService{TContext}"/>.
/// Globaler Broadcaster für strukturierte Logs in Echtzeit.
/// </summary>
public static class FinlyticLogBroadcaster
{
public static Func<LogMessageDto, Task>? OnLogPublished { get; set; }
public static void Broadcast(LogMessageDto dto)
{
if (OnLogPublished != null)
{
_ = Task.Run(async () =>
{
try
{
await OnLogPublished(dto);
}
catch
{
// Ignore broadcast errors to never disrupt execution
}
});
}
}
}
/// <summary>
/// Bietet kanalbasierte, dynamisch steuerbare Logging-Funktionalitäten über den <see cref="ISettingsService"/>.
/// </summary>
/// <typeparam name="TContextClass">Die aufrufende Klasse (für Log-Kategorien).</typeparam>
/// <typeparam name="TDbContext">Der DbContext des Services für den Zugriff auf die Settings.</typeparam>
public interface IFinlyticLogger<TContextClass, TDbContext> where TDbContext : DbContext
public interface IFinlyticLogger<TContextClass>
{
// --- Debug ---
Task LogDebugAsync(SettingKey<bool> channelKey, string message, params object[] args);
@@ -36,22 +61,49 @@ public interface IFinlyticLogger<TContextClass, TDbContext> where TDbContext : D
/// <summary>
/// Kanalbasierte Logger-Implementierung, die Einstellungen und Stummschaltungen
/// in Echtzeit aus dem <see cref="ISettingsService{TContext}"/> bezieht.
/// in Echtzeit aus dem <see cref="ISettingsService"/> bezieht.
/// </summary>
public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContextClass, TDbContext>
where TDbContext : DbContext
public class FinlyticLogger<TContextClass> : IFinlyticLogger<TContextClass>
{
private static readonly string ServiceName = typeof(TContextClass).Assembly.GetName().Name ?? "Finlytic";
private readonly ILogger<TContextClass> _logger;
private readonly ISettingsService<TDbContext> _settingsService;
private readonly ISettingsService _settingsService;
public FinlyticLogger(
ILogger<TContextClass> logger,
ISettingsService<TDbContext> settingsService)
ISettingsService settingsService)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_settingsService = settingsService ?? throw new ArgumentNullException(nameof(settingsService));
}
private void DispatchBroadcast(SettingKey<bool> channelKey, LogLevel level, string message, Exception? exception, params object[] args)
{
try
{
string formattedMsg = args != null && args.Length > 0 ? string.Format(message, args) : message;
FinlyticLogBroadcaster.Broadcast(new LogMessageDto(
Timestamp: DateTime.UtcNow,
ServiceName: ServiceName,
Channel: channelKey.Name,
Level: level.ToString(),
Message: formattedMsg,
Exception: exception?.ToString()
));
}
catch
{
FinlyticLogBroadcaster.Broadcast(new LogMessageDto(
Timestamp: DateTime.UtcNow,
ServiceName: ServiceName,
Channel: channelKey.Name,
Level: level.ToString(),
Message: message,
Exception: exception?.ToString()
));
}
}
#region Debug
public async Task LogDebugAsync(SettingKey<bool> channelKey, string message, params object[] args)
@@ -59,6 +111,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Debug))
{
_logger.LogDebug(message, args);
DispatchBroadcast(channelKey, LogLevel.Debug, message, null, args);
}
}
@@ -70,6 +123,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogDebug(exception, message, args);
else
_logger.LogDebug(message, args);
DispatchBroadcast(channelKey, LogLevel.Debug, message, exception, args);
}
}
@@ -82,6 +136,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Information))
{
_logger.LogInformation(message, args);
DispatchBroadcast(channelKey, LogLevel.Information, message, null, args);
}
}
@@ -93,6 +148,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogInformation(exception, message, args);
else
_logger.LogInformation(message, args);
DispatchBroadcast(channelKey, LogLevel.Information, message, exception, args);
}
}
@@ -105,6 +161,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Warning))
{
_logger.LogWarning(message, args);
DispatchBroadcast(channelKey, LogLevel.Warning, message, null, args);
}
}
@@ -116,6 +173,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogWarning(exception, message, args);
else
_logger.LogWarning(message, args);
DispatchBroadcast(channelKey, LogLevel.Warning, message, exception, args);
}
}
@@ -128,6 +186,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Error))
{
_logger.LogError(message, args);
DispatchBroadcast(channelKey, LogLevel.Error, message, null, args);
}
}
@@ -139,6 +198,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogError(exception, message, args);
else
_logger.LogError(message, args);
DispatchBroadcast(channelKey, LogLevel.Error, message, exception, args);
}
}
@@ -151,6 +211,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Trace))
{
_logger.LogTrace(message, args);
DispatchBroadcast(channelKey, LogLevel.Trace, message, null, args);
}
}
@@ -162,14 +223,12 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogCritical(exception, message, args);
else
_logger.LogCritical(message, args);
DispatchBroadcast(channelKey, LogLevel.Critical, message, exception, args);
}
}
#endregion
/// <summary>
/// Prüft, ob ein spezifischer Kanal und die aufrufende Klasse aktives Logging erlauben.
/// </summary>
private async Task<bool> ShouldLogAsync(SettingKey<bool> channelKey, LogLevel level)
{
ArgumentNullException.ThrowIfNull(channelKey);
@@ -1,9 +1,14 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using Microsoft.Playwright;
namespace FinlyticCore.Services.PlaywrightScrapper;
public interface IPlaywrightBrowserFactory : IAsyncDisposable
public interface IPlaywrightBrowserFactory : IAsyncDisposable, IDisposable
{
/// <summary>
/// Stellt sicher, dass die IBrowser-Instanz verbunden ist.
@@ -18,15 +23,15 @@ public interface IPlaywrightBrowserFactory : IAsyncDisposable
public class PlaywrightBrowserFactory : IPlaywrightBrowserFactory
{
private readonly ILogger<PlaywrightBrowserFactory> _logger;
private readonly IFinlyticLogger<PlaywrightBrowserFactory> _finlyticLogger;
private readonly SemaphoreSlim _browserLock = new(1, 1);
private IPlaywright? _playwright;
private IBrowser? _browser;
public PlaywrightBrowserFactory(ILogger<PlaywrightBrowserFactory> logger)
public PlaywrightBrowserFactory(IFinlyticLogger<PlaywrightBrowserFactory> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
}
public async Task<IBrowser> GetBrowserAsync(CancellationToken cancellationToken = default)
@@ -51,7 +56,7 @@ public class PlaywrightBrowserFactory : IPlaywrightBrowserFactory
}
});
_logger.LogInformation("[PlaywrightFactory] Shared Chromium Instance successfully launched.");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.PlaywrightChannel, "[PlaywrightFactory] Shared Chromium Instance successfully launched.");
return _browser;
}
finally
@@ -79,12 +84,41 @@ public class PlaywrightBrowserFactory : IPlaywrightBrowserFactory
}
};
public void Dispose()
{
try
{
if (_browser != null)
{
_browser.CloseAsync().GetAwaiter().GetResult();
_browser.DisposeAsync().GetAwaiter().GetResult();
}
}
catch
{
// Ignore any sync disposal timeouts
}
finally
{
_playwright?.Dispose();
_browserLock.Dispose();
GC.SuppressFinalize(this);
}
}
public async ValueTask DisposeAsync()
{
if (_browser != null)
{
await _browser.CloseAsync();
await _browser.DisposeAsync();
try
{
await _browser.CloseAsync();
await _browser.DisposeAsync();
}
catch
{
// Ignore disposal errors
}
}
_playwright?.Dispose();
+395 -42
View File
@@ -1,50 +1,54 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Database;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Entities.Settings;
using FinlyticCore.Models.Settings;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services;
public interface ISettingsService<TContext> where TContext : DbContext
public interface ISettingsService
{
// --- 1. Typsicherer Zugriff über SettingKey<T> (Empfohlen) ---
Task<T> GetSettingAsync<T>(SettingKey<T> key,
CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(SettingKey<T> key, T value,
CancellationToken cancellationToken = default);
Task<T> GetSettingAsync<T>(SettingKey<T> key, CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(SettingKey<T> key, T value, CancellationToken cancellationToken = default);
// --- 2. Dynamischer Zugriff über Enum-Key ---
Task<T> GetSettingAsync<TEnum, T>(TEnum enumKey, T defaultValue = default!,
CancellationToken cancellationToken = default) where TEnum : struct, Enum;
Task SetSettingAsync<TEnum, T>(TEnum enumKey, T value,
CancellationToken cancellationToken = default) where TEnum : struct, Enum;
Task<T> GetSettingAsync<TEnum, T>(TEnum enumKey, T defaultValue = default!, CancellationToken cancellationToken = default) where TEnum : struct, Enum;
Task SetSettingAsync<TEnum, T>(TEnum enumKey, T value, CancellationToken cancellationToken = default) where TEnum : struct, Enum;
// --- 3. Dynamischer Zugriff über String-Key ---
Task<T> GetSettingAsync<T>(string key, T defaultValue = default!,
CancellationToken cancellationToken = default);
Task<T> GetSettingAsync<T>(string key, T defaultValue = default!, CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(string key, T value, CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(string key, T value,
CancellationToken cancellationToken = default);
// --- 4. Reflection-Erkennung & Bulk-Verwaltung für Web UI / MQTT ---
Task<List<DynamicSettingDto>> GetAllRegisteredSettingsAsync(IEnumerable<Type>? customKeyHolders = null, CancellationToken cancellationToken = default);
Task UpdateSettingsAsync(Dictionary<string, object?> updatedSettings, CancellationToken cancellationToken = default);
}
public class SettingsService<TContext> : ISettingsService<TContext> where TContext : DbContext
public class SettingsService : ISettingsService
{
private readonly Microsoft.Extensions.DependencyInjection.IServiceScopeFactory _scopeFactory;
private readonly ILogger<SettingsService<TContext>>? _logger;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<SettingsService>? _logger;
// Fast In-Memory Cache: Key Schema: "KeyName"
private readonly ConcurrentDictionary<string, string> _cache = new();
// Fast In-Memory Cache: Key Schema: "KeyName" -> JSON string
private readonly ConcurrentDictionary<string, string> _cache = new(StringComparer.OrdinalIgnoreCase);
public SettingsService(Microsoft.Extensions.DependencyInjection.IServiceScopeFactory scopeFactory, ILogger<SettingsService<TContext>>? logger = null)
public SettingsService(
IServiceScopeFactory scopeFactory,
ILogger<SettingsService>? logger = null)
{
_scopeFactory = scopeFactory;
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
_logger = logger;
}
@@ -52,11 +56,13 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
public Task<T> GetSettingAsync<T>(SettingKey<T> key, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(key);
return GetSettingInternalAsync(key.Name, key.DefaultValue, cancellationToken);
}
public Task SetSettingAsync<T>(SettingKey<T> key, T value, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(key);
return SetSettingInternalAsync(key.Name, value, cancellationToken);
}
@@ -96,11 +102,138 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
#endregion
#region Core Engine Logik
#region Bulk & Reflection Discovery
public async Task<List<DynamicSettingDto>> GetAllRegisteredSettingsAsync(
IEnumerable<Type>? customKeyHolders = null,
CancellationToken cancellationToken = default)
{
var holderTypes = new List<Type> { typeof(CoreSettingKeys) };
if (customKeyHolders != null)
{
holderTypes.AddRange(customKeyHolders);
}
var resultList = new List<DynamicSettingDto>();
var seenKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
// 1. Reflection auf allen SettingKey<T> Feldern
foreach (var type in holderTypes.Distinct())
{
var fields = type.GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy);
foreach (var field in fields)
{
var fieldType = field.FieldType;
if (fieldType.IsGenericType && fieldType.GetGenericTypeDefinition() == typeof(SettingKey<>))
{
var valType = fieldType.GetGenericArguments()[0];
var settingKeyObj = field.GetValue(null);
if (settingKeyObj == null) continue;
var nameProp = fieldType.GetProperty("Name");
var defaultProp = fieldType.GetProperty("DefaultValue");
var keyName = nameProp?.GetValue(settingKeyObj)?.ToString() ?? field.Name;
if (seenKeys.Contains(keyName)) continue;
seenKeys.Add(keyName);
var defVal = defaultProp?.GetValue(settingKeyObj);
var typeName = MapToSimpleTypeName(valType);
// Aktuellen Wert aus DB / Cache lesen
var currentRawValue = await GetSettingInternalObjectAsync(keyName, valType, defVal, cancellationToken);
resultList.Add(new DynamicSettingDto(
Key: keyName,
Value: currentRawValue,
Type: typeName,
Description: FormatDescriptionFromKey(keyName),
UpdatedAt: DateTime.UtcNow
));
}
}
}
// 2. Prüfen, ob in der DB weitere gespeicherte Settings existieren, die nicht im Code deklariert sind
try
{
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext != null)
{
var dbSettings = await dbContext.DynamicSettings.AsNoTracking().ToListAsync(cancellationToken);
foreach (var dbSetting in dbSettings)
{
if (!seenKeys.Contains(dbSetting.Key))
{
seenKeys.Add(dbSetting.Key);
var (inferredVal, inferredType) = InferJsonValueAndType(dbSetting.ValueJson);
resultList.Add(new DynamicSettingDto(
Key: dbSetting.Key,
Value: inferredVal,
Type: inferredType,
Description: FormatDescriptionFromKey(dbSetting.Key),
UpdatedAt: dbSetting.LastUpdatedUtc
));
}
}
}
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "[SettingsService] Error reading database settings during GetAllRegisteredSettingsAsync.");
}
return resultList.OrderBy(s => s.Key).ToList();
}
public async Task UpdateSettingsAsync(Dictionary<string, object?> updatedSettings, CancellationToken cancellationToken = default)
{
if (updatedSettings == null || updatedSettings.Count == 0) return;
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
foreach (var (key, rawValue) in updatedSettings)
{
if (string.IsNullOrWhiteSpace(key)) continue;
string jsonValue = NormalizeJsonValue(rawValue);
_cache[key] = jsonValue;
if (dbContext != null)
{
var entity = await dbContext.DynamicSettings.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
if (entity == null)
{
entity = new SettingEntity
{
Key = key,
ValueJson = jsonValue,
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.DynamicSettings.Add(entity);
}
else
{
entity.ValueJson = jsonValue;
entity.LastUpdatedUtc = DateTime.UtcNow;
}
}
}
if (dbContext != null)
{
await dbContext.SaveChangesAsync(cancellationToken);
}
}
#endregion
#region Internal Engine Logic
private async Task<T> GetSettingInternalAsync<T>(string key, T defaultValue, CancellationToken cancellationToken)
{
// 1. Zuerst im In-Memory Cache prüfen
if (_cache.TryGetValue(key, out var cachedJson))
{
return DeserializeValue(cachedJson, defaultValue);
@@ -108,17 +241,21 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
try
{
using var scope = _scopeFactory.CreateScope();
var dbContext = Microsoft.Extensions.DependencyInjection.ServiceProviderServiceExtensions.GetRequiredService<TContext>(scope.ServiceProvider);
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext == null)
{
var defaultJson = NormalizeJsonValue(defaultValue);
_cache[key] = defaultJson;
return defaultValue;
}
// 2. Aus DB der spezifischen TContext-Instanz laden
var entity = await dbContext.Set<SettingEntity>()
var entity = await dbContext.DynamicSettings
.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
// 3. Falls noch nicht vorhanden: In DB anlegen (Seed on Demand)
if (entity == null)
{
var defaultJson = JsonSerializer.Serialize(defaultValue);
var defaultJson = NormalizeJsonValue(defaultValue);
entity = new SettingEntity
{
Key = key,
@@ -126,35 +263,76 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.Set<SettingEntity>().Add(entity);
dbContext.DynamicSettings.Add(entity);
await dbContext.SaveChangesAsync(cancellationToken);
_cache[key] = defaultJson;
return defaultValue;
}
// In Cache legen & Wert zurückgeben
_cache[key] = entity.ValueJson;
return DeserializeValue(entity.ValueJson, defaultValue);
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "Setting '{Key}' could not be loaded or initialized in DB. Using default value in memory.", key);
_cache[key] = JsonSerializer.Serialize(defaultValue);
_logger?.LogWarning(ex, "[SettingsService] Setting '{Key}' could not be loaded or initialized in DB. Using default value.", key);
_cache[key] = NormalizeJsonValue(defaultValue);
return defaultValue;
}
}
private async Task<object?> GetSettingInternalObjectAsync(string key, Type valueType, object? defaultValue, CancellationToken cancellationToken)
{
if (_cache.TryGetValue(key, out var cachedJson))
{
return DeserializeObject(cachedJson, valueType, defaultValue);
}
try
{
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext == null) return defaultValue;
var entity = await dbContext.DynamicSettings.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
if (entity == null)
{
var defaultJson = NormalizeJsonValue(defaultValue);
entity = new SettingEntity
{
Key = key,
ValueJson = defaultJson,
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.DynamicSettings.Add(entity);
await dbContext.SaveChangesAsync(cancellationToken);
_cache[key] = defaultJson;
return defaultValue;
}
_cache[key] = entity.ValueJson;
return DeserializeObject(entity.ValueJson, valueType, defaultValue);
}
catch
{
return defaultValue;
}
}
private async Task SetSettingInternalAsync<T>(string key, T value, CancellationToken cancellationToken)
{
var jsonValue = JsonSerializer.Serialize(value);
var jsonValue = NormalizeJsonValue(value);
_cache[key] = jsonValue;
try
{
using var scope = _scopeFactory.CreateScope();
var dbContext = Microsoft.Extensions.DependencyInjection.ServiceProviderServiceExtensions.GetRequiredService<TContext>(scope.ServiceProvider);
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext == null) return;
var entity = await dbContext.Set<SettingEntity>()
var entity = await dbContext.DynamicSettings
.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
if (entity == null)
@@ -165,7 +343,7 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
ValueJson = jsonValue,
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.Set<SettingEntity>().Add(entity);
dbContext.DynamicSettings.Add(entity);
}
else
{
@@ -177,17 +355,91 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "Setting '{Key}' could not be saved to DB.", key);
_logger?.LogWarning(ex, "[SettingsService] Setting '{Key}' could not be saved to DB.", key);
}
}
private static string NormalizeJsonValue(object? rawValue)
{
if (rawValue == null) return "null";
if (rawValue is JsonElement jsonElem)
{
if (jsonElem.ValueKind == JsonValueKind.String)
{
var str = jsonElem.GetString()?.Trim() ?? string.Empty;
var unquoted = str.Trim('\"', ' ');
if (bool.TryParse(unquoted, out var b)) return b ? "true" : "false";
if (long.TryParse(unquoted, out var l)) return l.ToString();
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d.ToString(CultureInfo.InvariantCulture);
return JsonSerializer.Serialize(str);
}
return jsonElem.GetRawText();
}
// Cache trotzdem aktualisieren
_cache[key] = jsonValue;
if (rawValue is string s)
{
var unquoted = s.Trim('\"', ' ');
if (bool.TryParse(unquoted, out var b)) return b ? "true" : "false";
if (long.TryParse(unquoted, out var l)) return l.ToString();
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d.ToString(CultureInfo.InvariantCulture);
return JsonSerializer.Serialize(s);
}
if (rawValue is bool bVal) return bVal ? "true" : "false";
if (rawValue is int or long or short or byte) return rawValue.ToString()!;
if (rawValue is double or float or decimal) return Convert.ToString(rawValue, CultureInfo.InvariantCulture)!;
return JsonSerializer.Serialize(rawValue);
}
private static T DeserializeValue<T>(string json, T defaultValue)
{
if (string.IsNullOrWhiteSpace(json)) return defaultValue;
try
{
var unquoted = json.Trim('\"', ' ');
if (typeof(T) == typeof(bool))
{
if (bool.TryParse(unquoted, out var b))
{
return (T)(object)b;
}
}
else if (typeof(T) == typeof(int))
{
if (int.TryParse(unquoted, out var i))
{
return (T)(object)i;
}
}
else if (typeof(T) == typeof(long))
{
if (long.TryParse(unquoted, out var l))
{
return (T)(object)l;
}
}
else if (typeof(T) == typeof(double))
{
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d))
{
return (T)(object)d;
}
}
else if (typeof(T) == typeof(string))
{
var trimmed = json.Trim();
if (trimmed.StartsWith("\"") && trimmed.EndsWith("\"") && trimmed.Length >= 2)
{
try { return (T)(object)(JsonSerializer.Deserialize<string>(trimmed) ?? trimmed.Trim('\"')); }
catch { return (T)(object)trimmed.Trim('\"'); }
}
return (T)(object)trimmed;
}
var result = JsonSerializer.Deserialize<T>(json);
return result ?? defaultValue;
}
@@ -197,5 +449,106 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
}
}
private static object? DeserializeObject(string json, Type valueType, object? defaultValue)
{
if (string.IsNullOrWhiteSpace(json)) return defaultValue;
try
{
var unquoted = json.Trim('\"', ' ');
if (valueType == typeof(bool))
{
if (bool.TryParse(unquoted, out var b)) return b;
}
else if (valueType == typeof(int))
{
if (int.TryParse(unquoted, out var i)) return i;
}
else if (valueType == typeof(long))
{
if (long.TryParse(unquoted, out var l)) return l;
}
else if (valueType == typeof(double))
{
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d;
}
else if (valueType == typeof(string))
{
var trimmed = json.Trim();
if (trimmed.StartsWith("\"") && trimmed.EndsWith("\"") && trimmed.Length >= 2)
{
try { return JsonSerializer.Deserialize<string>(trimmed) ?? trimmed.Trim('\"'); }
catch { return trimmed.Trim('\"'); }
}
return trimmed;
}
return JsonSerializer.Deserialize(json, valueType) ?? defaultValue;
}
catch
{
return defaultValue;
}
}
private static string MapToSimpleTypeName(Type type)
{
if (type == typeof(bool)) return "bool";
if (type == typeof(int) || type == typeof(short) || type == typeof(byte) || type == typeof(long)) return "int";
if (type == typeof(double) || type == typeof(float) || type == typeof(decimal)) return "double";
return "string";
}
private static (object? Value, string Type) InferJsonValueAndType(string json)
{
if (string.IsNullOrWhiteSpace(json)) return (string.Empty, "string");
try
{
var unquoted = json.Trim('\"', ' ');
if (bool.TryParse(unquoted, out var b)) return (b, "bool");
if (long.TryParse(unquoted, out var l)) return (l, "int");
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return (d, "double");
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
return root.ValueKind switch
{
JsonValueKind.True => (true, "bool"),
JsonValueKind.False => (false, "bool"),
JsonValueKind.Number when root.TryGetInt64(out var i) => (i, "int"),
JsonValueKind.Number => (root.GetDouble(), "double"),
JsonValueKind.String => (root.GetString(), "string"),
_ => (json, "string")
};
}
catch
{
return (json, "string");
}
}
private static string FormatDescriptionFromKey(string key)
{
if (key.StartsWith("Logging.Channel.", StringComparison.OrdinalIgnoreCase))
{
return $"Logging-Kanal für {key.Substring(16)} (Ein/Aus)";
}
if (key.StartsWith("Feature.", StringComparison.OrdinalIgnoreCase))
{
return $"Feature-Toggle für {key.Substring(8)}";
}
if (key.StartsWith("Cache.", StringComparison.OrdinalIgnoreCase))
{
return $"Cache-Konfiguration ({key.Substring(6)})";
}
if (key.StartsWith("Scraper.", StringComparison.OrdinalIgnoreCase))
{
return $"Scraper-Konfiguration ({key.Substring(8)})";
}
return key;
}
#endregion
}
@@ -4,8 +4,9 @@ using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services.TradeRepublic;
@@ -15,7 +16,7 @@ namespace FinlyticCore.Services.TradeRepublic;
/// </summary>
public class TradeRepublicClient : ManagedWebSocket
{
private readonly ILogger<TradeRepublicClient> _logger;
private readonly IFinlyticLogger<TradeRepublicClient> _finlyticLogger;
private int _currentSub;
private readonly ConcurrentDictionary<int, TaskCompletionSource<ReceivedMessage>> _pendingRequests = new();
private readonly ConcurrentDictionary<int, Action<string>> _tickerSubscriptions = new();
@@ -26,10 +27,10 @@ public class TradeRepublicClient : ManagedWebSocket
/// <summary>
/// Initializes a new instance of the <see cref="TradeRepublicClient"/> class.
/// </summary>
/// <param name="logger">The logger instance.</param>
public TradeRepublicClient(ILogger<TradeRepublicClient> logger)
/// <param name="finlyticLogger">The logger instance.</param>
public TradeRepublicClient(IFinlyticLogger<TradeRepublicClient> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -57,7 +58,7 @@ public class TradeRepublicClient : ManagedWebSocket
var isConnected = res.Type == "connected";
if (isConnected)
{
_logger.LogInformation("[{Channel}] WebSocket connection to Trade Republic established.", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] WebSocket connection to Trade Republic established.");
}
return isConnected;
@@ -65,7 +66,7 @@ public class TradeRepublicClient : ManagedWebSocket
catch (Exception ex)
{
_pendingRequests.TryRemove(-1, out _);
_logger.LogWarning(ex, "[{Channel}] Failed or timed out establishing Trade Republic WebSocket connection.", "TradeRepublicChannel");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicClient] Failed or timed out establishing Trade Republic WebSocket connection.");
return false;
}
}
@@ -84,7 +85,7 @@ public class TradeRepublicClient : ManagedWebSocket
var tempSub = Interlocked.Increment(ref _currentSub);
var msg = $"sub {tempSub} {JsonSerializer.Serialize(request, typeof(TRequest), FinlyticJsonSerializerContext.Default)}";
_logger.LogDebug("[{Channel}] TR WS Sent (Request): {Message}", "TradeRepublicChannel", msg);
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] TR WS Sent (Request): {Message}", msg);
var tcs = new TaskCompletionSource<ReceivedMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
_pendingRequests.TryAdd(tempSub, tcs);
@@ -99,7 +100,7 @@ public class TradeRepublicClient : ManagedWebSocket
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Error waiting for Trade Republic response ID {SubId}", "TradeRepublicChannel", tempSub);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicClient] Error waiting for Trade Republic response ID {SubId}", tempSub);
return null;
}
finally
@@ -125,7 +126,6 @@ public class TradeRepublicClient : ManagedWebSocket
_tickerSubscriptions[tempSub] = jsonPayload =>
{
// Skip empty or non-JSON payloads (e.g. TR protocol ack messages)
if (string.IsNullOrWhiteSpace(jsonPayload) || (!jsonPayload.TrimStart().StartsWith('{') && !jsonPayload.TrimStart().StartsWith('[')))
return;
@@ -139,12 +139,12 @@ public class TradeRepublicClient : ManagedWebSocket
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to parse real-time ticker payload for {TickerId}", "TradeRepublicChannel", tickerId);
_ = _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicClient] Failed to parse real-time ticker payload for {TickerId}", tickerId);
}
};
_logger.LogInformation("[{Channel}] Subscribing to Trade Republic real-time ticker {TickerId} (Sub ID: {SubId})", "TradeRepublicChannel", tickerId, tempSub);
_logger.LogDebug("[{Channel}] TR WS Sent: {Message}", "TradeRepublicChannel", msg);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] Subscribing to Trade Republic real-time ticker {TickerId} (Sub ID: {SubId})", tickerId, tempSub);
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] TR WS Sent: {Message}", msg);
await SendAsync(msg);
return tempSub;
}
@@ -165,85 +165,81 @@ public class TradeRepublicClient : ManagedWebSocket
}
/// <inheritdoc />
protected override void OnMessageReceived(string message)
{
if (string.IsNullOrWhiteSpace(message)) return;
_logger.LogDebug("[{Channel}] TR WS Recv: {Message}", "TradeRepublicChannel", message);
var trimmed = message.Trim();
int subId;
string type;
string payload;
_logger.LogDebug("Trade republic response: " + message);
if (trimmed.Equals("connected", StringComparison.OrdinalIgnoreCase))
protected override void OnMessageReceived(string message)
{
subId = -1;
type = "connected";
payload = trimmed;
}
else
{
// Ziffern am Anfang zählen (Sub-ID)
var digitLen = 0;
while (digitLen < trimmed.Length && char.IsDigit(trimmed[digitLen]))
{
digitLen++;
}
if (string.IsNullOrWhiteSpace(message)) return;
// Keine Ziffer am Anfang (Reines System-Event/Error ohne ID)
if (digitLen == 0)
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
_ = _finlyticLogger.LogDebugAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] TR WS Recv: {Message}", message);
if (!int.TryParse(trimmed.Substring(0, digitLen), out subId))
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
var trimmed = message.Trim();
var remainder = trimmed.Substring(digitLen).TrimStart();
int subId;
string type;
string payload;
// 2. FALL: "34 connected" oder "34connected"
if (remainder.StartsWith("connected", StringComparison.OrdinalIgnoreCase))
if (trimmed.Equals("connected", StringComparison.OrdinalIgnoreCase))
{
subId = -1; // Mapping auf deine interne -1 für InitAsync
subId = -1;
type = "connected";
payload = remainder;
}
else if (remainder.Length > 0)
{
// Standard Trade Republic Data Push (z.B. "22A {...}")
type = remainder[0].ToString();
payload = remainder.Substring(1).TrimStart();
payload = trimmed;
}
else
{
type = "ack";
payload = string.Empty;
// Ziffern am Anfang zählen (Sub-ID)
var digitLen = 0;
while (digitLen < trimmed.Length && char.IsDigit(trimmed[digitLen]))
{
digitLen++;
}
// Keine Ziffer am Anfang (Reines System-Event/Error ohne ID)
if (digitLen == 0)
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
if (!int.TryParse(trimmed.Substring(0, digitLen), out subId))
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
var remainder = trimmed.Substring(digitLen).TrimStart();
// 2. FALL: "34 connected" oder "34connected"
if (remainder.StartsWith("connected", StringComparison.OrdinalIgnoreCase))
{
subId = -1;
type = "connected";
payload = remainder;
}
else if (remainder.Length > 0)
{
type = remainder[0].ToString();
payload = remainder.Substring(1).TrimStart();
}
else
{
type = "ack";
payload = string.Empty;
}
}
var received = new ReceivedMessage(subId, type, payload);
if (_pendingRequests.TryGetValue(subId, out var tcs))
{
tcs.TrySetResult(received);
}
if (_tickerSubscriptions.TryGetValue(subId, out var handler))
{
handler(payload);
}
UnhandledMessageReceived?.Invoke(received);
}
var received = new ReceivedMessage(subId, type, payload);
// Löst jetzt garantiert dein TaskCompletionSource(-1) in InitAsync auf!
if (_pendingRequests.TryGetValue(subId, out var tcs))
{
tcs.TrySetResult(received);
}
if (_tickerSubscriptions.TryGetValue(subId, out var handler))
{
handler(payload);
}
UnhandledMessageReceived?.Invoke(received);
}
}
/// <summary>
@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using System.Timers;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Models.Assets;
using Microsoft.Extensions.Logging;
using FinlyticCore.Models.Settings;
namespace FinlyticCore.Services.TradeRepublic;
@@ -16,72 +16,50 @@ public interface ITradeRepublicService
/// <summary>
/// Fetches asset metadata from Trade Republic by ISIN.
/// </summary>
/// <param name="isin">The ISIN to search for.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The Trade Republic search response, or null if not found/failed.</returns>
Task<TradeRepublicAssetResponse?> GetAsset(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the total count of available assets grouped by their types.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>An <see cref="AssetsCount"/> object containing the metrics.</returns>
Task<AssetsCount> GetAssetsCount(CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves a paginated chunk of assets filtered by a specific type.
/// </summary>
/// <param name="type">The type of assets to retrieve.</param>
/// <param name="page">The zero-based page index.</param>
/// <param name="pageSize">The number of elements per page.</param>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A <see cref="TradeRepublicAssetResponse"/> containing the elements, or null if the request fails.</returns>
Task<TradeRepublicAssetResponse?> GetAssets(AssetType type, int page, int pageSize, CancellationToken cancellationToken = default);
/// <summary>
/// Subscribes to the real-time ticker stream for a specific ISIN.
/// </summary>
/// <param name="isin">The ISIN.</param>
/// <param name="onTick">The callback action when a tick is received.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The subscription ID, or null if failed.</returns>
Task<int?> SubscribeRealtimeTickerAsync(string isin, Action<TradeRepublicTickerResponse> onTick, CancellationToken cancellationToken = default);
/// <summary>
/// Unsubscribes from a real-time ticker stream.
/// </summary>
/// <param name="subId">The subscription ID to unsubscribe.</param>
/// <returns>A task representing the async operation.</returns>
Task UnsubscribeRealtimeTickerAsync(int subId);
/// <summary>
/// Fetches stock details (company description, events, earnings, analyst ratings) for a specific ISIN.
/// </summary>
/// <param name="isin">The ISIN of the stock.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The stock details response, or null if failed.</returns>
Task<TradeRepublicStockDetailsResponse?> GetStockDetailsAsync(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Fetches derivative products (KnockOuts, Warrants, Factor Certificates) for an underlying ISIN.
/// </summary>
/// <param name="request">The derivative query parameters.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The derivatives response, or null if failed.</returns>
Task<TradeRepublicDerivativesResponse?> GetDerivativesAsync(TradeRepublicDerivativesRequest request, CancellationToken cancellationToken = default);
}
public class TradeRepublicService : ITradeRepublicService, IDisposable
{
private readonly TradeRepublicClient _client;
private readonly ILogger<TradeRepublicService> _logger;
private readonly IFinlyticLogger<TradeRepublicService> _finlyticLogger;
private readonly System.Timers.Timer _inactivityTimer;
private readonly SemaphoreSlim _lock = new(1, 1);
public TradeRepublicService(TradeRepublicClient client, ILogger<TradeRepublicService> logger)
public TradeRepublicService(TradeRepublicClient client, IFinlyticLogger<TradeRepublicService> finlyticLogger)
{
_client = client;
_logger = logger;
_finlyticLogger = finlyticLogger;
_inactivityTimer = new System.Timers.Timer(TimeSpan.FromSeconds(461).TotalMilliseconds);
_inactivityTimer.AutoReset = false;
@@ -96,14 +74,14 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
_inactivityTimer.Stop();
if (!_client.IsConnected)
{
_logger.LogInformation("[{Channel}] Connecting to Trade Republic API WebSocket...", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Connecting to Trade Republic API WebSocket...");
bool connected = await _client.InitAsync();
if (!connected)
{
_logger.LogWarning("[{Channel}] Trade Republic WebSocket connection failed or timed out.", "TradeRepublicChannel");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Trade Republic WebSocket connection failed or timed out.");
throw new InvalidOperationException("Trade Republic WebSocket is not connected.");
}
_logger.LogInformation("[{Channel}] Successfully connected to Trade Republic API.", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Successfully connected to Trade Republic API.");
}
_inactivityTimer.Start();
}
@@ -131,7 +109,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error while fetching asset metadata for ISIN {Isin}", "TradeRepublicChannel", isin);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicService] Error while fetching asset metadata for ISIN {Isin}", isin);
return null;
}
}
@@ -206,7 +184,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error while fetching stock details for ISIN {Isin}", "TradeRepublicChannel", isin);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicService] Error while fetching stock details for ISIN {Isin}", isin);
return null;
}
}
@@ -221,7 +199,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error while fetching derivatives for underlying {Underlying}", "TradeRepublicChannel", request.Underlying);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicService] Error while fetching derivatives for underlying {Underlying}", request.Underlying);
return null;
}
}
@@ -232,7 +210,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
{
await _lock.WaitAsync();
if (!_client.IsConnected) return;
_logger.LogInformation("[{Channel}] Inactivity timer expired. Auto-disconnecting Trade Republic WebSocket.", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Inactivity timer expired. Auto-disconnecting Trade Republic WebSocket.");
await _client.DisconnectAsync();
}
catch { }
@@ -1,19 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Clients;
using FinlyticCore.Dtos.Fundamentals;
using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Util;
using Microsoft.Extensions.Logging;
using FinlyticCore.Utils;
using Microsoft.Extensions.Configuration;
namespace FinlyticFundamentals.Services;
namespace FinlyticCore.Services.Yahoo;
public interface IYahooFinanceScraper
{
@@ -31,10 +29,6 @@ public interface IYahooFinanceScraper
/// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab
/// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind.
/// </summary>
/// <param name="symbolOrIsin">Das Tickersymbol (z. B. "MSFT") oder die ISIN.</param>
/// <param name="forceHtmlScrape">Erzwingt sofortiges HTML-Scraping ohne API-Vorprüfung.</param>
/// <param name="cancellationToken">Abbruch-Token.</param>
/// <returns>Das aggregierte <see cref="YahooQuoteSummaryModulesDto"/> oder <c>null</c>.</returns>
Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
@@ -43,16 +37,17 @@ public interface IYahooFinanceScraper
public class YahooFinanceScraper : IYahooFinanceScraper
{
private const string _serviceName = nameof(YahooFinanceScraper);
private readonly YahooFinanceClient _yahooApiClient;
private readonly IYahooFinanceHtmlClient _htmlScraperClient;
private readonly Microsoft.Extensions.Configuration.IConfiguration _configuration;
private readonly IFinlyticLogger<YahooFinanceScraper, FundamentalsDbContext> _finlyticLogger;
private readonly IConfiguration _configuration;
private readonly IFinlyticLogger<YahooFinanceScraper> _finlyticLogger;
public YahooFinanceScraper(
YahooFinanceClient yahooApiClient,
IYahooFinanceHtmlClient htmlScraperClient,
Microsoft.Extensions.Configuration.IConfiguration configuration,
IFinlyticLogger<YahooFinanceScraper, FundamentalsDbContext> finlyticLogger)
IConfiguration configuration,
IFinlyticLogger<YahooFinanceScraper> finlyticLogger)
{
_yahooApiClient = yahooApiClient;
_htmlScraperClient = htmlScraperClient;
@@ -78,7 +73,7 @@ public class YahooFinanceScraper : IYahooFinanceScraper
// Crypto / Trade Republic interne ISINs (beginnend mit 'X', z. B. XF000BTC0017)
if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase))
{
var (cryptoSubtitle, cryptoName) = await FinlyticCore.Utils.CryptoSubtitleResolver.ResolveCryptoInfoAsync(
var (cryptoSubtitle, cryptoName) = await CryptoSubtitleResolver.ResolveCryptoInfoAsync(
cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken);
if (!string.IsNullOrWhiteSpace(cryptoSubtitle))
@@ -105,9 +100,8 @@ public class YahooFinanceScraper : IYahooFinanceScraper
}
catch { }
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[YahooFinanceScraper] Resolved Crypto ISIN {Isin} to {Symbol} using Subtitle {Sub}",
cleanIsin, cryptoEur, cryptoSubtitle);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel,
$"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}");
return symbols
.OrderBy(s => s.priority)
@@ -130,7 +124,6 @@ public class YahooFinanceScraper : IYahooFinanceScraper
foreach (var q in validQuotes.Skip(1))
{
if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin))));
@@ -158,13 +151,13 @@ public class YahooFinanceScraper : IYahooFinanceScraper
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] Fehler beim Auflösen des Tickers für ISIN '{Isin}'", cleanIsin);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'");
}
return symbols
.OrderBy(s => s.priority)
.Select(s => new TickerInfoDto(){Ticker = s.symbol, Exchange = s.exchange})
.Select(s => new TickerInfoDto { Ticker = s.symbol, Exchange = s.exchange })
.ToList();
}
@@ -197,26 +190,26 @@ public class YahooFinanceScraper : IYahooFinanceScraper
{
try
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Starte primären API-Abruf für '{Symbol}'...", symbol);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte primären API-Abruf für '{symbol}'...");
var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken);
apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault();
if (apiModules != null && HasSufficientData(apiModules))
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Erfolgreich Daten über API bezogen für '{Symbol}'.", symbol);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Erfolgreich Daten über API bezogen für '{symbol}'.");
return apiModules;
}
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] API lieferte unvollständige Daten für '{Symbol}'. Initiiere Fallback...", symbol);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] API lieferte unvollständige Daten für '{symbol}'. Initiiere Fallback...");
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] API-Abruf fehlgeschlagen für '{Symbol}'. Wechsle zu Scraper...", symbol);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] API-Abruf fehlgeschlagen für '{symbol}'. Wechsle zu Scraper...");
}
}
@@ -226,15 +219,15 @@ public class YahooFinanceScraper : IYahooFinanceScraper
YahooQuoteSummaryModulesDto? htmlModules = null;
try
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Starte HTML-Scraper Fallback für '{Symbol}'...", symbol);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}'...");
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{Symbol}'.", symbol);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{symbol}'.");
}
// -------------------------------------------------------------
@@ -246,9 +239,6 @@ public class YahooFinanceScraper : IYahooFinanceScraper
return MergeModules(apiModules, htmlModules);
}
/// <summary>
/// Prüft, ob das Modul-DTO die wesentlichen Fundamentalblöcke enthält.
/// </summary>
private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules)
{
return modules.SummaryDetail != null ||
@@ -256,9 +246,6 @@ public class YahooFinanceScraper : IYahooFinanceScraper
modules.DefaultKeyStatistics != null;
}
/// <summary>
/// Führt API- und Scraper-Daten zusammen, damit Lücken in API-Responses geschlossen werden.
/// </summary>
private static YahooQuoteSummaryModulesDto MergeModules(
YahooQuoteSummaryModulesDto primary,
YahooQuoteSummaryModulesDto secondary)
@@ -18,6 +18,8 @@ namespace FinlyticCore.Util;
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(TradeProposalDto))]
[JsonSerializable(typeof(List<TradeProposalDto>))]
[JsonSerializable(typeof(FinlyticCore.Dtos.Logging.LogMessageDto))]
[JsonSerializable(typeof(List<FinlyticCore.Dtos.Logging.LogMessageDto>))]
[JsonSerializable(typeof(TradeAcceptanceDto))]
[JsonSerializable(typeof(List<TradeAcceptanceDto>))]
[JsonSerializable(typeof(CloseTradeRequest))]
+69 -19
View File
@@ -6,6 +6,8 @@ using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Models;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using Microsoft.Extensions.Logging;
using MQTTnet;
@@ -14,10 +16,13 @@ namespace FinlyticCore.Util;
/// <summary>
/// An abstract, resilient MQTT client wrapper designed for microservice architectures.
/// Handles automatic reconnection, structured JSON publishing, thread-safe subscription management, and synchronous Request-Reply (RPC).
/// Supports channel-controlled logging via <see cref="CoreSettingKeys.MqttChannel"/>.
/// </summary>
public abstract class ManagedMqttClient : IDisposable
{
private readonly ILogger<ManagedMqttClient> _logger;
private readonly ISettingsService? _settingsService;
private readonly IFinlyticLogger<ManagedMqttClient>? _finlyticLogger;
private readonly IMqttClient _mqttClient;
private CancellationTokenSource? _cts;
@@ -29,15 +34,58 @@ public abstract class ManagedMqttClient : IDisposable
/// </summary>
public bool IsConnected => _mqttClient.IsConnected;
protected ManagedMqttClient(ILogger<ManagedMqttClient> logger)
protected ManagedMqttClient(
ILogger<ManagedMqttClient> logger,
ISettingsService? settingsService = null,
IFinlyticLogger<ManagedMqttClient>? finlyticLogger = null)
{
_logger = logger;
_settingsService = settingsService;
_finlyticLogger = finlyticLogger;
_mqttClient = new MqttClientFactory().CreateMqttClient();
_mqttClient.ApplicationMessageReceivedAsync += HandleIncomingMessageAsync;
_mqttClient.DisconnectedAsync += HandleDisconnectAsync;
}
private async Task LogMqttInfoAsync(string message, params object[] args)
{
if (_finlyticLogger != null)
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.MqttChannel, message, args);
}
else if (_settingsService != null)
{
if (await _settingsService.GetSettingAsync(CoreSettingKeys.MqttChannel))
{
_logger.LogInformation(message, args);
}
}
else
{
_logger.LogInformation(message, args);
}
}
private async Task LogMqttDebugAsync(string message, params object[] args)
{
if (_finlyticLogger != null)
{
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.MqttChannel, message, args);
}
else if (_settingsService != null)
{
if (await _settingsService.GetSettingAsync(CoreSettingKeys.MqttChannel))
{
_logger.LogDebug(message, args);
}
}
else
{
_logger.LogDebug(message, args);
}
}
/// <summary>
/// Establishes a connection to the MQTT broker and initializes the background auto-reconnection loop.
/// </summary>
@@ -61,12 +109,12 @@ public abstract class ManagedMqttClient : IDisposable
var options = optionsBuilder.Build();
_logger.LogInformation("Connecting to MQTT broker at {Host}:{Port}...", config.Host, config.Port);
await LogMqttInfoAsync("Connecting to MQTT broker at {Host}:{Port}...", config.Host, config.Port);
try
{
await _mqttClient.ConnectAsync(options, _cts.Token);
_logger.LogInformation("Successfully connected to MQTT broker.");
await LogMqttInfoAsync("Successfully connected to MQTT broker.");
await OnConnectedAsync();
}
@@ -94,7 +142,7 @@ public abstract class ManagedMqttClient : IDisposable
{
Reason = MqttClientDisconnectOptionsReason.NormalDisconnection
});
_logger.LogInformation("MQTT connection gracefully closed.");
await LogMqttInfoAsync("MQTT connection gracefully closed.");
}
catch (Exception ex)
{
@@ -127,7 +175,7 @@ public abstract class ManagedMqttClient : IDisposable
.Build();
await _mqttClient.SubscribeAsync(subscribeOptions, CancellationToken.None);
_logger.LogDebug("Successfully subscribed to topic: {Topic} (NoLocal: {NoLocal})", topic, noLocal);
await LogMqttDebugAsync("Successfully subscribed to topic: {Topic} (NoLocal: {NoLocal})", topic, noLocal);
}
/// <summary>
@@ -180,15 +228,21 @@ public abstract class ManagedMqttClient : IDisposable
return _mqttClient.PublishAsync(message, CancellationToken.None);
}
/// <summary>
/// Sends a parameterless request to an RPC channel and asynchronously blocks until a matching response arrives.
/// </summary>
public Task<TResponse?> SendRpcRequestAsync<TResponse>(
string channel,
TimeSpan? timeout = null)
where TResponse : class
{
return SendRpcRequestAsync<TResponse, string>(channel, string.Empty, timeout);
}
/// <summary>
/// Sends a generic request payload to an RPC channel and asynchronously blocks until a matching response arrives.
/// Uses the topic conventions: <c>services/request/{channel}/{correlationId}</c> and <c>services/response/{channel}/{correlationId}</c>.
/// </summary>
/// <typeparam name="TResponse">The expected strongly-typed object type of the reply.</typeparam>
/// <typeparam name="TRequest">The type of the payload being transmitted.</typeparam>
/// <param name="channel">The target sub-channel or service name (e.g., "sentix", "assets").</param>
/// <param name="requestData">The object that will be serialized to JSON and sent.</param>
/// <param name="timeout">Optional. Maximum time to wait before returning null. Defaults to 10 seconds.</param>
public async Task<TResponse?> SendRpcRequestAsync<TResponse, TRequest>(
string channel,
TRequest requestData,
@@ -209,7 +263,7 @@ public abstract class ManagedMqttClient : IDisposable
// 2. Serialize and dispatch via the existing JSON helper
await PublishAsync(requestTopic, requestData);
_logger.LogInformation("RPC request published to '{Topic}' [CorrelationId: {Id}]", requestTopic, correlationId);
await LogMqttInfoAsync("RPC request published to '{Topic}' [CorrelationId: {Id}]", requestTopic, correlationId);
try
{
@@ -250,7 +304,7 @@ public abstract class ManagedMqttClient : IDisposable
{
var topic = e.ApplicationMessage.Topic;
var payload = Encoding.UTF8.GetString(e.ApplicationMessage.Payload);
_logger.LogInformation("MQTT message received on topic '{Topic}', length={Length}", topic, payload?.Length ?? 0);
await LogMqttDebugAsync("MQTT message received on topic '{Topic}', length={Length}", topic, payload?.Length ?? 0);
// Intercept message if it belongs to the RPC response convention
if (topic.StartsWith("services/response/"))
@@ -282,7 +336,6 @@ public abstract class ManagedMqttClient : IDisposable
private async Task HandleDisconnectAsync(MqttClientDisconnectedEventArgs e)
{
// Prevent trigger during deliberate connection shutdowns
if (_cts == null || _cts.IsCancellationRequested)
return;
@@ -296,19 +349,19 @@ public abstract class ManagedMqttClient : IDisposable
try
{
_logger.LogInformation("Reconnect attempt {Attempt} in {Delay}s...", attempt, delaySeconds);
await LogMqttInfoAsync("Reconnect attempt {Attempt} in {Delay}s...", attempt, delaySeconds);
await Task.Delay(TimeSpan.FromSeconds(delaySeconds), _cts.Token);
await _mqttClient.ReconnectAsync(_cts.Token);
if (_mqttClient.IsConnected)
{
_logger.LogInformation("MQTT client reconnected successfully after {Attempt} attempt(s).", attempt);
await LogMqttInfoAsync("MQTT client reconnected successfully after {Attempt} attempt(s).", attempt);
await OnConnectedAsync();
return;
}
}
catch (OperationCanceledException) { return; /* Expected on application shutdown */ }
catch (OperationCanceledException) { return; }
catch (Exception ex)
{
_logger.LogWarning(ex, "Reconnect attempt {Attempt} to the MQTT broker failed.", attempt);
@@ -318,15 +371,12 @@ public abstract class ManagedMqttClient : IDisposable
/// <summary>
/// Fired automatically whenever a connection or reconnection is successfully established.
/// Ideal place to trigger <see cref="SubscribeAsync"/> operations.
/// </summary>
protected abstract Task OnConnectedAsync();
/// <summary>
/// Fired whenever a new message lands on a registered subscription channel.
/// </summary>
/// <param name="topic">The specific topic where the message was broadcasted.</param>
/// <param name="payload">The deserialized UTF-8 payload string.</param>
protected abstract Task OnMessageReceivedAsync(string topic, string payload);
/// <summary>