Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(fundamentals): refactor entities, add Yahoo modules scraper, 52W metrics and migrations

Browse Source
This commit is contained in:
Kleidukos
2026-08-14 23:55:15 +02:00
parent 4f733bf8c3
commit 1447f0aa4c
18 changed files with 2565 additions and 1562 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticFundamentals/Util/FundamentalsMqttClient.cs
+37 -76
View File
@@ -1,11 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos;
using FinlyticCore.Models;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
@@ -18,18 +19,15 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
{
private readonly ILogger<FundamentalsMqttClient> _logger;
private readonly IConfiguration _configuration;
private readonly IFundamentalsDbService _dbService;
private readonly IServiceScopeFactory _scopeFactory;
public FundamentalsMqttClient(
ILogger<FundamentalsMqttClient> logger,
IConfiguration configuration,
IFundamentalsDbService dbService,
IServiceScopeFactory scopeFactory) : base(logger)
{
_logger = logger;
_configuration = configuration;
_dbService = dbService;
_scopeFactory = scopeFactory;
}
@@ -43,7 +41,7 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
ClientId = _configuration["MQTT:ClientId"] ?? "finlytic_fundamentals_" + Guid.NewGuid().ToString("N")
};
_logger.LogInformation("[{Channel}] Starting Fundamentals MQTT client. Host: {Host}, ClientId: {ClientId}", "FundamentalsChannel", config.Host, config.ClientId);
_logger.LogInformation("[{Channel}] [MQTT_Client] Starting Fundamentals MQTT client. Host: {Host}, ClientId: {ClientId}", "MqttChannel", config.Host, config.ClientId);
await ConnectAsync(config);
}
@@ -51,19 +49,18 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
/// <inheritdoc />
public async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("[{Channel}] Stopping Fundamentals MQTT client.", "FundamentalsChannel");
_logger.LogInformation("[{Channel}] [MQTT_Client] Stopping Fundamentals MQTT client.", "MqttChannel");
await DisconnectAsync();
}
/// <inheritdoc />
protected override async Task OnConnectedAsync()
{
_logger.LogInformation("[{Channel}] Fundamentals MQTT client connected. Subscribing to RPC request topics...", "FundamentalsChannel");
_logger.LogInformation("[{Channel}] [MQTT_Client] Connected. Subscribing to RPC request topics...", "MqttChannel");
await SubscribeAsync("services/request/fundamentals_Get/#");
await SubscribeAsync("services/request/events_GetAll/#");
await SubscribeAsync("services/request/events_GetByMonth/#");
await SubscribeAsync("services/request/health_Ping/#");
await SubscribeAsync("services/config/updated/#");
}
/// <inheritdoc />
@@ -71,23 +68,11 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
{
if (string.IsNullOrWhiteSpace(topic)) return;
// 1. Config update events
if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase))
{
if (topic.EndsWith("FinlyticFundamentals", StringComparison.OrdinalIgnoreCase))
{
await OnConfigUpdatedAsync(payload);
}
return;
}
// Extract correlationId from topic suffix (e.g. services/request/fundamentals_Get/{correlationId})
var lastSlash = topic.LastIndexOf('/');
if (lastSlash < 0 || lastSlash >= topic.Length - 1) return;
var correlationId = topic.Substring(lastSlash + 1);
// 2. Dispatch to specific channel handlers
if (topic.StartsWith("services/request/fundamentals_Get", StringComparison.OrdinalIgnoreCase))
{
await OnFundamentalsGetAsync(payload, correlationId);
@@ -106,14 +91,15 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
}
}
/// <summary>
/// Handles fundamentals_Get RPC requests using source-generated DTO deserialization.
/// </summary>
private async Task OnFundamentalsGetAsync(string payload, string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
var dbService = scope.ServiceProvider.GetRequiredService<IFundamentalsDbService>();
if (string.IsNullOrWhiteSpace(payload))
{
_logger.LogWarning("[{Channel}] [FundamentalsMqttClient] Received empty payload for fundamentals_Get request.", "FundamentalsChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Received empty payload for fundamentals_Get request.");
return;
}
@@ -122,108 +108,83 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
var request = (IsinRequest?)JsonSerializer.Deserialize(payload, typeof(IsinRequest), FinlyticJsonSerializerContext.Default);
if (request == null || string.IsNullOrWhiteSpace(request.Isin))
{
_logger.LogWarning("[{Channel}] [FundamentalsMqttClient] Request missing mandatory ISIN parameter in payload.", "FundamentalsChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Request missing mandatory ISIN parameter in payload.");
return;
}
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Processing RPC fundamentals_Get for ISIN '{Isin}' (forceRefresh={ForceRefresh}) [CorrelationId: {CorrelationId}]",
"FundamentalsChannel", request.Isin, request.ForceRefresh.ToString(), correlationId);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC fundamentals_Get for ISIN '{Isin}' (forceRefresh={ForceRefresh}) [CorrelationId: {CorrelationId}]",
request.Isin, request.ForceRefresh.ToString(), correlationId);
var fundamentals = await _dbService.GetFundamentalsAsync(request.Isin, request.Ticker, request.ForceRefresh);
var fundamentals = await dbService.GetFundamentalsAsync(request.Isin, request.Ticker, request.ForceRefresh);
var responseTopic = $"services/response/fundamentals_Get/{correlationId}";
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Publishing RPC fundamentals response to '{ResponseTopic}'", "FundamentalsChannel", responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Publishing RPC fundamentals response to '{ResponseTopic}'", responseTopic);
await PublishAsync(responseTopic, fundamentals);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Failed to process fundamentals_Get request.", "FundamentalsChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [MQTT_Client] Failed to process fundamentals_Get request.");
}
}
/// <summary>
/// Handles events_GetAll RPC requests.
/// </summary>
private async Task OnEventsGetAllAsync(string correlationId)
{
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Processing RPC events_GetAll request [CorrelationId: {CorrelationId}]", "FundamentalsChannel", correlationId);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
var dbService = scope.ServiceProvider.GetRequiredService<IFundamentalsDbService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC events_GetAll request [CorrelationId: {CorrelationId}]", correlationId);
try
{
var events = await _dbService.GetAllEventsAsync();
var events = await dbService.GetAllEventsAsync();
var responseTopic = $"services/response/events_GetAll/{correlationId}";
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Publishing events RPC response to '{ResponseTopic}'", "FundamentalsChannel", responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Publishing events RPC response to '{ResponseTopic}'", responseTopic);
await PublishAsync(responseTopic, events);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Failed to process events_GetAll request.", "FundamentalsChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [MQTT_Client] Failed to process events_GetAll request.");
}
}
/// <summary>
/// Handles events_GetByMonth RPC requests.
/// </summary>
private async Task OnEventsGetByMonthAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
var dbService = scope.ServiceProvider.GetRequiredService<IFundamentalsDbService>();
try
{
var request = (GetEventsByMonthRequest?)JsonSerializer.Deserialize(payload, typeof(GetEventsByMonthRequest), FinlyticJsonSerializerContext.Default);
if (request == null) return;
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Processing RPC events_GetByMonth request for {Year}/{Month} [CorrelationId: {CorrelationId}]", "FundamentalsChannel", request.Year, request.Month, correlationId);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC events_GetByMonth request for {Year}/{Month} [CorrelationId: {CorrelationId}]", request.Year, request.Month, correlationId);
var events = await _dbService.GetEventsByMonthAsync(request.Year, request.Month);
var events = await dbService.GetEventsByMonthAsync(request.Year, request.Month);
var responseTopic = $"services/response/events_GetByMonth/{correlationId}";
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Publishing monthly events RPC response to '{ResponseTopic}'", "FundamentalsChannel", responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Publishing monthly events RPC response to '{ResponseTopic}'", responseTopic);
await PublishAsync(responseTopic, events);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Failed to process events_GetByMonth request.", "FundamentalsChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [MQTT_Client] Failed to process events_GetByMonth request.");
}
}
/// <summary>
/// Handles health_Ping RPC requests.
/// </summary>
private async Task OnHealthPingAsync(string topic, string correlationId)
{
if (topic.Contains("FinlyticFundamentals", StringComparison.OrdinalIgnoreCase) || !topic.Contains("/", StringComparison.OrdinalIgnoreCase))
{
string respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticFundamentals", "Online", DateTime.UtcNow, "Connected"));
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", "FundamentalsChannel", correlationId);
}
}
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
/// <summary>
/// Handles dynamic service config update events.
/// </summary>
private async Task OnConfigUpdatedAsync(string payload)
{
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Received config update event for FinlyticFundamentals.", "FundamentalsChannel");
try
{
using var doc = JsonDocument.Parse(payload);
if (doc.RootElement.TryGetProperty("settings", out var settingsProp))
{
var dict = (Dictionary<string, string>?)JsonSerializer.Deserialize(settingsProp.GetRawText(), typeof(Dictionary<string, string>), FinlyticJsonSerializerContext.Default);
if (dict != null && dict.Count > 0)
{
using var scope = _scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsDb.UpdateSettingsFromDictionaryAsync(dict);
_logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Persisted {Count} updated settings to FinlyticFundamentals database.", "FundamentalsChannel", dict.Count);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Error processing MQTT config update event.", "FundamentalsChannel");
var respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticFundamentals", "Online", DateTime.UtcNow, "Connected"));
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[FinlyticFundamentals] [Health_Ping] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
}
}
}
FinlyticFundamentals/Util/SettingKeys.cs
+14 -2
View File
@@ -1,6 +1,18 @@
namespace FinlyticFundamentals.Util;
using FinlyticCore.Models.Settings;
namespace FinlyticFundamentals.Util;
public class SettingKeys
{
// --- Logging Channels ---
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> FundamentalsChannel = new("Logging.Channel.Fundamentals", true);
public static readonly SettingKey<bool> HtmlScrapperChannel = new("Logging.Channel.HtmlScrapper", true);
public static readonly SettingKey<bool> YahooClientChannel = new("Logging.Channel.YahooClient", true);
// --- Features & Toggles ---
public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true);
public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true);
public static readonly SettingKey<int> FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30);
}