Compare commits

...

11 Commits

Author SHA1 Message Date
Kleidukos 2ba54e8057 feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings 2026-08-16 14:05:57 +02:00
Kleidukos b0f8d4b78b feat(app): live terminal log console widget and service dynamic settings management 2026-08-15 21:31:00 +02:00
Kleidukos f18f75c1ab feat(backend): generic settings RPC bridge, LogStreamHub SignalR, and log ringbuffer 2026-08-15 21:30:56 +02:00
Kleidukos 1f9d66405a feat(assets): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:30:46 +02:00
Kleidukos 57554a9582 feat(trades): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:30:19 +02:00
Kleidukos 0d370d09e7 feat(analyzer): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:30:16 +02:00
Kleidukos 62e030e2cf feat(sentiment): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:30:12 +02:00
Kleidukos 1522c3480f feat(ta): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:30:05 +02:00
Kleidukos a94c36a878 feat(news): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:30:01 +02:00
Kleidukos a1f2b888f6 feat(fundamentals): dynamic settings, IFinlyticLogger, live log streaming, and EF migration 2026-08-15 21:29:50 +02:00
Kleidukos 3dbee36ca0 feat(core): dynamic settings service, IFinlyticLogger, log broadcaster, and persistent Yahoo auth 2026-08-15 21:29:38 +02:00
105 changed files with 7859 additions and 2684 deletions
@@ -5,11 +5,12 @@ using System.Threading.Tasks;
using FinlyticAnalyzer.Database;
using FinlyticAnalyzer.Entities;
using FinlyticAnalyzer.Services;
using FinlyticAnalyzer.Util;
using FinlyticCore.Models.Analyzer;
using FinlyticCore.Models.Trades;
using FinlyticCore.Services;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace FinlyticAnalyzer.Controllers;
@@ -36,20 +37,20 @@ public class ManualAnalysisController : ControllerBase
private readonly IN8nEvaluationService _n8nService;
private readonly IWinRateCalculator _winRateCalculator;
private readonly AnalyzerDbContext _dbContext;
private readonly ILogger<ManualAnalysisController> _logger;
private readonly IFinlyticLogger<ManualAnalysisController> _finlyticLogger;
public ManualAnalysisController(
IVixTrackerService vixTracker,
IN8nEvaluationService n8nService,
IWinRateCalculator winRateCalculator,
AnalyzerDbContext dbContext,
ILogger<ManualAnalysisController> logger)
IFinlyticLogger<ManualAnalysisController> finlyticLogger)
{
_vixTracker = vixTracker;
_n8nService = n8nService;
_winRateCalculator = winRateCalculator;
_dbContext = dbContext;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -181,6 +182,8 @@ public class ManualAnalysisController : ControllerBase
_dbContext.Analyses.Add(analysisEntity);
await _dbContext.SaveChangesAsync(cancellationToken);
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ManualAnalysisController] Manual analysis completed for {Symbol} (TradeProposed: {Proposed})", request.Symbol, shouldProceed);
if (!shouldProceed)
{
return Ok(new
+14 -2
View File
@@ -1,10 +1,12 @@
using FinlyticAnalyzer.Entities;
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticAnalyzer.Database;
public class AnalyzerDbContext : DbContext
public class AnalyzerDbContext : DbContext, ISettingsDbContext
{
public AnalyzerDbContext(DbContextOptions<AnalyzerDbContext> options) : base(options) { }
@@ -20,7 +22,7 @@ public class AnalyzerDbContext : DbContext
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key);
entity.HasIndex(e => e.Key).IsUnique();
});
modelBuilder.Entity<AnalysisEntity>(entity =>
@@ -39,3 +41,13 @@ public class AnalyzerDbContext : DbContext
});
}
}
public class AnalyzerDbContextFactory : IDesignTimeDbContextFactory<AnalyzerDbContext>
{
public AnalyzerDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<AnalyzerDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=analyzer;Username=postgres;Password=postgres");
return new AnalyzerDbContext(optionsBuilder.Options);
}
}
@@ -0,0 +1,308 @@
// <auto-generated />
using System;
using FinlyticAnalyzer.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticAnalyzer.Migrations
{
[DbContext(typeof(AnalyzerDbContext))]
[Migration("20260815184017_AddDynamicSettings")]
partial class AddDynamicSettings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticAnalyzer.Entities.AnalysisEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AiOutputJson")
.IsRequired()
.HasColumnType("jsonb");
b.Property<string>("AnalysisId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<DateTime>("CreatedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("EventId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<double>("ImpactScore")
.HasColumnType("double precision");
b.Property<bool>("IsTradeProposed")
.HasColumnType("boolean");
b.Property<string>("Isin")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<string>("N8nDecision")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<double>("N8nEvalScore")
.HasColumnType("double precision");
b.Property<string>("N8nResponseJson")
.IsRequired()
.HasColumnType("jsonb");
b.Property<string>("RawDataJson")
.IsRequired()
.HasColumnType("jsonb");
b.Property<string>("Sector")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<string>("Symbol")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<int>("VixRegime")
.HasColumnType("integer");
b.Property<decimal>("VixValue")
.HasColumnType("numeric");
b.Property<double>("WinRate")
.HasColumnType("double precision");
b.HasKey("Id");
b.HasIndex("AnalysisId")
.IsUnique();
b.HasIndex("CreatedAt");
b.HasIndex("EventId");
b.HasIndex("Isin");
b.HasIndex("Sector");
b.ToTable("analyses");
});
modelBuilder.Entity("FinlyticAnalyzer.Entities.AnalyzerSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<bool>("EnableLogAnalyzerAuto")
.HasColumnType("boolean");
b.Property<bool>("EnableLogAnalyzerManual")
.HasColumnType("boolean");
b.Property<bool>("EnableLogDatabaseOps")
.HasColumnType("boolean");
b.Property<bool>("EnableLogMqttGeneral")
.HasColumnType("boolean");
b.Property<bool>("EnableLogMqttHealthPing")
.HasColumnType("boolean");
b.Property<double>("MinSignalScore")
.HasColumnType("double precision");
b.Property<string>("ScanCronSchedule")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticAnalyzer.Entities.TradeProposalEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AnalysisId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<double>("ConfidenceScore")
.HasColumnType("double precision");
b.Property<DateTime>("CreatedAt")
.HasColumnType("timestamp with time zone");
b.Property<decimal>("EntryPrice")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("EntryZoneMax")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("EntryZoneMin")
.HasColumnType("decimal(18,4)");
b.Property<string>("EventId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<DateTime>("ExpiresAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("FundamentalRationale")
.IsRequired()
.HasColumnType("text");
b.Property<string>("InstrumentType")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<string>("Isin")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<decimal?>("MaxLeverage")
.HasColumnType("decimal(18,4)");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<string>("ProposedAction")
.IsRequired()
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<string>("ReasonSummary")
.IsRequired()
.HasColumnType("text");
b.Property<decimal?>("RiskRewardRatio")
.HasColumnType("decimal(18,4)");
b.Property<string>("RiskTolerance")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<string>("RiskWarning")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Sector")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<decimal>("StopLoss")
.HasColumnType("decimal(18,4)");
b.Property<string>("Symbol")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<decimal>("TakeProfit")
.HasColumnType("decimal(18,4)");
b.Property<string>("TakeProfitTargets")
.HasColumnType("text");
b.Property<string>("TechnicalRationale")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Timeframe")
.IsRequired()
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<int>("Type")
.HasColumnType("integer");
b.Property<int>("VixRegime")
.HasColumnType("integer");
b.Property<decimal>("VixValue")
.HasColumnType("decimal(18,4)");
b.Property<double>("WinRate")
.HasColumnType("double precision");
b.HasKey("Id");
b.HasIndex("ExpiresAt");
b.HasIndex("Isin");
b.ToTable("trade_proposals");
});
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,43 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticAnalyzer.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DynamicSettings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
ValueJson = table.Column<string>(type: "text", nullable: false),
ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DynamicSettings", x => x.Id);
});
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DynamicSettings");
}
}
}
@@ -268,6 +268,37 @@ namespace FinlyticAnalyzer.Migrations
b.ToTable("trade_proposals");
});
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
#pragma warning restore 612, 618
}
}
+11 -5
View File
@@ -2,14 +2,24 @@ using System;
using FinlyticAnalyzer.Database;
using FinlyticAnalyzer.Services;
using FinlyticAnalyzer.Util;
using FinlyticCore.Database;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
var builder = Host.CreateApplicationBuilder(args);
// Register DB Context
builder.Services.AddDbContext<AnalyzerDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddScoped<ISettingsDbContext>(sp => sp.GetRequiredService<AnalyzerDbContext>());
// Register Core Services
builder.Services.AddSingleton<ISettingsService, SettingsService>();
builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<>));
// Register HTTP Clients for external webhooks (HttpClientFactory manages pool)
builder.Services.AddHttpClient<IN8nEvaluationService, N8nEvaluationService>();
@@ -38,14 +48,10 @@ using (var scope = host.Services.CreateScope())
var context = scope.ServiceProvider.GetRequiredService<AnalyzerDbContext>();
await context.Database.MigrateAsync();
Console.WriteLine("Database migrations successfully executed for FinlyticAnalyzer.");
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsService.GetSettingsAsync();
}
catch (Exception ex)
{
var logger = scope.ServiceProvider.GetRequiredService<ILogger<Program>>();
logger.LogError(ex, "An error occurred during database migration for FinlyticAnalyzer on startup.");
Console.WriteLine($"Critical error during database migration for FinlyticAnalyzer: {ex.Message}");
}
}
@@ -9,30 +9,32 @@ using FinlyticCore.Dtos;
using FinlyticCore.Dtos.TechnicalAnalysis;
using FinlyticCore.Models.Analyzer;
using FinlyticCore.Models.Trades;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticAnalyzer.Services;
public class ActiveTradeMonitorWorker : BackgroundService
{
private readonly ILogger<ActiveTradeMonitorWorker> _logger;
private readonly IFinlyticLogger<ActiveTradeMonitorWorker> _finlyticLogger;
private readonly IServiceScopeFactory _scopeFactory;
private readonly AnalyzerMqttClient _mqttClient;
public ActiveTradeMonitorWorker(ILogger<ActiveTradeMonitorWorker> logger, IServiceScopeFactory scopeFactory,
public ActiveTradeMonitorWorker(
IFinlyticLogger<ActiveTradeMonitorWorker> finlyticLogger,
IServiceScopeFactory scopeFactory,
AnalyzerMqttClient mqttClient)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_scopeFactory = scopeFactory;
_mqttClient = mqttClient;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("[{Channel}] ActiveTradeMonitorWorker started.", "AnalyzerChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] ActiveTradeMonitorWorker started.");
try
{
@@ -51,7 +53,7 @@ public class ActiveTradeMonitorWorker : BackgroundService
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
_logger.LogError(ex, "[{Channel}] Error in ActiveTradeMonitorWorker loop.", "AnalyzerChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[ActiveTradeMonitorWorker] Error in ActiveTradeMonitorWorker loop.");
}
try
@@ -64,24 +66,22 @@ public class ActiveTradeMonitorWorker : BackgroundService
}
}
_logger.LogInformation("[{Channel}] ActiveTradeMonitorWorker stopped.", "AnalyzerChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] ActiveTradeMonitorWorker stopped.");
}
private async Task MonitorActiveTradesAsync(CancellationToken cancellationToken)
{
if (!_mqttClient.IsConnected)
{
_logger.LogWarning("[{Channel}] Skipping trade monitoring. RPC client not connected.", "AnalyzerChannel");
await _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] Skipping trade monitoring. RPC client not connected.");
return;
}
// Fetch active trades
var activeTrades = await _mqttClient.SendRpcRequestAsync<List<TradeProposalDto>, GetTradesRequest>(
"trades_Get",
new GetTradesRequest(null, "Active"),
TimeSpan.FromSeconds(10));
// Fetch proposed global trades
var proposedTrades = await _mqttClient.SendRpcRequestAsync<List<TradeProposalDto>, GetTradesRequest>(
"trades_Get",
new GetTradesRequest(null, "Proposed"),
@@ -93,13 +93,11 @@ public class ActiveTradeMonitorWorker : BackgroundService
if (trades.Count == 0)
{
_logger.LogInformation("[{Channel}] No active or proposed global trades found to monitor.",
"AnalyzerChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] No active or proposed global trades found to monitor.");
return;
}
_logger.LogInformation("[{Channel}] Found {Count} trades to monitor. Starting evaluation...", "AnalyzerChannel",
trades.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] Found {Count} trades to monitor. Starting evaluation...", trades.Count);
using var scope = _scopeFactory.CreateScope();
var n8nService = scope.ServiceProvider.GetRequiredService<IN8nEvaluationService>();
@@ -115,8 +113,7 @@ public class ActiveTradeMonitorWorker : BackgroundService
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to monitor trade {TradeId} ({Symbol}).", "AnalyzerChannel",
trade.TradeId, trade.Symbol);
await _finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[ActiveTradeMonitorWorker] Failed to monitor trade {TradeId} ({Symbol}).", trade.TradeId, trade.Symbol);
}
}
}
@@ -124,22 +121,18 @@ public class ActiveTradeMonitorWorker : BackgroundService
private async Task ProcessTradeAsync(TradeProposalDto trade, IN8nEvaluationService n8nService,
IVixTrackerService vixService, CancellationToken cancellationToken)
{
// 1. Get Live Price
var livePriceReq = new IsinRequest(trade.Isin);
var livePriceDto = await _mqttClient.SendRpcRequestAsync<LivePriceDto, IsinRequest>(
"tr_GetLivePrice", livePriceReq, TimeSpan.FromSeconds(3));
decimal currentPrice = livePriceDto?.CurrentPrice > 0 ? livePriceDto.CurrentPrice : trade.EntryPrice;
// 2. Evaluate Hard Stops (StopLoss / TakeProfit / TimeStop)
bool isLong = string.Equals(trade.SignalType, "BUY", StringComparison.OrdinalIgnoreCase) ||
string.Equals(trade.SignalType, "LONG", StringComparison.OrdinalIgnoreCase);
// Time-Stop Evaluierung
int maxHoldingDays = EstimateMaxHoldingDays(trade.Timeframe);
double daysOpen = (DateTime.UtcNow - trade.CreatedAt).TotalDays;
// 50% Grace Period. Bei z.B. 10 Tagen max. Haltedauer wird nach 15 Tagen ohne Zielerreichung glattgestellt.
if (daysOpen > (maxHoldingDays * 1.5))
{
await SendUpdateAsync(trade, currentPrice, "Close",
@@ -176,7 +169,6 @@ public class ActiveTradeMonitorWorker : BackgroundService
}
}
// 3. Run AI evaluation for soft/dynamic updates
var taResult = await _mqttClient.SendRpcRequestAsync<TechnicalAnalysisDto, IsinRequest>(
"ta_GetAnalysis", livePriceReq, TimeSpan.FromSeconds(5));
@@ -225,8 +217,7 @@ public class ActiveTradeMonitorWorker : BackgroundService
var aiResponse = await n8nService.EvaluateAssetAsync(n8nReq, cancellationToken);
if (aiResponse == null)
{
_logger.LogWarning("[{Channel}] AI evaluation returned null for {TradeId}. Skipping update.",
"AnalyzerChannel", trade.TradeId);
await _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] AI evaluation returned null for {TradeId}. Skipping update.", trade.TradeId);
return;
}
@@ -235,7 +226,6 @@ public class ActiveTradeMonitorWorker : BackgroundService
decimal? newStopLoss = trade.StopLoss;
decimal? newTakeProfit = trade.TakeProfit;
// Check for trend reversal
bool aiSuggestsShort =
string.Equals(aiResponse.SuggestedDirection, "Short", StringComparison.OrdinalIgnoreCase) ||
string.Equals(aiResponse.SuggestedDirection, "Sell", StringComparison.OrdinalIgnoreCase);
@@ -256,13 +246,11 @@ public class ActiveTradeMonitorWorker : BackgroundService
}
else if (aiResponse.ExecutionPlan != null)
{
// Ratchet / Trailing Logic: StopLoss darf das Risiko nicht vergrößern!
if (aiResponse.ExecutionPlan.StopLoss > 0)
{
var proposedSl = aiResponse.ExecutionPlan.StopLoss;
if (isLong)
{
// Bei Long darf der StopLoss nur NACH OBEN angepasst werden
if (trade.StopLoss <= 0 || proposedSl > trade.StopLoss)
{
newStopLoss = proposedSl;
@@ -271,7 +259,6 @@ public class ActiveTradeMonitorWorker : BackgroundService
}
else
{
// Bei Short darf der StopLoss nur NACH UNTEN angepasst werden
if (trade.StopLoss <= 0 || proposedSl < trade.StopLoss)
{
newStopLoss = proposedSl;
@@ -310,18 +297,16 @@ public class ActiveTradeMonitorWorker : BackgroundService
Timestamp = DateTime.UtcNow
};
// Direktes Objekt-Publishing nutzen (ManagedMqttClient serialisiert typgerecht)
string topic = $"finlytic/trades/updates/{trade.Isin}";
await _mqttClient.PublishAsync(topic, update);
_logger.LogInformation(
"[{Channel}] Published trade update for {TradeId} to topic {Topic}. Recommendation: {Rec}, Reasoning: {Reason}",
"AnalyzerChannel", trade.TradeId, topic, recommendation, reasoning);
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ActiveTradeMonitorWorker] Published trade update for {TradeId} to topic {Topic}. Recommendation: {Rec}, Reasoning: {Reason}",
trade.TradeId, topic, recommendation, reasoning);
}
private static int EstimateMaxHoldingDays(string timeframe)
{
if (string.IsNullOrWhiteSpace(timeframe)) return 14; // Default
if (string.IsNullOrWhiteSpace(timeframe)) return 14;
string tfLower = timeframe.ToLowerInvariant();
int multiplier = 1;
@@ -351,7 +336,7 @@ public class ActiveTradeMonitorWorker : BackgroundService
int maxNum = numbers.Count > 0 ? numbers.Max() : 14;
if (maxNum == 0) maxNum = 14;
if (multiplier == 1 && maxNum < 3) maxNum = 3; // Mindestens 3 Tage Kulanz
if (multiplier == 1 && maxNum < 3) maxNum = 3;
return maxNum * multiplier;
}
@@ -1,26 +1,33 @@
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAnalyzer.Util;
using FinlyticCore.Models.Analyzer;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.Extensions.Configuration;
namespace FinlyticAnalyzer.Services;
public class N8nEvaluationService : IN8nEvaluationService
{
private readonly HttpClient _httpClient;
private readonly ILogger<N8nEvaluationService> _logger;
private readonly IFinlyticLogger<N8nEvaluationService> _finlyticLogger;
private readonly string _webhookUrl;
public N8nEvaluationService(HttpClient httpClient, IConfiguration configuration, ILogger<N8nEvaluationService> logger)
public N8nEvaluationService(HttpClient httpClient, IConfiguration configuration, IFinlyticLogger<N8nEvaluationService> finlyticLogger)
{
_httpClient = httpClient;
_logger = logger;
_finlyticLogger = finlyticLogger;
_webhookUrl = configuration["N8N:WebhookUrl"] ?? configuration["N8N__WebhookUrl"] ?? string.Empty;
if (string.IsNullOrWhiteSpace(_webhookUrl))
{
_logger.LogWarning("[{Channel}] N8N:WebhookUrl configuration is missing or empty.", "AnalyzerChannel");
_ = _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[N8nEvaluationService] N8N:WebhookUrl configuration is missing or empty.");
}
// Timeout auf 45 Sekunden erhöht für komplexere LLM/Gemini Chains in n8n
_httpClient.Timeout = TimeSpan.FromSeconds(45);
}
@@ -31,16 +38,15 @@ public class N8nEvaluationService : IN8nEvaluationService
{
if (string.IsNullOrWhiteSpace(_webhookUrl))
{
_logger.LogError("[{Channel}] Cannot execute AI evaluation for {Symbol}: N8N:WebhookUrl is not configured.", "AnalyzerChannel", request.TargetAsset.Symbol);
await _finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, "[N8nEvaluationService] Cannot execute AI evaluation for {Symbol}: N8N:WebhookUrl is not configured.", request.TargetAsset.Symbol);
return null;
}
try
{
_logger.LogInformation("[{Channel}] Sending n8n AI Evaluation request {RequestId} for Asset {Symbol} (ISIN: {Isin}) to {Url}...",
"AnalyzerChannel", request.RequestId, request.TargetAsset.Symbol, request.TargetAsset.Isin, _webhookUrl);
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[N8nEvaluationService] Sending n8n AI Evaluation request {RequestId} for Asset {Symbol} (ISIN: {Isin}) to {Url}...",
request.RequestId, request.TargetAsset.Symbol, request.TargetAsset.Isin, _webhookUrl);
// Typsichere AOT-Serialisierung verwenden
using var content = JsonContent.Create(
request,
FinlyticJsonSerializerContext.Default.N8nAnalysisRequestDto);
@@ -53,11 +59,10 @@ public class N8nEvaluationService : IN8nEvaluationService
if (string.IsNullOrWhiteSpace(contentStr) || contentStr.Trim() == "{}" || contentStr.Trim() == "[]")
{
_logger.LogWarning("[{Channel}] n8n Webhook returned an EMPTY response for Request {RequestId}. Flagging as AI Rejection (Too Risky).", "AnalyzerChannel", request.RequestId);
await _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[N8nEvaluationService] n8n Webhook returned an EMPTY response for Request {RequestId}. Flagging as AI Rejection (Too Risky).", request.RequestId);
return CreateRejectionFallback(request, "Die KI (n8n/Gemini) stuft den Trade als zu riskant ein und empfiehlt keine Positionierung.");
}
// N8n schickt Ergebnisse manchmal als JSON-Array [{...}] zurück
string jsonToDeserialize = contentStr.Trim();
if (jsonToDeserialize.StartsWith('[') && jsonToDeserialize.EndsWith(']'))
{
@@ -74,28 +79,28 @@ public class N8nEvaluationService : IN8nEvaluationService
if (responseDto != null && !string.IsNullOrWhiteSpace(responseDto.AiDecision))
{
_logger.LogInformation("[{Channel}] Received n8n AI Response for Request {RequestId}: Decision={Decision}, Score={Score:F2}, Direction={Direction}, Timeframe={Timeframe}",
"AnalyzerChannel", request.RequestId, responseDto.AiDecision, responseDto.EvalScore, responseDto.SuggestedDirection, responseDto.SuggestedTimeframe);
await _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[N8nEvaluationService] Received n8n AI Response for Request {RequestId}: Decision={Decision}, Score={Score:F2}, Direction={Direction}, Timeframe={Timeframe}",
request.RequestId, responseDto.AiDecision, responseDto.EvalScore, responseDto.SuggestedDirection, responseDto.SuggestedTimeframe);
return responseDto;
}
}
else
{
_logger.LogWarning("[{Channel}] n8n Webhook returned HTTP {StatusCode} for Request {RequestId}",
"AnalyzerChannel", response.StatusCode, request.RequestId);
await _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[N8nEvaluationService] n8n Webhook returned HTTP {StatusCode} for Request {RequestId}",
response.StatusCode, request.RequestId);
}
}
catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogError(ex, "[{Channel}] Timeout waiting for n8n AI Evaluation Webhook (Request {RequestId}). Consider increasing timeout.", "AnalyzerChannel", request.RequestId);
await _finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[N8nEvaluationService] Timeout waiting for n8n AI Evaluation Webhook (Request {RequestId}). Consider increasing timeout.", request.RequestId);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error calling n8n AI Evaluation Webhook for Request {RequestId}", "AnalyzerChannel", request.RequestId);
await _finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[N8nEvaluationService] Error calling n8n AI Evaluation Webhook for Request {RequestId}", request.RequestId);
}
return null; // Signals RPC/Service failure to caller
return null;
}
private static N8nAnalysisResponseDto CreateRejectionFallback(N8nAnalysisRequestDto request, string reasoning)
@@ -1,21 +1,22 @@
using System;
using System.Collections.Concurrent;
using FinlyticAnalyzer.Util;
using FinlyticCore.Dtos.News;
using FinlyticCore.Models.Analyzer;
using Microsoft.Extensions.Logging;
using FinlyticCore.Services;
namespace FinlyticAnalyzer.Services;
public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
{
private readonly ILogger<ThreeLayerFilterEngine> _logger;
private readonly IFinlyticLogger<ThreeLayerFilterEngine> _finlyticLogger;
private readonly ConcurrentDictionary<string, DateTime> _seenEvents = new();
private readonly object _cleanupLock = new();
private DateTime _lastCleanupTime = DateTime.UtcNow;
public ThreeLayerFilterEngine(ILogger<ThreeLayerFilterEngine> logger)
public ThreeLayerFilterEngine(IFinlyticLogger<ThreeLayerFilterEngine> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -25,9 +26,6 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
{
var result = new FilterResult();
// -------------------------------------------------------------
// Layer 1: Relevance, ISIN & Deduplication
// -------------------------------------------------------------
if (newsEvent == null || newsEvent.Id == Guid.Empty)
{
result.Passed = false;
@@ -38,7 +36,6 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
string eventId = newsEvent.Id.ToString();
var now = DateTime.UtcNow;
// Safely clean up dictionary every 30 minutes (thread-safe lock)
if ((now - _lastCleanupTime).TotalMinutes > 30 || _seenEvents.Count > 10000)
{
lock (_cleanupLock)
@@ -50,7 +47,6 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
}
}
// Deduplication check (keep history for 12 hours)
if (_seenEvents.TryGetValue(eventId, out var prevTime) && (now - prevTime).TotalHours < 12.0)
{
result.Passed = false;
@@ -63,7 +59,6 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
string isin = string.Empty;
string assetName = string.Empty;
// Extract parameters strictly from MatchedAssets
if (newsEvent.MatchedAssets != null && newsEvent.MatchedAssets.Count > 0)
{
var firstAsset = newsEvent.MatchedAssets[0];
@@ -71,7 +66,6 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
assetName = !string.IsNullOrWhiteSpace(firstAsset.Name) ? firstAsset.Name.Trim() : string.Empty;
}
// Mandatory check: Must have a valid ISIN
if (string.IsNullOrWhiteSpace(isin))
{
result.Passed = false;
@@ -80,13 +74,9 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
}
result.Isin = isin;
// Asset-Symbol fallback to ISIN, Name is mapped appropriately later
result.Symbol = isin;
result.Sector = "General"; // Will be enriched downstream via Fundamentals RPC if available
result.Sector = "General";
// -------------------------------------------------------------
// Layer 2: Impact & Dynamic VIX Threshold
// -------------------------------------------------------------
double impactScore = newsEvent.Confidence ?? 0.75;
if (impactScore <= 0) impactScore = 0.75;
@@ -106,14 +96,11 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
{
result.Passed = false;
result.RejectReason = $"Layer 2: Impact score ({impactScore:F2}) below dynamic VIX threshold ({requiredThreshold:F2}) for regime {regime}";
_logger.LogInformation("[{Channel}] Event {EventId} (ISIN: {Isin}) rejected by Layer 2 filter. Impact: {Impact:F2}, Threshold: {Threshold:F2}, Regime: {Regime}",
"AnalyzerChannel", eventId, isin, impactScore, requiredThreshold, regime);
_ = _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ThreeLayerFilterEngine] Event {EventId} (ISIN: {Isin}) rejected by Layer 2 filter. Impact: {Impact:F2}, Threshold: {Threshold:F2}, Regime: {Regime}",
eventId, isin, impactScore, requiredThreshold, regime);
return result;
}
// -------------------------------------------------------------
// Layer 3: Dynamic Parameter & Risk Engine
// -------------------------------------------------------------
result.RiskTolerance = regime switch
{
VixMarketRegime.Panic => "Conservative",
@@ -125,8 +112,8 @@ public class ThreeLayerFilterEngine : IThreeLayerFilterEngine
result.InstrumentType = regime == VixMarketRegime.Panic ? "Option" : "Stock";
result.Passed = true;
_logger.LogInformation("[{Channel}] Event {EventId} passed 3-Layer Filter for ISIN {Isin}. Impact: {Impact:F2}, Regime: {Regime}",
"AnalyzerChannel", eventId, result.Isin, impactScore, regime);
_ = _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ThreeLayerFilterEngine] Event {EventId} passed 3-Layer Filter for ISIN {Isin}. Impact: {Impact:F2}, Regime: {Regime}",
eventId, result.Isin, impactScore, regime);
return result;
}
+11 -12
View File
@@ -1,25 +1,26 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAnalyzer.Util;
using FinlyticCore.Models.Analyzer;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using Microsoft.Extensions.Logging;
namespace FinlyticAnalyzer.Services;
public class VixTrackerService : IVixTrackerService
{
private readonly YahooFinanceClient _yahooClient;
private readonly ILogger<VixTrackerService> _logger;
private readonly IFinlyticLogger<VixTrackerService> _finlyticLogger;
private decimal _currentVix = 18.5m; // Default: Normal Regime
private decimal _currentVix = 18.5m;
private VixMarketRegime _currentRegime = VixMarketRegime.Normal;
private readonly object _lock = new();
public VixTrackerService(YahooFinanceClient yahooClient, ILogger<VixTrackerService> logger)
public VixTrackerService(YahooFinanceClient yahooClient, IFinlyticLogger<VixTrackerService> finlyticLogger)
{
_yahooClient = yahooClient;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
public decimal GetCurrentVix()
@@ -52,13 +53,13 @@ public class VixTrackerService : IVixTrackerService
if (oldRegime != _currentRegime)
{
_logger.LogWarning("[{Channel}] [VIX REGIME SHIFT] Markt-Regime gewechselt: {OldRegime} -> {NewRegime} (VIX: {Vix:F2})",
"AnalyzerChannel", oldRegime, _currentRegime, _currentVix);
_ = _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[VixTrackerService] [VIX REGIME SHIFT] Markt-Regime gewechselt: {OldRegime} -> {NewRegime} (VIX: {Vix:F2})",
oldRegime, _currentRegime, _currentVix);
}
else if (Math.Abs(oldVix - vixValue) >= 0.5m)
{
_logger.LogInformation("[{Channel}] VIX aktualisiert: {Vix:F2} (Regime: {Regime})",
"AnalyzerChannel", _currentVix, _currentRegime);
_ = _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[VixTrackerService] VIX aktualisiert: {Vix:F2} (Regime: {Regime})",
_currentVix, _currentRegime);
}
}
}
@@ -77,12 +78,10 @@ public class VixTrackerService : IVixTrackerService
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Graceful shutdown
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Fehler beim Abfragen von ^VIX über YahooFinanceClient. Nutze gecachten Wert {Vix}.",
"AnalyzerChannel", GetCurrentVix());
await _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, ex, "[VixTrackerService] Fehler beim Abfragen von ^VIX über YahooFinanceClient. Nutze gecachten Wert {Vix}.", GetCurrentVix());
}
return GetCurrentVix();
+13 -22
View File
@@ -3,15 +3,16 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using FinlyticAnalyzer.Util;
using FinlyticCore.Models.Analyzer;
using FinlyticCore.Models.Trades;
using Microsoft.Extensions.Logging;
using FinlyticCore.Services;
namespace FinlyticAnalyzer.Services;
public class WinRateCalculator : IWinRateCalculator
{
private readonly ILogger<WinRateCalculator> _logger;
private readonly IFinlyticLogger<WinRateCalculator> _finlyticLogger;
private readonly string _feedbackDir;
private readonly object _cacheLock = new();
@@ -19,9 +20,9 @@ public class WinRateCalculator : IWinRateCalculator
private DateTime _lastCacheTime = DateTime.MinValue;
private static readonly TimeSpan CacheTtl = TimeSpan.FromMinutes(3);
public WinRateCalculator(ILogger<WinRateCalculator> logger)
public WinRateCalculator(IFinlyticLogger<WinRateCalculator> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_feedbackDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "data", "feedback");
if (!Directory.Exists(_feedbackDir))
{
@@ -31,7 +32,6 @@ public class WinRateCalculator : IWinRateCalculator
/// <summary>
/// Calculates the win rate for a given sector and symbol under the specified market regime.
/// Uses cached feedback records (3-minute TTL) to prevent disk I/O bottlenecks.
/// </summary>
public double CalculateWinRate(string sector, string symbol, VixMarketRegime regime)
{
@@ -53,27 +53,23 @@ public class WinRateCalculator : IWinRateCalculator
{
try
{
// 1. N8n AI Confidence Score (Weight: 40%)
double n8nComponent = 62.0;
if (n8nEvalScore.HasValue && n8nEvalScore.Value > 0)
{
n8nComponent = n8nEvalScore.Value <= 1.0 ? n8nEvalScore.Value * 100.0 : n8nEvalScore.Value;
}
// 2. Technical Score (Weight: 30%)
double taComponent = 60.0;
if (technicalScore.HasValue && technicalScore.Value > 0)
{
taComponent = technicalScore.Value <= 1.0 ? technicalScore.Value * 100.0 : technicalScore.Value;
}
// 3. Sentiment Score (Weight: 15%)
double sentComponent = 58.0;
if (sentimentScore.HasValue)
{
if (sentimentScore.Value >= -1.0 && sentimentScore.Value <= 1.0)
{
// Map sentiment from -1.0..+1.0 into 35.0..85.0
sentComponent = 50.0 + (sentimentScore.Value * 25.0);
}
else
@@ -82,29 +78,25 @@ public class WinRateCalculator : IWinRateCalculator
}
}
// 4. Fundamental Score (Weight: 15%)
double fundComponent = 60.0;
if (fundamentalScore.HasValue && fundamentalScore.Value > 0)
{
fundComponent = fundamentalScore.Value <= 1.0 ? fundamentalScore.Value * 100.0 : fundamentalScore.Value;
}
// Multi-factor weighted composite
double composite = (n8nComponent * 0.40) + (taComponent * 0.30) + (sentComponent * 0.15) + (fundComponent * 0.15);
// 5. Market Regime & Volatility Adjustment
double vixAdjustment = regime switch
{
VixMarketRegime.LowVol => +4.0, // Calm trending market
VixMarketRegime.Normal => +1.5, // Normal conditions
VixMarketRegime.HighVol => -3.5, // Increased whipsaws
VixMarketRegime.Panic => -8.0, // High panic / uncertainty
VixMarketRegime.LowVol => +4.0,
VixMarketRegime.Normal => +1.5,
VixMarketRegime.HighVol => -3.5,
VixMarketRegime.Panic => -8.0,
_ => 0.0
};
composite += vixAdjustment;
// 6. Historical track record calibration (if available in feedback records)
var records = GetCachedOrLoadRecords();
if (records.Count > 0)
{
@@ -120,17 +112,16 @@ public class WinRateCalculator : IWinRateCalculator
}
}
// Clamp between realistic financial statistical bounds (45.0% to 92.0%)
double finalWinRate = Math.Clamp(Math.Round(composite, 1), 45.0, 92.0);
_logger.LogInformation("[{Channel}] Dynamic Win-Rate for {Symbol} ({Sector}): {WinRate:F1}% [AI: {N8n:F1}%, TA: {TA:F1}%, Sent: {Sent:F1}%, Regime: {Regime}]",
"AnalyzerChannel", symbol, sector, finalWinRate, n8nComponent, taComponent, sentComponent, regime);
_ = _finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[WinRateCalculator] Dynamic Win-Rate for {Symbol} ({Sector}): {WinRate:F1}% [AI: {N8n:F1}%, TA: {TA:F1}%, Sent: {Sent:F1}%, Regime: {Regime}]",
symbol, sector, finalWinRate, n8nComponent, taComponent, sentComponent, regime);
return finalWinRate;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Error calculating dynamic win-rate for {Symbol}. Fallback applied.", "AnalyzerChannel", symbol);
_ = _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, ex, "[WinRateCalculator] Error calculating dynamic win-rate for {Symbol}. Fallback applied.", symbol);
return 65.0;
}
}
@@ -162,7 +153,7 @@ public class WinRateCalculator : IWinRateCalculator
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to read or parse feedback file '{File}'", "AnalyzerChannel", file);
_ = _finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, ex, "[WinRateCalculator] Failed to read or parse feedback file '{File}'", file);
}
}
}
+168 -96
View File
@@ -8,9 +8,11 @@ using FinlyticAnalyzer.Database;
using FinlyticAnalyzer.Entities;
using FinlyticAnalyzer.Services;
using FinlyticCore.Dtos;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Models;
using FinlyticCore.Models.Analyzer;
using FinlyticCore.Models.Trades;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
@@ -64,19 +66,19 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "finlytic_analyzer")}_{Guid.NewGuid():N}"
};
_logger.LogInformation("[{Channel}] Starting Unified Analyzer MQTT Client. Host: {Host}, ClientId: {ClientId}", "AnalyzerChannel", config.Host, config.ClientId);
_logger.LogInformation("Starting Unified Analyzer MQTT Client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId);
await ConnectAsync(config);
}
public async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("[{Channel}] Stopping Unified Analyzer MQTT Client.", "AnalyzerChannel");
_logger.LogInformation("Stopping Unified Analyzer MQTT Client.");
await DisconnectAsync();
}
protected override async Task OnConnectedAsync()
{
_logger.LogInformation("[{Channel}] Analyzer MQTT Client connected. Subscribing to topics and RPC response channels...", "AnalyzerChannel");
_logger.LogInformation("Analyzer MQTT Client connected. Subscribing to topics and RPC response channels...");
// Incoming Event Topics
await SubscribeAsync("services/news/#");
@@ -85,16 +87,29 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
await SubscribeAsync("services/config/updated/#");
await SubscribeAsync("services/request/health_Ping/#");
await SubscribeAsync("services/request/analyzer_TriggerManual/#");
await SubscribeAsync("services/request/analyzer_settings_GetAll/#");
await SubscribeAsync("services/request/analyzer_settings_Update/#");
await SubscribeAsync("finlytic/trades/closed/#");
// RPC Response Channels
await SubscribeAsync("services/response/ta_GetAnalysis/#");
await SubscribeAsync("services/response/fundamentals_Get/#");
await SubscribeAsync("services/response/sentiment_GetIsin/#");
await SubscribeAsync("services/response/sentiment_Analyze/#");
await SubscribeAsync("services/response/trades_Get/#");
await SubscribeAsync("services/response/tr_GetLivePrice/#");
await SubscribeAsync("services/response/events_GetByMonth/#");
await SubscribeAsync("services/response/events_GetAll/#");
_logger.LogInformation("[{Channel}] Successfully subscribed to all event and RPC channels.", "AnalyzerChannel");
FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
{
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticAnalyzer", StringComparison.OrdinalIgnoreCase))
{
await PublishAsync("finlytic/logs/FinlyticAnalyzer", logDto);
}
};
_logger.LogInformation("Successfully subscribed to all event and RPC channels.");
}
protected override async Task OnMessageReceivedAsync(string topic, string payloadStr)
@@ -114,10 +129,9 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
string respTopic = $"services/response/health_Ping/{correlationId}";
var healthResp = new ServiceHealthResponse("FinlyticAnalyzer", "Online", DateTime.UtcNow, "Connected");
await PublishAsync(respTopic, healthResp);
if (LogCategoryFilter.IsEnabled(LogCategory.MqttHealthPing))
{
_logger.LogInformation("[{Channel}] [AnalyzerMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", "AnalyzerChannel", correlationId);
}
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[AnalyzerMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
}
return;
}
@@ -126,21 +140,22 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
{
if (topic.EndsWith("FinlyticAnalyzer", StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation("[{Channel}] [AnalyzerMqttClient] Received config update event for FinlyticAnalyzer.", "AnalyzerChannel");
try
{
var configUpdate = JsonSerializer.Deserialize(payloadStr, FinlyticJsonSerializerContext.Default.ServiceConfigUpdatePayload);
if (configUpdate?.Settings != null && configUpdate.Settings.Count > 0)
{
using var scope = _scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsDb.UpdateSettingsFromDictionaryAsync(configUpdate.Settings);
_logger.LogInformation("[{Channel}] [AnalyzerMqttClient] Persisted {Count} updated settings to FinlyticAnalyzer database.", "AnalyzerChannel", configUpdate.Settings.Count);
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
var dict = configUpdate.Settings.ToDictionary(k => k.Key, v => (object?)v.Value);
await settings.UpdateSettingsAsync(dict);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [AnalyzerMqttClient] Error processing MQTT config update event.", "AnalyzerChannel");
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
await finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[AnalyzerMqttClient] Error processing MQTT config update event.");
}
}
return;
@@ -160,6 +175,16 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
var correlationId = topic.Split('/').Last();
await HandleManualTriggerAsync(correlationId, payloadStr, CancellationToken.None);
}
else if (topic.StartsWith("services/request/analyzer_settings_GetAll", StringComparison.OrdinalIgnoreCase))
{
var correlationId = topic.Split('/').Last();
await HandleSettingsGetAllAsync(correlationId);
}
else if (topic.StartsWith("services/request/analyzer_settings_Update", StringComparison.OrdinalIgnoreCase))
{
var correlationId = topic.Split('/').Last();
await HandleSettingsUpdateAsync(payloadStr, correlationId);
}
else if (topic.StartsWith("finlytic/trades/closed/"))
{
await HandleClosedTradeFeedbackAsync(payloadStr);
@@ -167,12 +192,80 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error processing incoming MQTT message on topic {Topic}", "AnalyzerChannel", topic);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
await finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[AnalyzerMqttClient] Error processing incoming MQTT message on topic {Topic}", topic);
}
}
private async Task HandleSettingsGetAllAsync(string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAnalyzer] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
try
{
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/analyzer_settings_GetAll/{correlationId}";
await PublishAsync(responseTopic, settings);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAnalyzer] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticAnalyzer] [Settings_GetAll] Failed to retrieve settings.");
}
}
private async Task HandleSettingsUpdateAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAnalyzer] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
try
{
Dictionary<string, object?>? updates = null;
try
{
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
}
catch
{
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
if (list != null)
{
updates = new Dictionary<string, object?>();
foreach (var item in list) updates[item.Key] = item.Value;
}
}
if (updates != null && updates.Count > 0)
{
await settingsService.UpdateSettingsAsync(updates);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAnalyzer] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
}
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/analyzer_settings_Update/{correlationId}";
await PublishAsync(responseTopic, currentSettings);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticAnalyzer] [Settings_Update] Failed to update settings.");
}
}
private async Task HandleClosedTradeFeedbackAsync(string payloadStr)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
try
{
var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
@@ -209,32 +302,31 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
string filePath = System.IO.Path.Combine(feedbackDir, $"{closedDto.TradeId}.json");
await System.IO.File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(new[] { feedback }, options));
_logger.LogInformation("[{Channel}] Processed closed trade feedback for {TradeId}. Saved to {FilePath}", "AnalyzerChannel", closedDto.TradeId, filePath);
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AnalyzerMqttClient] Processed closed trade feedback for {TradeId}. Saved to {FilePath}", closedDto.TradeId, filePath);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error processing closed trade feedback.", "AnalyzerChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[AnalyzerMqttClient] Error processing closed trade feedback.");
}
}
private async Task HandleManualTriggerAsync(string correlationId, string payloadStr, CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
try
{
var manualReq = JsonSerializer.Deserialize(payloadStr, FinlyticJsonSerializerContext.Default.ManualAnalysisRpcRequest);
if (manualReq == null || string.IsNullOrWhiteSpace(manualReq.Isin))
{
_logger.LogWarning("[{Channel}] Manual trigger received without valid request or ISIN.", "AnalyzerChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, "[AnalyzerMqttClient] Manual trigger received without valid request or ISIN.");
return;
}
if (LogCategoryFilter.IsEnabled(LogCategory.AnalyzerManual))
{
_logger.LogInformation("[{Channel}] [ManualAnalyzer] [TRIGGERED] Processing rich manual trigger for ISIN '{Isin}' (Symbol: {Symbol}). CorrelationId: {CorrelationId}", "AnalyzerChannel", manualReq.Isin, manualReq.Symbol, correlationId);
}
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ManualAnalyzer] [TRIGGERED] Processing rich manual trigger for ISIN '{Isin}' (Symbol: {Symbol}). CorrelationId: {CorrelationId}", manualReq.Isin, manualReq.Symbol, correlationId);
using var scope = _scopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<AnalyzerDbContext>();
var regime = _vixTracker.GetCurrentRegime();
@@ -325,9 +417,8 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
var n8nResponse = await _n8nService.EvaluateAssetAsync(n8nRequest, cancellationToken);
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
var settings = await settingsService.GetSettingsAsync();
double minSignalScore = settings.MinSignalScore;
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
double minSignalScore = await settingsService.GetSettingAsync(SettingKeys.MinWinRateThreshold, cancellationToken);
double dynamicWinRate = _winRateCalculator.CalculateDynamicWinRate(
manualReq.Sector,
@@ -422,12 +513,12 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
{
string propTopic = $"finlytic/trades/proposed/{(string.IsNullOrWhiteSpace(manualReq.Sector) ? "general" : manualReq.Sector.ToLowerInvariant())}/{manualReq.Symbol.ToLowerInvariant()}";
await PublishAsync(propTopic, proposalDto);
_logger.LogInformation("[{Channel}] [ManualAnalyzer] [DISPATCHED] Dispatched Manual Trade Proposal {AnalysisId} to topic {Topic}", "AnalyzerChannel", analysisId, propTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[ManualAnalyzer] [DISPATCHED] Dispatched Manual Trade Proposal {AnalysisId} to topic {Topic}", analysisId, propTopic);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to handle manual trigger for correlation {CorrelationId}.", "AnalyzerChannel", correlationId);
await finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, ex, "[AnalyzerMqttClient] Failed to handle manual trigger for correlation {CorrelationId}.", correlationId);
try
{
var errorResponse = new ManualAnalysisResponseDto
@@ -439,7 +530,7 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
}
catch (Exception pubEx)
{
_logger.LogError(pubEx, "[{Channel}] Failed to publish error response for correlation {CorrelationId}.", "AnalyzerChannel", correlationId);
await finlyticLogger.LogErrorAsync(SettingKeys.AnalyzerChannel, pubEx, "[AnalyzerMqttClient] Failed to publish error response for correlation {CorrelationId}.", correlationId);
}
}
}
@@ -458,13 +549,16 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to parse VIX tick message.", "AnalyzerChannel");
_logger.LogWarning(ex, "Failed to parse VIX tick message.");
}
}
}
private async Task ProcessNewsMessageAsync(string payloadStr, CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AnalyzerMqttClient>>();
var newsArticle = JsonSerializer.Deserialize(payloadStr, FinlyticJsonSerializerContext.Default.NewsArticleDto);
if (newsArticle == null) return;
@@ -474,17 +568,11 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
var filterResult = _filterEngine.EvaluateNews(newsArticle, regime);
if (!filterResult.Passed)
{
if (LogCategoryFilter.IsEnabled(LogCategory.AnalyzerAuto))
{
_logger.LogInformation("[{Channel}] [AutoScreener] [SKIPPED] News message skipped for ISIN '{Isin}'. Reason: {Reason}", "AnalyzerChannel", filterResult.Isin, filterResult.RejectReason);
}
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AutoScreener] [SKIPPED] News message skipped for ISIN '{Isin}'. Reason: {Reason}", filterResult.Isin, filterResult.RejectReason);
return;
}
if (LogCategoryFilter.IsEnabled(LogCategory.AnalyzerAuto))
{
_logger.LogInformation("[{Channel}] [AutoScreener] [TRIGGERED] Screening market opportunity for ISIN '{Isin}'...", "AnalyzerChannel", filterResult.Isin);
}
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AutoScreener] [TRIGGERED] Screening market opportunity for ISIN '{Isin}'...", filterResult.Isin);
string analysisId = Guid.NewGuid().ToString("N");
string eventId = newsArticle.Id != Guid.Empty ? newsArticle.Id.ToString() : analysisId;
@@ -543,7 +631,6 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
{
var isinReq = new IsinRequest(filterResult.Isin);
// Parallel RPC calls (was sequential — up to 12s latency reduced to ~3s)
var livePriceTask = SendRpcRequestAsync<FinlyticCore.Dtos.TechnicalAnalysis.LivePriceDto, IsinRequest>(
"tr_GetLivePrice", isinReq, TimeSpan.FromSeconds(5));
var taTask = SendRpcRequestAsync<FinlyticCore.Dtos.TechnicalAnalysis.TechnicalAnalysisDto, IsinRequest>(
@@ -606,7 +693,6 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
if (sentResp != null)
{
double compound = sentResp.CurrentSummary?.CompoundScore ?? 0.0;
// FinBERT compound score is in range [-1.0, +1.0]. Normalize to [0.0, 1.0] for AI prompt context
double normalizedScore = Math.Clamp((compound + 1.0) / 2.0, 0.0, 1.0);
sentInfo = new SentimentContextInfo
@@ -620,7 +706,7 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to fetch context data for auto screener analysis.", "AnalyzerChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.AnalyzerChannel, ex, "[AnalyzerMqttClient] Failed to fetch context data for auto screener analysis.");
}
var n8nRequest = new N8nAnalysisRequestDto
@@ -670,13 +756,8 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
var n8nResponse = await _n8nService.EvaluateAssetAsync(n8nRequest, cancellationToken);
double minSignalScore = 75.0;
using (var scope = _scopeFactory.CreateScope())
{
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
var settings = await settingsService.GetSettingsAsync();
minSignalScore = settings.MinSignalScore;
}
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
double minSignalScore = await settingsService.GetSettingAsync(SettingKeys.MinWinRateThreshold, cancellationToken);
double confidenceScore = n8nResponse?.EvalScore > 0 ? n8nResponse.EvalScore : 0.75;
bool isHighConviction = n8nResponse != null &&
@@ -752,47 +833,44 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
sentimentScore: sentResp?.CurrentSummary?.CompoundScore,
signalType: n8nResponse?.SuggestedDirection ?? "BUY");
using (var scope = _scopeFactory.CreateScope())
var dbContext = scope.ServiceProvider.GetRequiredService<AnalyzerDbContext>();
bool hasRecentProposal = await dbContext.Analyses.AnyAsync(a =>
a.Isin == filterResult.Isin &&
a.IsTradeProposed &&
a.CreatedAt >= DateTime.UtcNow.AddHours(-4),
cancellationToken);
if (hasRecentProposal && isHighConviction)
{
var dbContext = scope.ServiceProvider.GetRequiredService<AnalyzerDbContext>();
bool hasRecentProposal = await dbContext.Analyses.AnyAsync(a =>
a.Isin == filterResult.Isin &&
a.IsTradeProposed &&
a.CreatedAt >= DateTime.UtcNow.AddHours(-4),
cancellationToken);
if (hasRecentProposal && isHighConviction)
{
_logger.LogInformation("[{Channel}] [AutoScreener] Asset {Symbol} ({Isin}) already has an active trade proposal in the last 4 hours. Skipping duplicate trade proposal generation.",
"AnalyzerChannel", finalSymbol, filterResult.Isin);
isHighConviction = false;
}
var analysisEntity = new AnalysisEntity
{
AnalysisId = analysisId,
EventId = eventId,
Sector = filterResult.Sector,
Symbol = finalSymbol,
Isin = filterResult.Isin,
VixRegime = regime,
VixValue = currentVix,
ImpactScore = filterResult.ImpactScore,
WinRate = dynamicWinRate,
RawDataJson = payloadStr,
AiOutputJson = JsonSerializer.Serialize(recommendation),
N8nResponseJson = n8nResponse != null ? JsonSerializer.Serialize(n8nResponse) : "{}",
N8nEvalScore = n8nResponse?.EvalScore ?? 0,
N8nDecision = n8nResponse?.AiDecision ?? "None",
IsTradeProposed = isHighConviction,
CreatedAt = DateTime.UtcNow
};
dbContext.Analyses.Add(analysisEntity);
await dbContext.SaveChangesAsync(cancellationToken);
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AutoScreener] Asset {Symbol} ({Isin}) already has an active trade proposal in the last 4 hours. Skipping duplicate trade proposal generation.",
finalSymbol, filterResult.Isin);
isHighConviction = false;
}
var analysisEntity = new AnalysisEntity
{
AnalysisId = analysisId,
EventId = eventId,
Sector = filterResult.Sector,
Symbol = finalSymbol,
Isin = filterResult.Isin,
VixRegime = regime,
VixValue = currentVix,
ImpactScore = filterResult.ImpactScore,
WinRate = dynamicWinRate,
RawDataJson = payloadStr,
AiOutputJson = JsonSerializer.Serialize(recommendation),
N8nResponseJson = n8nResponse != null ? JsonSerializer.Serialize(n8nResponse) : "{}",
N8nEvalScore = n8nResponse?.EvalScore ?? 0,
N8nDecision = n8nResponse?.AiDecision ?? "None",
IsTradeProposed = isHighConviction,
CreatedAt = DateTime.UtcNow
};
dbContext.Analyses.Add(analysisEntity);
await dbContext.SaveChangesAsync(cancellationToken);
if (isHighConviction && n8nResponse != null)
{
var autoProposalDto = new TradeProposalDto
@@ -830,7 +908,7 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
string propTopic = $"finlytic/trades/proposed/{(string.IsNullOrWhiteSpace(filterResult.Sector) ? "general" : filterResult.Sector.ToLowerInvariant())}/{finalSymbol.ToLowerInvariant()}";
await PublishAsync(propTopic, autoProposalDto);
_logger.LogInformation("[{Channel}] [AutoScreener] Dispatched High-Conviction Proposal {TradeId} to topic {Topic}", "AnalyzerChannel", autoProposalDto.TradeId, propTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AutoScreener] Dispatched High-Conviction Proposal {TradeId} to topic {Topic}", autoProposalDto.TradeId, propTopic);
}
if (isHighConviction)
@@ -839,19 +917,13 @@ public class AnalyzerMqttClient : ManagedMqttClient, IHostedService
await PublishAsync(recTopic, recommendation);
await PublishAsync("finlytic/recommendations/auto", recommendation);
if (LogCategoryFilter.IsEnabled(LogCategory.AnalyzerAuto))
{
_logger.LogInformation("[{Channel}] [AutoScreener] [RECOMMENDED] High-Conviction Opportunity found for {Symbol} (Bias: {Bias}, Confidence: {Score:F2}). Published to {Topic}",
"AnalyzerChannel", finalSymbol, recommendation.RecommendedAsset.Bias, recommendation.RecommendedAsset.ConfidenceScore, recTopic);
}
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AutoScreener] [RECOMMENDED] High-Conviction Opportunity found for {Symbol} (Bias: {Bias}, Confidence: {Score:F2}). Published to {Topic}",
finalSymbol, recommendation.RecommendedAsset.Bias, recommendation.RecommendedAsset.ConfidenceScore, recTopic);
}
else
{
if (LogCategoryFilter.IsEnabled(LogCategory.AnalyzerAuto))
{
_logger.LogInformation("[{Channel}] [AutoScreener] [DROPPED] Low-conviction signal for {Symbol} dropped (Confidence: {Score:F2}, Action: NO_ACTION)",
"AnalyzerChannel", finalSymbol, recommendation.RecommendedAsset.ConfidenceScore);
}
await finlyticLogger.LogInfoAsync(SettingKeys.AnalyzerChannel, "[AutoScreener] [DROPPED] Low-conviction signal for {Symbol} dropped (Confidence: {Score:F2}, Action: NO_ACTION)",
finalSymbol, recommendation.RecommendedAsset.ConfidenceScore);
}
}
+30
View File
@@ -0,0 +1,30 @@
using FinlyticCore.Models.Settings;
namespace FinlyticAnalyzer.Util;
public static class SettingKeys
{
// --- Logging-Kanäle ---
public static readonly SettingKey<bool> AnalyzerChannel = new("Logging.Channel.Analyzer", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
// --- Makro & VIX Schwellenwerte ---
public static readonly SettingKey<double> VixPanicThreshold = new("Macro.VixPanicThreshold", 28.0);
public static readonly SettingKey<double> VixElevatedThreshold = new("Macro.VixElevatedThreshold", 20.0);
public static readonly SettingKey<int> VixPollIntervalSeconds = new("Macro.VixPollIntervalSeconds", 60);
// --- Filter & Winrate-Logik ---
public static readonly SettingKey<double> MinWinRateThreshold = new("Filter.MinWinRateThreshold", 60.0);
public static readonly SettingKey<double> WeightMacro = new("Filter.WeightMacro", 0.30);
public static readonly SettingKey<double> WeightFundamental = new("Filter.WeightFundamental", 0.30);
public static readonly SettingKey<double> WeightSentiment = new("Filter.WeightSentiment", 0.20);
public static readonly SettingKey<double> WeightTechnical = new("Filter.WeightTechnical", 0.20);
// --- Trade & Risiko-Parameter ---
public static readonly SettingKey<double> DefaultTakeProfitPercent = new("Trade.DefaultTakeProfitPercent", 15.0);
public static readonly SettingKey<double> DefaultStopLossPercent = new("Trade.DefaultStopLossPercent", 5.0);
public static readonly SettingKey<int> MaxAllowedLeverage = new("Trade.MaxAllowedLeverage", 10);
public static readonly SettingKey<double> MaxRiskPerTradePercent = new("Trade.MaxRiskPerTradePercent", 2.0);
public static readonly SettingKey<int> ProposalValidityHours = new("Trade.ProposalValidityHours", 24);
}
@@ -0,0 +1,44 @@
class LogMessageDto {
final DateTime timestamp;
final String serviceName;
final String channel;
final String level; // 'Information', 'Warning', 'Error', 'Debug', 'Trace'
final String message;
final String? exception;
const LogMessageDto({
required this.timestamp,
required this.serviceName,
required this.channel,
required this.level,
required this.message,
this.exception,
});
factory LogMessageDto.fromJson(Map<String, dynamic> json) {
DateTime parsedTime = DateTime.now();
if (json['timestamp'] != null) {
parsedTime = DateTime.tryParse(json['timestamp'].toString()) ?? DateTime.now();
}
return LogMessageDto(
timestamp: parsedTime.toLocal(),
serviceName: json['serviceName']?.toString() ?? '',
channel: json['channel']?.toString() ?? '',
level: json['level']?.toString() ?? 'Information',
message: json['message']?.toString() ?? '',
exception: json['exception']?.toString(),
);
}
Map<String, dynamic> toJson() {
return {
'timestamp': timestamp.toUtc().toIso8601String(),
'serviceName': serviceName,
'channel': channel,
'level': level,
'message': message,
'exception': exception,
};
}
}
@@ -1,11 +1,13 @@
class ServiceSettingDto {
final String key;
final String value;
final String type; // 'bool', 'int', 'double', 'string'
final String description;
const ServiceSettingDto({
required this.key,
required this.value,
this.type = 'string',
this.description = '',
});
@@ -13,6 +15,7 @@ class ServiceSettingDto {
return ServiceSettingDto(
key: json['key']?.toString() ?? '',
value: json['value']?.toString() ?? '',
type: json['dataType']?.toString() ?? json['type']?.toString() ?? 'string',
description: json['description']?.toString() ?? '',
);
}
@@ -21,6 +24,7 @@ class ServiceSettingDto {
return {
'key': key,
'value': value,
'type': type,
'description': description,
};
}
@@ -54,7 +54,7 @@ class AdminRepository {
return {};
}
Future<void> updateServiceSettings(String serviceName, Map<String, String> settings) async {
Future<void> updateServiceSettings(String serviceName, Map<String, dynamic> settings) async {
final res = await apiClient.put('/api/v1/admin/settings/$serviceName', data: settings);
if (res.statusCode != 200 && res.statusCode != 204) {
throw Exception('Einstellungen konnten nicht gespeichert werden');
@@ -6,6 +6,7 @@ import '../../../core/widgets/glass_container.dart';
import '../../../core/widgets/status_badge.dart';
import '../models/service_setting_dto.dart';
import '../repositories/admin_repository.dart';
import '../widgets/live_log_console.dart';
class ServiceDetailScreen extends StatefulWidget {
final String serviceName;
@@ -75,9 +76,20 @@ class _ServiceDetailScreenState extends State<ServiceDetailScreen> {
Future<void> _saveSettings() async {
setState(() => _isSaving = true);
try {
final payload = <String, String>{};
final payload = <String, dynamic>{};
_controllers.forEach((k, v) {
payload[k] = v.text;
final text = v.text.trim();
if (text.toLowerCase() == 'true') {
payload[k] = true;
} else if (text.toLowerCase() == 'false') {
payload[k] = false;
} else if (int.tryParse(text) != null) {
payload[k] = int.parse(text);
} else if (double.tryParse(text) != null) {
payload[k] = double.parse(text);
} else {
payload[k] = text;
}
});
await _repository.updateServiceSettings(widget.serviceName, payload);
@@ -170,10 +182,13 @@ class _ServiceDetailScreenState extends State<ServiceDetailScreen> {
..._settings.map((s) {
final key = s.key;
final desc = s.description;
final type = s.type.toLowerCase();
final controller = _controllers[key];
if (controller == null) return const SizedBox.shrink();
final isBoolean = controller.text.toLowerCase() == 'true' || controller.text.toLowerCase() == 'false';
final isBoolean = type == 'bool' ||
controller.text.toLowerCase() == 'true' ||
controller.text.toLowerCase() == 'false';
if (isBoolean) {
final boolVal = controller.text.toLowerCase() == 'true';
@@ -183,7 +198,11 @@ class _ServiceDetailScreenState extends State<ServiceDetailScreen> {
decoration: BoxDecoration(
color: AppTheme.glassSurface,
borderRadius: BorderRadius.circular(12),
border: Border.all(color: AppTheme.glassBorder),
border: Border.all(
color: boolVal
? AppTheme.primaryEmerald.withValues(alpha: 0.4)
: AppTheme.glassBorder,
),
),
child: SwitchListTile(
contentPadding: EdgeInsets.zero,
@@ -191,21 +210,31 @@ class _ServiceDetailScreenState extends State<ServiceDetailScreen> {
subtitle: desc.isNotEmpty ? Text(desc, style: TextStyle(fontSize: 11, color: AppTheme.textMuted)) : null,
value: boolVal,
activeThumbColor: AppTheme.primaryEmerald,
activeTrackColor: AppTheme.primaryEmerald.withValues(alpha: 0.3),
onChanged: (val) => setState(() => controller.text = val.toString()),
),
);
}
final isNumeric = type == 'int' || type == 'double' || type == 'number' || type == 'decimal';
return Padding(
padding: const EdgeInsets.only(bottom: 14),
child: TextField(
controller: controller,
keyboardType: isNumeric
? const TextInputType.numberWithOptions(decimal: true)
: TextInputType.text,
style: const TextStyle(color: Colors.white),
decoration: InputDecoration(
labelText: _formatLabel(key),
helperText: desc.isNotEmpty ? desc : null,
helperMaxLines: 2,
prefixIcon: Icon(Icons.tune_outlined, size: 18, color: AppTheme.primaryEmerald),
prefixIcon: Icon(
isNumeric ? Icons.numbers_outlined : Icons.tune_outlined,
size: 18,
color: AppTheme.primaryEmerald,
),
),
),
);
@@ -236,6 +265,11 @@ class _ServiceDetailScreenState extends State<ServiceDetailScreen> {
),
),
const SizedBox(height: 24),
LiveLogConsole(
serviceName: widget.serviceName,
apiClient: widget.apiClient,
),
const SizedBox(height: 24),
GlassContainer(
padding: const EdgeInsets.all(24),
child: Column(
@@ -0,0 +1,383 @@
import 'dart:async';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:signalr_core/signalr_core.dart';
import '../../../core/network/api_client.dart';
import '../../../core/network/signalr_service.dart';
import '../../../core/services/secure_storage_service.dart';
import '../../../core/theme/app_theme.dart';
import '../../../core/widgets/glass_container.dart';
import '../models/log_message_dto.dart';
class LiveLogConsole extends StatefulWidget {
final String serviceName;
final ApiClient apiClient;
const LiveLogConsole({
super.key,
required this.serviceName,
required this.apiClient,
});
@override
State<LiveLogConsole> createState() => _LiveLogConsoleState();
}
class _LiveLogConsoleState extends State<LiveLogConsole> {
HubConnection? _hubConnection;
final List<LogMessageDto> _logs = [];
final ScrollController _scrollController = ScrollController();
final TextEditingController _searchController = TextEditingController();
bool _isConnected = false;
bool _isPaused = false;
bool _autoScroll = true;
String _selectedLevel = 'ALL';
String _searchQuery = '';
@override
void initState() {
super.initState();
_fetchInitialLogs();
_connectSignalR();
}
@override
void dispose() {
_disconnectSignalR();
_scrollController.dispose();
_searchController.dispose();
super.dispose();
}
Future<void> _fetchInitialLogs() async {
try {
final res = await widget.apiClient.get('/api/v1/admin/settings/logs/${widget.serviceName}');
if (res.data is List) {
final list = (res.data as List).map((item) => LogMessageDto.fromJson(Map<String, dynamic>.from(item as Map))).toList();
if (mounted) {
setState(() {
_logs.addAll(list);
});
_scrollToBottomIfNeeded();
}
}
} catch (e) {
if (kDebugMode) debugPrint('[LiveLogConsole] Error fetching initial logs: $e');
}
}
Future<void> _connectSignalR() async {
try {
final storage = SecureStorageService();
final token = await storage.getToken();
_hubConnection = HubConnectionBuilder()
.withUrl(
'${SignalRService.baseUrl}/hubs/logs',
HttpConnectionOptions(
accessTokenFactory: () async => token,
transport: HttpTransportType.webSockets,
logging: (level, message) {
if (kDebugMode) debugPrint('[SignalR Logs WS] $message');
},
),
)
.withAutomaticReconnect()
.build();
_hubConnection!.on('ReceiveLogMessage', (arguments) {
if (arguments != null && arguments.isNotEmpty) {
try {
final map = Map<String, dynamic>.from(arguments.first as Map);
final log = LogMessageDto.fromJson(map);
if (log.serviceName.isEmpty || log.serviceName.toLowerCase() == widget.serviceName.toLowerCase()) {
if (mounted && !_isPaused) {
setState(() {
_logs.add(log);
if (_logs.length > 500) {
_logs.removeAt(0);
}
});
_scrollToBottomIfNeeded();
}
}
} catch (e) {
if (kDebugMode) debugPrint('[SignalR Log Parse Error] $e');
}
}
});
_hubConnection!.onclose((error) {
if (mounted) setState(() => _isConnected = false);
});
_hubConnection!.onreconnected((connectionId) {
if (mounted) {
setState(() => _isConnected = true);
_hubConnection?.invoke('JoinServiceLogs', args: [widget.serviceName]);
}
});
await _hubConnection!.start();
await _hubConnection!.invoke('JoinServiceLogs', args: [widget.serviceName]);
if (mounted) {
setState(() => _isConnected = true);
}
} catch (e) {
if (kDebugMode) debugPrint('[SignalR Log Connection Error] $e');
if (mounted) setState(() => _isConnected = false);
}
}
Future<void> _disconnectSignalR() async {
try {
if (_hubConnection != null) {
await _hubConnection!.invoke('LeaveServiceLogs', args: [widget.serviceName]);
await _hubConnection!.stop();
}
} catch (_) {}
_hubConnection = null;
}
void _scrollToBottomIfNeeded() {
if (_autoScroll && _scrollController.hasClients) {
WidgetsBinding.instance.addPostFrameCallback((_) {
if (_scrollController.hasClients) {
_scrollController.animateTo(
_scrollController.position.maxScrollExtent,
duration: const Duration(milliseconds: 200),
curve: Curves.easeOut,
);
}
});
}
}
List<LogMessageDto> get _filteredLogs {
return _logs.where((log) {
if (_selectedLevel != 'ALL' && log.level.toUpperCase() != _selectedLevel) {
return false;
}
if (_searchQuery.isNotEmpty) {
final query = _searchQuery.toLowerCase();
final matchMsg = log.message.toLowerCase().contains(query);
final matchChannel = log.channel.toLowerCase().contains(query);
return matchMsg || matchChannel;
}
return true;
}).toList();
}
Color _getLevelColor(String level) {
switch (level.toUpperCase()) {
case 'ERROR':
case 'CRITICAL':
return Colors.redAccent;
case 'WARNING':
case 'WARN':
return Colors.amberAccent;
case 'DEBUG':
case 'TRACE':
return Colors.blueGrey.shade300;
case 'INFORMATION':
case 'INFO':
default:
return AppTheme.primaryEmerald;
}
}
String _formatTime(DateTime time) {
final h = time.hour.toString().padLeft(2, '0');
final m = time.minute.toString().padLeft(2, '0');
final s = time.second.toString().padLeft(2, '0');
final ms = time.millisecond.toString().padLeft(3, '0');
return '$h:$m:$s.$ms';
}
@override
Widget build(BuildContext context) {
final filtered = _filteredLogs;
return GlassContainer(
padding: const EdgeInsets.all(20),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
// Header Bar
Row(
mainAxisAlignment: MainAxisAlignment.spaceBetween,
children: [
Row(
children: [
Icon(Icons.terminal_rounded, color: AppTheme.primaryEmerald, size: 22),
const SizedBox(width: 10),
Text(
'Live Service-Logs (${widget.serviceName})',
style: const TextStyle(fontSize: 16, fontWeight: FontWeight.bold),
),
],
),
Row(
children: [
Container(
width: 8,
height: 8,
decoration: BoxDecoration(
shape: BoxShape.circle,
color: _isConnected ? AppTheme.primaryEmerald : Colors.redAccent,
),
),
const SizedBox(width: 6),
Text(
_isConnected ? 'Live WebSocket' : 'Offline',
style: TextStyle(
fontSize: 12,
color: _isConnected ? AppTheme.primaryEmerald : Colors.redAccent,
fontWeight: FontWeight.w600,
),
),
],
),
],
),
const SizedBox(height: 14),
// Control Bar: Search + Level Filters + Actions
Wrap(
spacing: 8,
runSpacing: 8,
crossAxisAlignment: WrapCrossAlignment.center,
children: [
// Search Field
SizedBox(
width: 220,
height: 36,
child: TextField(
controller: _searchController,
onChanged: (val) => setState(() => _searchQuery = val.trim()),
style: const TextStyle(fontSize: 13, color: Colors.white),
decoration: InputDecoration(
hintText: 'Logs durchsuchen...',
hintStyle: const TextStyle(fontSize: 12, color: Colors.white38),
prefixIcon: const Icon(Icons.search, size: 16, color: Colors.white54),
suffixIcon: _searchQuery.isNotEmpty
? IconButton(
icon: const Icon(Icons.clear, size: 14, color: Colors.white54),
onPressed: () {
_searchController.clear();
setState(() => _searchQuery = '');
},
)
: null,
contentPadding: const EdgeInsets.symmetric(horizontal: 10, vertical: 0),
filled: true,
fillColor: Colors.black26,
border: OutlineInputBorder(borderRadius: BorderRadius.circular(8), borderSide: BorderSide.none),
),
),
),
// Filter Chips
for (final lvl in ['ALL', 'INFO', 'WARN', 'ERROR', 'DEBUG'])
ChoiceChip(
label: Text(lvl, style: TextStyle(fontSize: 11, fontWeight: FontWeight.bold, color: _selectedLevel == lvl ? Colors.black : Colors.white70)),
selected: _selectedLevel == lvl,
selectedColor: AppTheme.primaryEmerald,
backgroundColor: Colors.white10,
visualDensity: VisualDensity.compact,
padding: const EdgeInsets.symmetric(horizontal: 4),
onSelected: (selected) {
if (selected) setState(() => _selectedLevel = lvl);
},
),
const SizedBox(width: 8),
// Action buttons
IconButton(
tooltip: _isPaused ? 'Stream Fortsetzen' : 'Stream Pausieren',
icon: Icon(_isPaused ? Icons.play_arrow_rounded : Icons.pause_rounded, size: 20, color: _isPaused ? Colors.amberAccent : Colors.white70),
onPressed: () => setState(() => _isPaused = !_isPaused),
),
IconButton(
tooltip: _autoScroll ? 'Auto-Scroll an' : 'Auto-Scroll aus',
icon: Icon(Icons.vertical_align_bottom_rounded, size: 20, color: _autoScroll ? AppTheme.primaryEmerald : Colors.white38),
onPressed: () => setState(() => _autoScroll = !_autoScroll),
),
IconButton(
tooltip: 'Konsole leeren',
icon: const Icon(Icons.delete_outline_rounded, size: 20, color: Colors.white54),
onPressed: () => setState(() => _logs.clear()),
),
],
),
const SizedBox(height: 12),
// Terminal Box
Container(
height: 380,
width: double.infinity,
padding: const EdgeInsets.all(12),
decoration: BoxDecoration(
color: const Color(0xFF0D1117), // Deep dark console
borderRadius: BorderRadius.circular(10),
border: Border.all(color: Colors.white12),
),
child: filtered.isEmpty
? Center(
child: Text(
_logs.isEmpty ? 'Warte auf Log-Nachrichten von ${widget.serviceName}...' : 'Keine Logs passend zum Filter.',
style: const TextStyle(fontSize: 12, color: Colors.white38, fontStyle: FontStyle.italic),
),
)
: ListView.builder(
controller: _scrollController,
itemCount: filtered.length,
itemBuilder: (context, index) {
final item = filtered[index];
final color = _getLevelColor(item.level);
return Padding(
padding: const EdgeInsets.symmetric(vertical: 2.5),
child: SelectableText.rich(
TextSpan(
style: const TextStyle(fontFamily: 'monospace', fontSize: 11.5, height: 1.4),
children: [
TextSpan(
text: '${_formatTime(item.timestamp)} ',
style: const TextStyle(color: Colors.white38),
),
TextSpan(
text: '[${item.level.toUpperCase().padRight(5)}] ',
style: TextStyle(color: color, fontWeight: FontWeight.bold),
),
if (item.channel.isNotEmpty)
TextSpan(
text: '{${item.channel}} ',
style: TextStyle(color: Colors.cyanAccent.withValues(alpha: 0.8)),
),
TextSpan(
text: item.message,
style: const TextStyle(color: Colors.white),
),
if (item.exception != null && item.exception!.isNotEmpty)
TextSpan(
text: '\n ${item.exception}',
style: const TextStyle(color: Colors.redAccent, fontSize: 10.5),
),
],
),
),
);
},
),
),
],
),
);
}
}
@@ -191,13 +191,17 @@ class FundamentalDataModel extends Equatable {
}
final totalRev = parseNullableDouble(fundMap?['totalRevenue'] ?? json['totalRevenue']);
final grossProf = parseNullableDouble(fundMap?['grossProfit'] ?? json['grossProfit']);
double? grossMarginVal = parseNullableDouble(fundMap?['grossMargin'] ?? json['grossMargin']);
if (grossMarginVal == null && grossProf != null) {
if (grossProf <= 1.0 && grossProf >= 0.0) {
grossMarginVal = grossProf;
final rawGrossProfit = parseNullableDouble(fundMap?['grossProfit'] ?? json['grossProfit']);
double? grossMarginVal = parseNullableDouble(fundMap?['grossMargins'] ?? fundMap?['grossMargin'] ?? json['grossMargin']);
double? grossProfVal = rawGrossProfit;
if (rawGrossProfit != null) {
if (rawGrossProfit <= 1.0 && rawGrossProfit >= 0.0) {
grossMarginVal ??= rawGrossProfit;
if (totalRev != null && totalRev > 0) {
grossProfVal = rawGrossProfit * totalRev;
}
} else if (totalRev != null && totalRev > 0) {
grossMarginVal = grossProf / totalRev;
grossMarginVal ??= rawGrossProfit / totalRev;
}
}
@@ -207,6 +211,50 @@ class FundamentalDataModel extends Equatable {
evToRevVal = evVal / totalRev;
}
String? exDivDateStr = fundMap?['exDividendDate']?.toString() ?? json['exDividendDate']?.toString();
String? nextEarningsDateStr = fundMap?['nextEarningsDate']?.toString() ?? json['nextEarningsDate']?.toString();
final rawEvents = json['events'];
if (rawEvents is List) {
final now = DateTime.now();
final divEvents = rawEvents.whereType<Map<String, dynamic>>().where((e) {
final t = e['type']?.toString().toUpperCase() ?? '';
return t == 'DIVIDEND' || t == 'EX_DIVIDEND';
}).toList();
if (exDivDateStr == null && divEvents.isNotEmpty) {
divEvents.sort((a, b) {
final da = DateTime.tryParse(a['date']?.toString() ?? '') ?? DateTime(1970);
final db = DateTime.tryParse(b['date']?.toString() ?? '') ?? DateTime(1970);
return da.compareTo(db);
});
final upcoming = divEvents.firstWhere((e) {
final d = DateTime.tryParse(e['date']?.toString() ?? '');
return d != null && d.isAfter(now.subtract(const Duration(days: 7)));
}, orElse: () => divEvents.last);
exDivDateStr = upcoming['date']?.toString();
}
final earningsEvents = rawEvents.whereType<Map<String, dynamic>>().where((e) {
final t = e['type']?.toString().toUpperCase() ?? '';
return t.contains('EARNINGS');
}).toList();
if (nextEarningsDateStr == null && earningsEvents.isNotEmpty) {
earningsEvents.sort((a, b) {
final da = DateTime.tryParse(a['date']?.toString() ?? '') ?? DateTime(1970);
final db = DateTime.tryParse(b['date']?.toString() ?? '') ?? DateTime(1970);
return da.compareTo(db);
});
final upcoming = earningsEvents.firstWhere((e) {
final d = DateTime.tryParse(e['date']?.toString() ?? '');
return d != null && d.isAfter(now.subtract(const Duration(days: 1)));
}, orElse: () => earningsEvents.last);
nextEarningsDateStr = upcoming['date']?.toString();
}
}
return FundamentalDataModel(
isin: isinVal,
primaryTicker: primaryTickerVal,
@@ -228,25 +276,25 @@ class FundamentalDataModel extends Equatable {
fiftyTwoWeekLow: parseNullableDouble(fundMap?['fiftyTwoWeekLow'] ?? json['fiftyTwoWeekLow']),
marketCapitalization: parseNullableDouble(fundMap?['marketCap'] ?? fundMap?['marketCapitalization'] ?? json['marketCapitalization']),
enterpriseValue: evVal,
peRatioTrailing: parseNullableDouble(fundMap?['trailingPE'] ?? fundMap?['peRatioTrailing'] ?? json['peRatioTrailing']),
peRatioForward: parseNullableDouble(fundMap?['forwardPE'] ?? fundMap?['peRatioForward'] ?? json['peRatioForward']),
peRatioTrailing: parseNullableDouble(fundMap?['trailingPe'] ?? fundMap?['trailingPE'] ?? fundMap?['peRatioTrailing'] ?? json['peRatioTrailing'] ?? json['trailingPe']),
peRatioForward: parseNullableDouble(fundMap?['forwardPe'] ?? fundMap?['forwardPE'] ?? fundMap?['peRatioForward'] ?? json['peRatioForward'] ?? json['forwardPe']),
pegRatio: parseNullableDouble(fundMap?['pegRatio'] ?? json['pegRatio']),
pbRatio: parseNullableDouble(fundMap?['priceToBook'] ?? fundMap?['pbRatio'] ?? json['pbRatio']),
psRatio: parseNullableDouble(fundMap?['priceToSalesTrailing12Months'] ?? fundMap?['psRatio'] ?? json['psRatio']),
evToEbitda: parseNullableDouble(fundMap?['enterpriseToEbitda'] ?? fundMap?['evToEbitda'] ?? json['evToEbitda']),
psRatio: parseNullableDouble(fundMap?['priceToSales'] ?? fundMap?['priceToSalesTrailing12Months'] ?? fundMap?['psRatio'] ?? json['psRatio']),
evToEbitda: parseNullableDouble(fundMap?['evToEbitda'] ?? fundMap?['enterpriseToEbitda'] ?? json['evToEbitda']),
evToRevenue: evToRevVal,
totalRevenue: totalRev,
revenueGrowthYoY: parseNullableDouble(fundMap?['revenueGrowth'] ?? fundMap?['revenueGrowthYoY'] ?? json['revenueGrowthYoY']),
grossProfit: grossProf,
revenueGrowthYoY: parseNullableDouble(fundMap?['revenueGrowthYoY'] ?? fundMap?['revenueGrowth'] ?? json['revenueGrowthYoY']),
grossProfit: grossProfVal,
ebitda: parseNullableDouble(fundMap?['ebitda'] ?? json['ebitda']),
dilutedEps: parseNullableDouble(fundMap?['trailingEps'] ?? fundMap?['dilutedEps'] ?? json['dilutedEps']),
dilutedEps: parseNullableDouble(fundMap?['dilutedEps'] ?? fundMap?['trailingEps'] ?? json['dilutedEps']),
totalCash: parseNullableDouble(fundMap?['totalCash'] ?? json['totalCash']),
totalDebt: parseNullableDouble(fundMap?['totalDebt'] ?? json['totalDebt']),
operatingCashFlow: parseNullableDouble(fundMap?['operatingCashflow'] ?? fundMap?['operatingCashFlow'] ?? json['operatingCashFlow']),
freeCashFlow: parseNullableDouble(fundMap?['freeCashflow'] ?? fundMap?['freeCashFlow'] ?? json['freeCashFlow']),
operatingCashFlow: parseNullableDouble(fundMap?['operatingCashFlow'] ?? fundMap?['operatingCashflow'] ?? json['operatingCashFlow']),
freeCashFlow: parseNullableDouble(fundMap?['freeCashFlow'] ?? fundMap?['freeCashflow'] ?? json['freeCashFlow']),
grossMargin: grossMarginVal,
operatingMargin: parseNullableDouble(fundMap?['operatingMargins'] ?? fundMap?['operatingMargin'] ?? json['operatingMargin']),
netProfitMargin: parseNullableDouble(fundMap?['profitMargins'] ?? fundMap?['netProfitMargin'] ?? json['netProfitMargin']),
operatingMargin: parseNullableDouble(fundMap?['operatingIncome'] ?? fundMap?['operatingMargins'] ?? fundMap?['operatingMargin'] ?? json['operatingMargin']),
netProfitMargin: parseNullableDouble(fundMap?['netIncome'] ?? fundMap?['profitMargins'] ?? fundMap?['netProfitMargin'] ?? json['netProfitMargin']),
returnOnEquity: parseNullableDouble(fundMap?['returnOnEquity'] ?? json['returnOnEquity']),
returnOnAssets: parseNullableDouble(fundMap?['returnOnAssets'] ?? json['returnOnAssets']),
returnOnInvestedCapital: parseNullableDouble(fundMap?['returnOnInvestedCapital'] ?? json['returnOnInvestedCapital']),
@@ -254,10 +302,10 @@ class FundamentalDataModel extends Equatable {
currentRatio: parseNullableDouble(fundMap?['currentRatio'] ?? json['currentRatio']),
quickRatio: parseNullableDouble(fundMap?['quickRatio'] ?? json['quickRatio']),
interestCoverage: parseNullableDouble(fundMap?['interestCoverage'] ?? json['interestCoverage']),
dividendYield: parseNullableDouble(fundMap?['dividendYield'] ?? json['dividendYield']),
dividendYield: parseNullableDouble(fundMap?['forwardDividendYield'] ?? fundMap?['dividendYield'] ?? json['dividendYield']),
payoutRatio: parseNullableDouble(fundMap?['payoutRatio'] ?? json['payoutRatio']),
exDividendDate: fundMap?['exDividendDate']?.toString() ?? json['exDividendDate']?.toString(),
nextEarningsDate: fundMap?['nextEarningsDate']?.toString() ?? json['nextEarningsDate']?.toString(),
exDividendDate: exDivDateStr,
nextEarningsDate: nextEarningsDateStr,
percentHeldByInstitutions: parseNullableDouble(fundMap?['percentHeldByInstitutions'] ?? json['percentHeldByInstitutions']),
percentHeldByInsiders: parseNullableDouble(fundMap?['percentHeldByInsiders'] ?? json['percentHeldByInsiders']),
shortRatio: parseNullableDouble(fundMap?['shortRatio'] ?? json['shortRatio']),
@@ -45,7 +45,7 @@ class FundamentalCategoryPanels extends StatelessWidget {
String _fmtPercent(double? val) {
if (val == null) return 'N/A';
final p = (val.abs() <= 1.0 && val != 0.0) ? val * 100.0 : val;
final p = (val.abs() <= 5.0 && val != 0.0) ? val * 100.0 : val;
return '${p.toStringAsFixed(2)}%';
}
+17 -3
View File
@@ -1,11 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FinlyticAssets.Entities;
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.ChangeTracking;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticAssets.Database;
public class AssetsDbContext : DbContext
public class AssetsDbContext : DbContext, ISettingsDbContext
{
public AssetsDbContext(DbContextOptions<AssetsDbContext> options) : base(options)
{
@@ -16,7 +21,6 @@ public class AssetsDbContext : DbContext
public DbSet<AssetEntity> TradeRepublicAssets { get; set; }
public DbSet<TagEntity> TradeRepublicTags { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
@@ -24,7 +28,7 @@ public class AssetsDbContext : DbContext
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key);
entity.HasIndex(e => e.Key).IsUnique();
});
modelBuilder.Entity<AssetEntity>(entity =>
{
@@ -95,3 +99,13 @@ public class AssetsDbContext : DbContext
.HaveConversion(typeof(FinlyticCore.Converters.NullableUtcDateTimeConverter));
}
}
public class AssetsDbContextFactory : IDesignTimeDbContextFactory<AssetsDbContext>
{
public AssetsDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<AssetsDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=assets;Username=postgres;Password=postgres");
return new AssetsDbContext(optionsBuilder.Options);
}
}
@@ -0,0 +1,366 @@
// <auto-generated />
using System;
using FinlyticAssets.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticAssets.Migrations
{
[DbContext(typeof(AssetsDbContext))]
[Migration("20260815184053_UpdateDynamicSettingsUniqueIndex")]
partial class UpdateDynamicSettingsUniqueIndex
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("AssetEntityTagEntity", b =>
{
b.Property<string>("TagsId")
.HasColumnType("text");
b.Property<string>("AssetsIsin")
.HasColumnType("text");
b.Property<string>("AssetsInstrumentCategory")
.HasColumnType("text");
b.HasKey("TagsId", "AssetsIsin", "AssetsInstrumentCategory");
b.HasIndex("AssetsIsin", "AssetsInstrumentCategory");
b.ToTable("AssetEntityTagEntity");
});
modelBuilder.Entity("FinlyticAssets.Entities.AssetEntity", b =>
{
b.Property<string>("Isin")
.HasColumnType("text");
b.Property<string>("InstrumentCategory")
.HasColumnType("text");
b.Property<string>("AssetType")
.IsRequired()
.HasMaxLength(13)
.HasColumnType("character varying(13)");
b.Property<bool>("HasCfd")
.HasColumnType("boolean");
b.Property<string>("ImageId")
.HasColumnType("text");
b.Property<DateTime>("LastUpdatedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Isin", "InstrumentCategory");
b.HasIndex("LastUpdatedAt");
b.ToTable("TradeRepublicAssets");
b.HasDiscriminator<string>("AssetType").HasValue("AssetEntity");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("FinlyticAssets.Entities.Settings", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("AssetUpdateTypeDelay")
.HasColumnType("integer");
b.Property<int>("BatchAssetUpdateDelay")
.HasColumnType("integer");
b.Property<int>("CurrentScanningPage")
.HasColumnType("integer");
b.Property<string>("CurrentScanningType")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<bool>("FinishedInitialScan")
.HasColumnType("boolean");
b.Property<int>("InitAssetUpdateTypeDelay")
.HasColumnType("integer");
b.Property<int>("InitBatchAssetUpdateDelay")
.HasColumnType("integer");
b.Property<int>("TradeRepublicMaxRequestPageSize")
.HasColumnType("integer");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticAssets.Entities.TagEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("TradeRepublicTags");
});
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticAssets.Entities.BondEntity", b =>
{
b.HasBaseType("FinlyticAssets.Entities.AssetEntity");
b.Property<string>("BondIssuerName")
.IsRequired()
.HasColumnType("text");
b.Property<string>("SearchSubtitle")
.IsRequired()
.HasColumnType("text");
b.HasDiscriminator().HasValue("Bond");
});
modelBuilder.Entity("FinlyticAssets.Entities.CryptoEntity", b =>
{
b.HasBaseType("FinlyticAssets.Entities.AssetEntity");
b.Property<string>("SearchSubtitle")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Subtitle")
.IsRequired()
.HasColumnType("text");
b.ToTable("TradeRepublicAssets", t =>
{
t.Property("SearchSubtitle")
.HasColumnName("CryptoEntity_SearchSubtitle");
});
b.HasDiscriminator().HasValue("Crypto");
});
modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b =>
{
b.HasBaseType("FinlyticAssets.Entities.AssetEntity");
b.Property<decimal>("Barrier")
.HasColumnType("numeric(18,6)");
b.Property<string>("Currency")
.IsRequired()
.HasMaxLength(10)
.HasColumnType("character varying(10)");
b.Property<decimal?>("Delta")
.HasColumnType("numeric");
b.Property<string>("DerivativeProductCategories")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime?>("Expiry")
.HasColumnType("timestamp with time zone");
b.Property<decimal?>("Factor")
.HasColumnType("numeric");
b.Property<string>("Issuer")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<string>("IssuerDisplayName")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<string>("IssuerImageId")
.HasColumnType("text");
b.Property<decimal>("Leverage")
.HasColumnType("numeric(10,4)");
b.Property<string>("NextGenProductCategoryName")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<int>("OptionType")
.HasColumnType("integer");
b.Property<string>("ProductCategoryName")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<decimal?>("Size")
.HasColumnType("numeric");
b.Property<decimal>("Strike")
.HasColumnType("numeric(18,6)");
b.Property<string>("UnderlyingIsin")
.HasMaxLength(12)
.HasColumnType("character varying(12)");
b.HasDiscriminator().HasValue("Derivative");
});
modelBuilder.Entity("FinlyticAssets.Entities.EtfEntity", b =>
{
b.HasBaseType("FinlyticAssets.Entities.AssetEntity");
b.Property<string>("DerivativeProductCategories")
.IsRequired()
.HasColumnType("text");
b.Property<string>("EtfDescription")
.IsRequired()
.HasColumnType("text");
b.Property<string>("MappedEtfIndexName")
.IsRequired()
.HasColumnType("text");
b.Property<string>("SearchSubtitle")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Subtitle")
.IsRequired()
.HasColumnType("text");
b.ToTable("TradeRepublicAssets", t =>
{
t.Property("DerivativeProductCategories")
.HasColumnName("EtfEntity_DerivativeProductCategories");
t.Property("SearchSubtitle")
.HasColumnName("EtfEntity_SearchSubtitle");
t.Property("Subtitle")
.HasColumnName("EtfEntity_Subtitle");
});
b.HasDiscriminator().HasValue("Etf");
});
modelBuilder.Entity("FinlyticAssets.Entities.StockEntity", b =>
{
b.HasBaseType("FinlyticAssets.Entities.AssetEntity");
b.Property<string>("DerivativeProductCategories")
.IsRequired()
.HasColumnType("text");
b.ToTable("TradeRepublicAssets", t =>
{
t.Property("DerivativeProductCategories")
.HasColumnName("StockEntity_DerivativeProductCategories");
});
b.HasDiscriminator().HasValue("Stock");
});
modelBuilder.Entity("FinlyticAssets.Entities.SyntheticEntity", b =>
{
b.HasBaseType("FinlyticAssets.Entities.AssetEntity");
b.Property<string>("DerivativeProductCategories")
.IsRequired()
.HasColumnType("text");
b.ToTable("TradeRepublicAssets", t =>
{
t.Property("DerivativeProductCategories")
.HasColumnName("SyntheticEntity_DerivativeProductCategories");
});
b.HasDiscriminator().HasValue("Synthetic");
});
modelBuilder.Entity("AssetEntityTagEntity", b =>
{
b.HasOne("FinlyticAssets.Entities.TagEntity", null)
.WithMany()
.HasForeignKey("TagsId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("FinlyticAssets.Entities.AssetEntity", null)
.WithMany()
.HasForeignKey("AssetsIsin", "AssetsInstrumentCategory")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticAssets.Migrations
{
/// <inheritdoc />
public partial class UpdateDynamicSettingsUniqueIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings");
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings");
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key");
}
}
}
@@ -161,7 +161,8 @@ namespace FinlyticAssets.Migrations
b.HasKey("Id");
b.HasIndex("Key");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
+15 -10
View File
@@ -1,19 +1,27 @@
using System.Text.Json;
using FinlyticAssets;
using System;
using FinlyticAssets.Database;
using FinlyticCore.Services.TradeRepublic;
using FinlyticAssets.Util;
using FinlyticAssets.Services;
using FinlyticAssets.Util;
using FinlyticCore.Database;
using FinlyticCore.Services;
using FinlyticCore.Services.TradeRepublic;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
var builder = Host.CreateApplicationBuilder(args);
// DB Context
builder.Services.AddDbContext<AssetsDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddScoped<ISettingsDbContext>(sp => sp.GetRequiredService<AssetsDbContext>());
// Core Services
builder.Services.AddSingleton<ISettingsService, SettingsService>();
builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<>));
builder.Services.AddScoped<TradeRepublicClient>();
builder.Services.AddScoped<ITradeRepublicService, TradeRepublicService>();
builder.Services.AddScoped<IAssetsDbService, AssetsDbService>();
builder.Services.AddScoped<ISettingsDbService, SettingsDbService>();
@@ -34,8 +42,6 @@ using (var scope = host.Services.CreateScope())
var context = scope.ServiceProvider.GetRequiredService<AssetsDbContext>();
await context.Database.MigrateAsync();
// Fix: Reset InitAssetUpdateTypeDelay from old default (3600s) to new default (0s = no delay).
// This ensures the scanner moves immediately to the next asset type during the initial scan.
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
var settings = await settingsService.GetSettings();
if (settings.InitAssetUpdateTypeDelay == 3600)
@@ -47,8 +53,7 @@ using (var scope = host.Services.CreateScope())
}
catch (Exception ex)
{
Console.WriteLine($"Critical error during database migration: {ex.Message}");
Console.WriteLine(ex.StackTrace);
Console.WriteLine($"Critical error during database migration for FinlyticAssets: {ex.Message}");
}
}
@@ -4,12 +4,13 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Models;
using FinlyticAssets.Util;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Models.Assets;
using FinlyticCore.Services;
using FinlyticCore.Services.TradeRepublic;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticAssets.Services;
@@ -20,31 +21,31 @@ namespace FinlyticAssets.Services;
public class AssetScannerBackgroundService : BackgroundService
{
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<AssetScannerBackgroundService> _logger;
private readonly IFinlyticLogger<AssetScannerBackgroundService> _finlyticLogger;
private AssetsCount? _assetsCount;
private AssetsCount? _currAssetsCount;
public AssetScannerBackgroundService(IServiceScopeFactory serviceScopeFactory, ILogger<AssetScannerBackgroundService> logger)
public AssetScannerBackgroundService(IServiceScopeFactory serviceScopeFactory, IFinlyticLogger<AssetScannerBackgroundService> finlyticLogger)
{
_serviceScopeFactory = serviceScopeFactory;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("[{Channel}] AssetScannerBackgroundService has started.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] AssetScannerBackgroundService has started.");
try
{
using var scope = _serviceScopeFactory.CreateScope();
var indexService = scope.ServiceProvider.GetRequiredService<IAssetsIndexService>();
_logger.LogInformation("[{Channel}] Building initial asset index on service startup...", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Building initial asset index on service startup...");
await indexService.ReCreateIndexFileAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to build initial asset index on startup. Continuing service execution.", "AssetsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetScannerBackgroundService] Failed to build initial asset index on startup. Continuing service execution.");
}
do
@@ -57,7 +58,7 @@ public class AssetScannerBackgroundService : BackgroundService
var assetsDbService = scope.ServiceProvider.GetRequiredService<IAssetsDbService>();
var indexService = scope.ServiceProvider.GetRequiredService<IAssetsIndexService>();
_logger.LogInformation("[{Channel}] Requesting total asset counts from Trade Republic...", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Requesting total asset counts from Trade Republic...");
_assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken);
_currAssetsCount = new AssetsCount();
@@ -72,7 +73,7 @@ public class AssetScannerBackgroundService : BackgroundService
{
if (type != initSettings.CurrentScanningType)
{
_logger.LogInformation("[{Channel}] Recovery: {AssetType} was already processed. Skipping.", "AssetsChannel", type);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Recovery: {AssetType} was already processed. Skipping.", type);
continue;
}
isRecoveryMode = false;
@@ -85,7 +86,7 @@ public class AssetScannerBackgroundService : BackgroundService
await settingsService.SaveSettings(settings);
}
_logger.LogInformation("[{Channel}] Processing asset type: {AssetType}...", "AssetsChannel", type);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Processing asset type: {AssetType}...", type);
await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken);
var currentSettings = await settingsService.GetSettings();
@@ -96,7 +97,7 @@ public class AssetScannerBackgroundService : BackgroundService
if (delaySeconds > 0)
{
var jitter = Random.Shared.Next(0, Math.Min(15, delaySeconds));
_logger.LogInformation("[{Channel}] Waiting {Delay}s before next asset type ({Type}).", "AssetsChannel", delaySeconds + jitter, type);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Waiting {Delay}s before next asset type ({Type}).", delaySeconds + jitter, type);
await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken);
}
}
@@ -106,23 +107,23 @@ public class AssetScannerBackgroundService : BackgroundService
if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested)
{
_logger.LogInformation("[{Channel}] Initial scan successfully completed. Switching FinishedInitialScan to true.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Initial scan successfully completed. Switching FinishedInitialScan to true.");
finalSettings.FinishedInitialScan = true;
}
await settingsService.SaveSettings(finalSettings);
_logger.LogInformation("[{Channel}] Full scan cycle completed. Waiting 1 minute before starting the next cycle.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting 1 minute before starting the next cycle.");
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
}
catch (Exception e) when (!stoppingToken.IsCancellationRequested)
{
_logger.LogError(e, "[{Channel}] An unhandled exception occurred in AssetScannerBackgroundService. Retrying in 10 seconds.", "AssetsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, e, "[AssetScannerBackgroundService] An unhandled exception occurred in AssetScannerBackgroundService. Retrying in 10 seconds.");
try { await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); } catch { /* Ignore */ }
}
} while (!stoppingToken.IsCancellationRequested);
_logger.LogInformation("[{Channel}] AssetScannerBackgroundService is stopping.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] AssetScannerBackgroundService is stopping.");
}
private async Task HandleAssetType(
@@ -136,7 +137,7 @@ public class AssetScannerBackgroundService : BackgroundService
var totalCount = _assetsCount?.GetCountFromType(type) ?? 0;
if (totalCount == 0)
{
_logger.LogWarning("[{Channel}] No assets found for type {AssetType}.", "AssetsChannel", type);
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] No assets found for type {AssetType}.", type);
return;
}
@@ -149,8 +150,8 @@ public class AssetScannerBackgroundService : BackgroundService
if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0)
{
currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize;
_logger.LogInformation("[{Channel}] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).",
"AssetsChannel", type, settings.CurrentScanningPage, currentItemOffset);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).",
type, settings.CurrentScanningPage, currentItemOffset);
}
while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested)
@@ -164,15 +165,14 @@ public class AssetScannerBackgroundService : BackgroundService
currentSettings.CurrentScanningPage = currentPage;
await settingsDbService.SaveSettings(currentSettings);
_logger.LogDebug("Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}",
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}",
type, currentPage, currentItemOffset, totalCount);
var assets = await tradeRepublicService.GetAssets(type, currentPage, pageSize, stoppingToken);
// Keine Ergebnisse geliefert -> Katalogende erreicht
if (assets?.Results == null || assets.Results.Count == 0)
{
_logger.LogInformation("[{Channel}] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", "AssetsChannel", type, currentPage);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", type, currentPage);
currentSettings.CurrentScanningPage = 0;
await settingsDbService.SaveSettings(currentSettings);
break;
@@ -183,10 +183,9 @@ public class AssetScannerBackgroundService : BackgroundService
currentItemOffset += assets.Results.Count;
// Unvollständige Seite -> Letzte Seite abgearbeitet
if (assets.Results.Count < pageSize)
{
_logger.LogInformation("[{Channel}] Reached the last page for {AssetType}.", "AssetsChannel", type);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Reached the last page for {AssetType}.", type);
currentSettings.CurrentScanningPage = 0;
await settingsDbService.SaveSettings(currentSettings);
break;
@@ -198,16 +197,15 @@ public class AssetScannerBackgroundService : BackgroundService
if (delaySeconds > 0)
{
// Angemessener Jitter (0 bis max. 5 Sek. bzw. kleiner als delaySeconds)
var maxJitter = Math.Min(5, delaySeconds);
var jitter = Random.Shared.Next(0, maxJitter + 1);
_logger.LogDebug("Waiting {Delay} seconds before the next batch.", delaySeconds + jitter);
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Waiting {Delay} seconds before the next batch.", delaySeconds + jitter);
await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken);
}
}
_logger.LogInformation("[{Channel}] Finished scanning {AssetType}. Total scanned in this cycle: {Count}/{Total}",
"AssetsChannel", type, _currAssetsCount.GetCountFromType(type), totalCount);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Finished scanning {AssetType}. Total scanned in this cycle: {Count}/{Total}",
type, _currAssetsCount.GetCountFromType(type), totalCount);
}
private async Task ProcessAssets(
@@ -219,11 +217,11 @@ public class AssetScannerBackgroundService : BackgroundService
if (assets == null || assets.Count == 0) return;
var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets);
_logger.LogInformation("[{Channel}] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", "AssetsChannel", assets.Count, changedRows);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows);
if (changedRows > 0)
{
_logger.LogInformation("[{Channel}] Database modifications detected. Recreating the asset index file...", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Database modifications detected. Recreating the asset index file...");
await indexService.ReCreateIndexFileAsync(stoppingToken);
}
}
+206 -273
View File
@@ -1,6 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Database;
using FinlyticAssets.Entities;
using FinlyticAssets.Util;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Services;
using FinlyticCore.Services.TradeRepublic;
using Microsoft.EntityFrameworkCore;
@@ -28,12 +35,12 @@ public class AssetsDbService : IAssetsDbService
{
private readonly AssetsDbContext _context;
private readonly ITradeRepublicService _tradeRepublicService;
private readonly ILogger<AssetsDbService> _logger;
private readonly IFinlyticLogger<AssetsDbService> _finlyticLogger;
public AssetsDbService(AssetsDbContext context, ILogger<AssetsDbService> logger, ITradeRepublicService tradeRepublicService)
public AssetsDbService(AssetsDbContext context, IFinlyticLogger<AssetsDbService> finlyticLogger, ITradeRepublicService tradeRepublicService)
{
_context = context;
_logger = logger;
_finlyticLogger = finlyticLogger;
_tradeRepublicService = tradeRepublicService;
}
@@ -60,42 +67,41 @@ public class AssetsDbService : IAssetsDbService
+ (a.Name.Length > 3 ? 5 : 0)
})
.OrderByDescending(x => x.Score)
.ThenByDescending(x => x.Asset.LastUpdatedAt)
.Take(limit)
.Select(x => x.Asset)
.ToList();
var result = new List<AssetEntity>();
var grouped = scored.GroupBy(x => x.Asset.Type).ToList();
int index = 0;
while (result.Count < limit && grouped.Any(g => g.Any()))
{
bool addedAny = false;
foreach (var group in grouped)
{
var item = group.Skip(index).FirstOrDefault();
if (item != null)
{
result.Add(item.Asset);
addedAny = true;
if (result.Count >= limit) break;
}
}
index++;
if (!addedAny) break;
}
return result;
return scored;
}
/// <summary>Inherits documentation from interface.</summary>
public async Task<List<AssetEntity>> GetAllValidAssetsAsync()
{
var cutoff = DateTime.UtcNow.AddDays(-90);
return await _context.TradeRepublicAssets
.AsNoTracking()
.Where(a => a.LastUpdatedAt >= cutoff)
.ToListAsync();
}
/// <summary>Inherits documentation from interface.</summary>
public async Task<List<AssetEntity>> GetAssetsByIsinAsync(string isin)
{
return await _context.TradeRepublicAssets
.AsNoTracking()
.Include(a => a.Tags)
.Where(a => a.LastUpdatedAt >= cutoff)
.Where(a => a.Isin == isin)
.ToListAsync();
}
/// <summary>Inherits documentation from interface.</summary>
public async Task<List<AssetEntity>> GetValidAssetsByIsinAsync(string isin)
{
var cutoff = DateTime.UtcNow.AddDays(-90);
return await _context.TradeRepublicAssets
.AsNoTracking()
.Include(a => a.Tags)
.Where(a => a.Isin == isin && a.LastUpdatedAt >= cutoff)
.ToListAsync();
}
@@ -114,7 +120,7 @@ public class AssetsDbService : IAssetsDbService
var existingTag = await _context.TradeRepublicTags.FirstOrDefaultAsync(t => t.Id == tagDto.Id);
if (existingTag == null)
{
_logger.LogTrace("Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id);
await _finlyticLogger.LogTraceAsync(SettingKeys.AssetsChannel, "Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id);
existingTag = new TagEntity { Id = tagDto.Id, Name = tagDto.Name, Type = tagDto.Type };
await _context.TradeRepublicTags.AddAsync(existingTag);
}
@@ -124,7 +130,7 @@ public class AssetsDbService : IAssetsDbService
if (existingEntity == null)
{
_logger.LogDebug("Asset with ISIN {Isin} not found. Mapping and creating a new entity.", dtoAsset.Isin);
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "Asset with ISIN {Isin} not found. Mapping and creating a new entity.", dtoAsset.Isin);
var newEntity = MapDtoToEntity(dtoAsset);
newEntity.LastUpdatedAt = now;
@@ -135,7 +141,7 @@ public class AssetsDbService : IAssetsDbService
return true;
}
_logger.LogDebug("Asset with ISIN {Isin} exists. Merging properties and updating database record.", dtoAsset.Isin);
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "Asset with ISIN {Isin} exists. Merging properties and updating database record.", dtoAsset.Isin);
existingEntity.Name = dtoAsset.Name;
existingEntity.Type = dtoAsset.Type;
@@ -187,7 +193,7 @@ public class AssetsDbService : IAssetsDbService
{
if (!tagCache.TryGetValue(tagDto.Id, out var tagEntity))
{
_logger.LogTrace("Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id);
await _finlyticLogger.LogTraceAsync(SettingKeys.AssetsChannel, "Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id);
tagEntity = new TagEntity { Id = tagDto.Id, Name = tagDto.Name, Type = tagDto.Type };
await _context.TradeRepublicTags.AddAsync(tagEntity);
tagCache.Add(tagDto.Id, tagEntity);
@@ -197,7 +203,7 @@ public class AssetsDbService : IAssetsDbService
if (existingEntity == null)
{
_logger.LogDebug("Asset with ISIN {Isin} not found. Mapping and creating a new entity.", dto.Isin);
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "Asset with ISIN {Isin} not found. Mapping and creating a new entity.", dto.Isin);
var newEntity = MapDtoToEntity(dto);
newEntity.LastUpdatedAt = now;
@@ -208,7 +214,7 @@ public class AssetsDbService : IAssetsDbService
}
else
{
_logger.LogDebug("Asset with ISIN {Isin} exists. Merging properties and updating database record.", dto.Isin);
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "Asset with ISIN {Isin} exists. Merging properties and updating database record.", dto.Isin);
if (existingEntity.Name != dto.Name ||
existingEntity.Type != dto.Type ||
@@ -223,140 +229,137 @@ public class AssetsDbService : IAssetsDbService
existingEntity.HasCfd = dto.HasCfd;
existingEntity.ImageId = dto.ImageId;
existingEntity.LastUpdatedAt = now;
UpdateSubtypeProperties(existingEntity, dto);
existingEntity.Tags = mappedTags;
UpdateSubtypeProperties(existingEntity, dto);
_context.TradeRepublicAssets.Update(existingEntity);
isChanged = true;
}
}
if (isChanged)
{
changedCount++;
}
}
if (changedCount > 0)
{
await _context.SaveChangesAsync();
if (isChanged) changedCount++;
}
await _context.SaveChangesAsync();
return changedCount;
}
/// <summary>Inherits documentation from interface.</summary>
public async Task<List<AssetEntity>> GetAssetsByIsinAsync(string isin)
private static AssetEntity MapDtoToEntity(TradeRepublicAsset dto)
{
var localAssets = await _context.TradeRepublicAssets
.Include(a => a.Tags)
.Where(a => a.Isin == isin)
.ToListAsync();
if (localAssets.Count > 0)
return dto switch
{
return localAssets;
}
// JIT-Fetch via API
var trAssetDto = await _tradeRepublicService.GetAsset(isin);
if (trAssetDto?.Results != null && trAssetDto.Results.Count > 0)
{
foreach (var asset in trAssetDto.Results)
TradeRepublicStock stock => new StockEntity
{
await AddOrUpdateAssetAsync(asset);
Isin = stock.Isin,
Name = stock.Name,
Type = stock.Type,
InstrumentCategory = stock.InstrumentCategory,
HasCfd = stock.HasCfd,
ImageId = stock.ImageId,
DerivativeProductCategories = stock.DerivativeProductCategories?.ToList() ?? new List<string>()
},
TradeRepublicCrypto crypto => new CryptoEntity
{
Isin = crypto.Isin,
Name = crypto.Name,
Type = crypto.Type,
InstrumentCategory = crypto.InstrumentCategory,
HasCfd = crypto.HasCfd,
ImageId = crypto.ImageId
},
TradeRepublicEtf etf => new EtfEntity
{
Isin = etf.Isin,
Name = etf.Name,
Type = etf.Type,
InstrumentCategory = etf.InstrumentCategory,
HasCfd = etf.HasCfd,
ImageId = etf.ImageId,
DerivativeProductCategories = etf.DerivativeProductCategories?.ToList() ?? new List<string>()
},
TradeRepublicSynthetic syn => new SyntheticEntity
{
Isin = syn.Isin,
Name = syn.Name,
Type = syn.Type,
InstrumentCategory = syn.InstrumentCategory,
HasCfd = syn.HasCfd,
ImageId = syn.ImageId,
DerivativeProductCategories = syn.DerivativeProductCategories?.ToList() ?? new List<string>()
},
TradeRepublicBond bond => new BondEntity
{
Isin = bond.Isin,
Name = bond.Name,
Type = bond.Type,
InstrumentCategory = bond.InstrumentCategory,
HasCfd = bond.HasCfd,
ImageId = bond.ImageId,
BondIssuerName = bond.BondIssuerName,
SearchSubtitle = bond.SearchSubtitle
},
TradeRepublicDerivative deriv => new DerivativeEntity
{
Isin = deriv.Isin,
Name = deriv.Name,
Type = deriv.Type,
InstrumentCategory = deriv.InstrumentCategory,
HasCfd = deriv.HasCfd,
ImageId = deriv.ImageId,
UnderlyingIsin = deriv.UnderlyingIsin,
DerivativeProductCategories = deriv.DerivativeProductCategories?.ToList() ?? new List<string>()
},
_ => new StockEntity
{
Isin = dto.Isin,
Name = dto.Name,
Type = dto.Type,
InstrumentCategory = dto.InstrumentCategory,
HasCfd = dto.HasCfd,
ImageId = dto.ImageId
}
return await _context.TradeRepublicAssets
.Include(a => a.Tags)
.Where(a => a.Isin == isin)
.ToListAsync();
}
return [];
};
}
/// <summary>Inherits documentation from interface.</summary>
public async Task<List<AssetEntity>> GetValidAssetsByIsinAsync(string isin)
private static void UpdateSubtypeProperties(AssetEntity entity, TradeRepublicAsset dto)
{
var cutoff = DateTime.UtcNow.AddDays(-14);
return await _context.TradeRepublicAssets
.AsNoTracking()
.Include(a => a.Tags)
.Where(a => a.Isin == isin && a.LastUpdatedAt >= cutoff)
.ToListAsync();
switch (entity)
{
case StockEntity stock when dto is TradeRepublicStock s:
stock.DerivativeProductCategories = s.DerivativeProductCategories?.ToList() ?? new List<string>();
break;
case EtfEntity etf when dto is TradeRepublicEtf e:
etf.DerivativeProductCategories = e.DerivativeProductCategories?.ToList() ?? new List<string>();
break;
case SyntheticEntity syn when dto is TradeRepublicSynthetic synDto:
syn.DerivativeProductCategories = synDto.DerivativeProductCategories?.ToList() ?? new List<string>();
break;
case BondEntity bond when dto is TradeRepublicBond b:
bond.BondIssuerName = b.BondIssuerName;
bond.SearchSubtitle = b.SearchSubtitle;
break;
case DerivativeEntity deriv when dto is TradeRepublicDerivative d:
deriv.UnderlyingIsin = d.UnderlyingIsin;
deriv.DerivativeProductCategories = d.DerivativeProductCategories?.ToList() ?? new List<string>();
break;
}
}
/// <summary>Inherits documentation from interface.</summary>
public async Task<List<AssetEntity>> FindAffectedActiveAssetsAsync(string searchQuery)
{
if (string.IsNullOrWhiteSpace(searchQuery)) return [];
var cutoff = DateTime.UtcNow.AddDays(-90);
string cleanQuery = searchQuery.Trim().ToLowerInvariant();
var cutoff = DateTime.UtcNow.AddDays(-14);
var searchTerms = searchQuery
.Split(',')
.Select(t => t.Trim())
.Where(t => !string.IsNullOrEmpty(t))
.Distinct()
.ToList();
if (searchTerms.Count == 0) return [];
var query = _context.TradeRepublicAssets
return await _context.TradeRepublicAssets
.AsNoTracking()
.Where(a => a.LastUpdatedAt >= cutoff)
.Include(a => a.Tags)
.AsQueryable();
foreach (var term in searchTerms)
{
var lowerTerm = term.ToLower();
query = query.Where(a =>
a.Isin.ToLower().Contains(lowerTerm) ||
a.Name.ToLower().Contains(lowerTerm) ||
a.Tags.Any(tag => tag.Name.ToLower().Contains(lowerTerm)));
}
var localAssets = await query.ToListAsync();
var possibleIsins = searchTerms
.Where(t => t.Length == 12 && char.IsLetter(t[0]) && char.IsLetter(t[1]))
.Select(t => t.ToUpper())
.ToList();
if (possibleIsins.Count > 0)
{
var foundIsins = localAssets.Select(a => a.Isin).ToHashSet();
var missingIsins = possibleIsins.Where(isin => !foundIsins.Contains(isin)).ToList();
if (missingIsins.Count > 0)
{
var fetchedNewAsset = false;
foreach (var missingIsin in missingIsins)
{
var trAssetDto = await _tradeRepublicService.GetAsset(missingIsin);
if (trAssetDto?.Results != null)
{
foreach (var asset in trAssetDto.Results)
{
await AddOrUpdateAssetAsync(asset);
}
fetchedNewAsset = true;
}
}
if (fetchedNewAsset)
{
return await query.ToListAsync();
}
}
}
return localAssets;
.Where(a => a.LastUpdatedAt >= cutoff && (
a.Isin.ToLower().Contains(cleanQuery) ||
a.Name.ToLower().Contains(cleanQuery)
))
.Take(25)
.ToListAsync();
}
/// <summary>Inherits documentation from interface.</summary>
@@ -373,7 +376,7 @@ public class AssetsDbService : IAssetsDbService
asset.ImageId = imageId;
}
await _context.SaveChangesAsync();
_logger.LogInformation("[{Channel}] Updated ImageId for ISIN {Isin} in database to '{ImageId}'", "AssetsChannel", isin, imageId);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Updated ImageId for ISIN {Isin} in database to '{ImageId}'", isin, imageId);
}
}
@@ -383,13 +386,13 @@ public class AssetsDbService : IAssetsDbService
var asset = await _context.TradeRepublicAssets.FirstOrDefaultAsync(a => a.Isin == isin);
if (asset == null)
{
_logger.LogWarning("[{Channel}] Delete execution cancelled. Asset with ISIN {Isin} does not exist.", "AssetsChannel", isin);
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Delete execution cancelled. Asset with ISIN {Isin} does not exist.", isin);
return false;
}
_context.TradeRepublicAssets.Remove(asset);
await _context.SaveChangesAsync();
_logger.LogInformation("[{Channel}] Asset with ISIN {Isin} has been successfully deleted.", "AssetsChannel", isin);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Asset with ISIN {Isin} has been successfully deleted.", isin);
return true;
}
@@ -410,11 +413,10 @@ public class AssetsDbService : IAssetsDbService
decimal levQuery = targetLeverage.HasValue && targetLeverage.Value > 0 ? targetLeverage.Value : 0m;
// Trade Republic uses page index (0, 1, 2, 3...) for the 'after' pagination parameter in derivatives
string trAfter = !string.IsNullOrEmpty(after) ? after : (pageIndex > 0 ? pageIndex.ToString() : "0");
_logger.LogInformation("[{Channel}] Fetching derivatives for {Isin} (OptionType: {Option}, Leverage: {Lev}, Page: {Page}, TR-After: {After})",
"AssetsChannel", underlyingIsin, cleanOptionType, levQuery, pageIndex, trAfter);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Fetching derivatives for {Isin} (OptionType: {Option}, Leverage: {Lev}, Page: {Page}, TR-After: {After})",
underlyingIsin, cleanOptionType, levQuery, pageIndex, trAfter);
var trReq = new TradeRepublicDerivativesRequest(
Underlying: underlyingIsin,
@@ -429,8 +431,8 @@ public class AssetsDbService : IAssetsDbService
var trResponse = await _tradeRepublicService.GetDerivativesAsync(trReq, cancellationToken);
var fetchedItems = trResponse?.Results ?? new List<TradeRepublicDerivativeItemDto>();
_logger.LogInformation("[{Channel}] TR returned {Count} derivatives for {Isin} (Cursors.After: {NextAfter})",
"AssetsChannel", fetchedItems.Count, underlyingIsin, trResponse?.Cursors?.After ?? "null");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] TR returned {Count} derivatives for {Isin} (Cursors.After: {NextAfter})",
fetchedItems.Count, underlyingIsin, trResponse?.Cursors?.After ?? "null");
if (fetchedItems.Count > 0)
{
@@ -445,144 +447,75 @@ public class AssetsDbService : IAssetsDbService
foreach (var item in fetchedItems)
{
if (!existingDerivatives.TryGetValue(item.Isin, out var entity))
DateTime? expiryDate = null;
if (!string.IsNullOrWhiteSpace(item.Expiry) && DateTime.TryParse(item.Expiry, out var parsedExp))
{
entity = new DerivativeEntity
{
Isin = item.Isin,
InstrumentCategory = "derivative",
Type = "derivative"
};
await _context.TradeRepublicAssets.AddAsync(entity, cancellationToken);
expiryDate = parsedExp.ToUniversalTime();
}
bool isShortItem = string.Equals(item.OptionType, "short", StringComparison.OrdinalIgnoreCase) ||
string.Equals(item.OptionType, "put", StringComparison.OrdinalIgnoreCase) ||
item.OptionType.Contains("short", StringComparison.OrdinalIgnoreCase) ||
item.OptionType.Contains("put", StringComparison.OrdinalIgnoreCase) ||
item.OptionType.Contains("bear", StringComparison.OrdinalIgnoreCase);
if (existingDerivatives.TryGetValue(item.Isin, out var existing))
{
existing.Name = !string.IsNullOrWhiteSpace(item.ProductCategoryName) ? item.ProductCategoryName : item.Isin;
existing.UnderlyingIsin = underlyingIsin;
existing.Strike = item.Strike ?? 0m;
existing.Barrier = item.Barrier ?? 0m;
existing.Leverage = item.Leverage ?? 0m;
existing.Expiry = expiryDate;
existing.OptionType = targetOptionType;
existing.ProductCategoryName = item.ProductCategoryName;
existing.NextGenProductCategoryName = item.NextGenProductCategoryName;
existing.Issuer = item.Issuer;
existing.IssuerDisplayName = item.IssuerDisplayName;
existing.IssuerImageId = item.IssuerImageId;
existing.Size = item.Size;
existing.Factor = item.Factor;
existing.Delta = item.Delta;
existing.Currency = item.Currency;
existing.LastUpdatedAt = now;
entity.UnderlyingIsin = underlyingIsin;
entity.OptionType = isShortItem ? OptionType.Short : OptionType.Long;
entity.ProductCategoryName = item.ProductCategoryName;
entity.NextGenProductCategoryName = item.NextGenProductCategoryName;
entity.Strike = item.Strike ?? 0m;
entity.Barrier = item.Barrier ?? 0m;
entity.Leverage = item.Leverage ?? 0m;
entity.Size = item.Size;
entity.Factor = item.Factor;
entity.Delta = item.Delta;
entity.Currency = item.Currency ?? "EUR";
entity.Expiry = DateTime.TryParse(item.Expiry, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal | System.Globalization.DateTimeStyles.AssumeUniversal, out var exp)
? DateTime.SpecifyKind(exp, DateTimeKind.Utc)
: (DateTime?)null;
entity.Issuer = item.Issuer;
entity.IssuerDisplayName = item.IssuerDisplayName;
entity.IssuerImageId = item.IssuerImageId;
entity.ImageId = item.ImageId;
entity.Name = $"{item.IssuerDisplayName} {item.NextGenProductCategoryName} ({(isShortItem ? "SHORT" : "LONG")})";
entity.LastUpdatedAt = now;
_context.TradeRepublicAssets.Update(existing);
resultEntities.Add(existing);
}
else
{
var newDeriv = new DerivativeEntity
{
Isin = item.Isin,
Name = !string.IsNullOrWhiteSpace(item.ProductCategoryName) ? item.ProductCategoryName : item.Isin,
Type = "derivative",
InstrumentCategory = "derivative",
UnderlyingIsin = underlyingIsin,
Strike = item.Strike ?? 0m,
Barrier = item.Barrier ?? 0m,
Leverage = item.Leverage ?? 0m,
Expiry = expiryDate,
OptionType = targetOptionType,
ProductCategoryName = item.ProductCategoryName,
NextGenProductCategoryName = item.NextGenProductCategoryName,
Issuer = item.Issuer,
IssuerDisplayName = item.IssuerDisplayName,
IssuerImageId = item.IssuerImageId,
Size = item.Size,
Factor = item.Factor,
Delta = item.Delta,
Currency = item.Currency,
LastUpdatedAt = now
};
resultEntities.Add(entity);
await _context.TradeRepublicAssets.AddAsync(newDeriv, cancellationToken);
resultEntities.Add(newDeriv);
}
}
await _context.SaveChangesAsync(cancellationToken);
return resultEntities;
}
// Fallback: Query from DB if Trade Republic returned 0 or was unreachable
var dbQuery = _context.TradeRepublicAssets
return await _context.TradeRepublicAssets
.OfType<DerivativeEntity>()
.AsNoTracking()
.Include(a => a.Tags)
.Where(d => d.UnderlyingIsin == underlyingIsin && d.OptionType == targetOptionType);
if (levQuery > 0)
{
dbQuery = dbQuery.Where(d => d.Leverage >= (levQuery - 0.2m));
}
var results = await dbQuery
.OrderBy(d => d.Leverage)
.Skip(pageIndex * pageSize)
.Where(d => d.UnderlyingIsin == underlyingIsin)
.Take(pageSize)
.ToListAsync(cancellationToken);
return results;
}
#region Helper & Mapping Methods
private AssetEntity MapDtoToEntity(TradeRepublicAsset dto)
{
AssetEntity entity = dto switch
{
TradeRepublicStock stock => new StockEntity
{ Isin = stock.Isin, DerivativeProductCategories = stock.DerivativeProductCategories.ToList() },
TradeRepublicCrypto crypto => new CryptoEntity
{ Isin = crypto.Isin, Subtitle = crypto.Subtitle, SearchSubtitle = crypto.SearchSubtitle },
TradeRepublicEtf etf => new EtfEntity
{
Isin = etf.Isin, EtfDescription = etf.EtfDescription, MappedEtfIndexName = etf.MappedEtfIndexName,
Subtitle = etf.Subtitle, SearchSubtitle = etf.SearchSubtitle,
DerivativeProductCategories = etf.DerivativeProductCategories.ToList()
},
TradeRepublicSynthetic synth => new SyntheticEntity
{ Isin = synth.Isin, DerivativeProductCategories = synth.DerivativeProductCategories.ToList() },
TradeRepublicBond bond => new BondEntity
{ Isin = bond.Isin, BondIssuerName = bond.BondIssuerName, SearchSubtitle = bond.SearchSubtitle },
TradeRepublicDerivative deriv => new DerivativeEntity
{
Isin = deriv.Isin, UnderlyingIsin = deriv.UnderlyingIsin,
DerivativeProductCategories = deriv.DerivativeProductCategories.ToList()
},
_ => throw new NotSupportedException($"Type {dto.GetType().Name} is not supported.")
};
return PopulateBaseProperties(entity, dto);
}
private AssetEntity PopulateBaseProperties(AssetEntity entity, TradeRepublicAsset dto)
{
entity.Name = dto.Name;
entity.Type = dto.Type;
entity.InstrumentCategory = dto.InstrumentCategory;
entity.HasCfd = dto.HasCfd;
entity.ImageId = dto.ImageId;
return entity;
}
private void UpdateSubtypeProperties(AssetEntity entity, TradeRepublicAsset dto)
{
switch (entity)
{
case StockEntity stockEntity when dto is TradeRepublicStock stockDto:
stockEntity.DerivativeProductCategories = stockDto.DerivativeProductCategories.ToList();
break;
case CryptoEntity cryptoEntity when dto is TradeRepublicCrypto cryptoDto:
cryptoEntity.Subtitle = cryptoDto.Subtitle;
cryptoEntity.SearchSubtitle = cryptoDto.SearchSubtitle;
break;
case EtfEntity etfEntity when dto is TradeRepublicEtf etfDto:
etfEntity.DerivativeProductCategories = etfDto.DerivativeProductCategories.ToList();
etfEntity.EtfDescription = etfDto.EtfDescription;
etfEntity.MappedEtfIndexName = etfDto.MappedEtfIndexName;
etfEntity.Subtitle = etfDto.Subtitle;
etfEntity.SearchSubtitle = etfDto.SearchSubtitle;
break;
case SyntheticEntity synthEntity when dto is TradeRepublicSynthetic synthDto:
synthEntity.DerivativeProductCategories = synthDto.DerivativeProductCategories.ToList();
break;
case BondEntity bondEntity when dto is TradeRepublicBond bondDto:
bondEntity.BondIssuerName = bondDto.BondIssuerName;
bondEntity.SearchSubtitle = bondDto.SearchSubtitle;
break;
case DerivativeEntity derivEntity when dto is TradeRepublicDerivative derivDto:
derivEntity.DerivativeProductCategories = derivDto.DerivativeProductCategories.ToList();
derivEntity.UnderlyingIsin = derivDto.UnderlyingIsin;
break;
}
}
#endregion
}
+18 -20
View File
@@ -1,7 +1,13 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Models;
using FinlyticAssets.Util;
using FinlyticCore.Services;
namespace FinlyticAssets.Services;
@@ -13,8 +19,6 @@ public interface IAssetsIndexService
/// <summary>
/// Recreates the index file containing basic asset identifiers (ISIN and Name) for all active, valid assets.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task ReCreateIndexFileAsync(CancellationToken cancellationToken = default);
/// <summary>
@@ -28,18 +32,13 @@ public interface IAssetsIndexService
/// </summary>
public class AssetsIndexService : IAssetsIndexService
{
private readonly ILogger<AssetsIndexService> _logger;
private readonly IFinlyticLogger<AssetsIndexService> _finlyticLogger;
private readonly IAssetsDbService _assetsDbService;
private static readonly HttpClient _httpClient = new();
/// <summary>
/// Initializes a new instance of the <see cref="AssetsIndexService"/> class.
/// </summary>
/// <param name="logger">The logger for documenting indexing events and errors.</param>
/// <param name="assetsDbService">The database service to query the assets from.</param>
public AssetsIndexService(ILogger<AssetsIndexService> logger, IAssetsDbService assetsDbService)
public AssetsIndexService(IFinlyticLogger<AssetsIndexService> finlyticLogger, IAssetsDbService assetsDbService)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_assetsDbService = assetsDbService;
}
@@ -51,7 +50,7 @@ public class AssetsIndexService : IAssetsIndexService
var assets = await _assetsDbService.GetAllValidAssetsAsync();
if (assets == null || !assets.Any())
{
_logger.LogWarning("[{Channel}] No valid assets found in the database to index.", "AssetsChannel");
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[AssetsIndexService] No valid assets found in the database to index.");
return;
}
@@ -59,7 +58,6 @@ public class AssetsIndexService : IAssetsIndexService
.DistinctBy(a => a.Isin)
.Select(a => {
string cleanIsin = a.Isin.Trim().ToUpperInvariant();
// Point directly to our own local backend logo endpoint
string imageUrl = $"/api/v1/logo/{cleanIsin}";
return new AssetIndex(cleanIsin, a.Name, imageUrl);
}).ToList();
@@ -78,22 +76,22 @@ public class AssetsIndexService : IAssetsIndexService
await JsonSerializer.SerializeAsync(fileStream, indexAssets, cancellationToken: cancellationToken);
}
_logger.LogInformation("[{Channel}] Successfully recreated asset index file with {Count} entries pointing to local logos at {Path}",
"AssetsChannel", indexAssets.Count, filePath);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsIndexService] Successfully recreated asset index file with {Count} entries pointing to local logos at {Path}",
indexAssets.Count, filePath);
}
catch (IOException ex)
{
_logger.LogError(ex, "[{Channel}] Disk I/O error occurred while writing the asset index file.", "AssetsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsIndexService] Disk I/O error occurred while writing the asset index file.");
throw;
}
catch (JsonException ex)
{
_logger.LogError(ex, "[{Channel}] Failed to serialize the asset index data to JSON.", "AssetsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsIndexService] Failed to serialize the asset index data to JSON.");
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] An unexpected error occurred while recreating the asset index file.", "AssetsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsIndexService] An unexpected error occurred while recreating the asset index file.");
throw;
}
}
@@ -131,13 +129,13 @@ public class AssetsIndexService : IAssetsIndexService
{
byte[] data = await response.Content.ReadAsByteArrayAsync(cancellationToken);
await File.WriteAllBytesAsync(filePath, data, cancellationToken);
_logger.LogInformation("[{Channel}] Successfully saved logo SVG for ISIN {Isin} to {Path} on demand", "AssetsChannel", cleanIsin, filePath);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsIndexService] Successfully saved logo SVG for ISIN {Isin} to {Path} on demand", cleanIsin, filePath);
return filePath;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to download logo for ISIN {Isin} from {Url}", "AssetsChannel", cleanIsin, targetUrl);
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[AssetsIndexService] Failed to download logo for ISIN {Isin} from {Url}", cleanIsin, targetUrl);
}
return null;
@@ -6,9 +6,9 @@ using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Util;
using FinlyticCore.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticAssets.Services;
@@ -19,7 +19,7 @@ namespace FinlyticAssets.Services;
/// </summary>
public class LogoFetcherBackgroundService : BackgroundService
{
private readonly ILogger<LogoFetcherBackgroundService> _logger;
private readonly IFinlyticLogger<LogoFetcherBackgroundService> _finlyticLogger;
private readonly IServiceScopeFactory _scopeFactory;
private readonly HttpClient _httpClient;
@@ -32,10 +32,10 @@ public class LogoFetcherBackgroundService : BackgroundService
""";
public LogoFetcherBackgroundService(
ILogger<LogoFetcherBackgroundService> logger,
IFinlyticLogger<LogoFetcherBackgroundService> finlyticLogger,
IServiceScopeFactory scopeFactory)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_scopeFactory = scopeFactory;
_httpClient = new HttpClient();
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
@@ -45,7 +45,7 @@ public class LogoFetcherBackgroundService : BackgroundService
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("[{Channel}] LogoFetcherBackgroundService started. Will fetch missing logos periodically.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] LogoFetcherBackgroundService started. Will fetch missing logos periodically.");
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
@@ -57,7 +57,7 @@ public class LogoFetcherBackgroundService : BackgroundService
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
_logger.LogError(ex, "[{Channel}] Error occurred while executing logo batch fetch.", "AssetsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Error occurred while executing logo batch fetch.");
}
try
@@ -70,7 +70,7 @@ public class LogoFetcherBackgroundService : BackgroundService
}
}
_logger.LogInformation("[{Channel}] LogoFetcherBackgroundService stopped.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] LogoFetcherBackgroundService stopped.");
}
private async Task ProcessMissingLogosBatchAsync(CancellationToken stoppingToken)
@@ -88,7 +88,6 @@ public class LogoFetcherBackgroundService : BackgroundService
Directory.CreateDirectory(directoryPath);
}
// ✅ Prüft sowohl DB-Eintrag ALS AUCH, ob die Datei bereits lokal existiert
var missingIsins = validAssets
.Select(a => a.Isin?.Trim().ToUpperInvariant())
.Where(isin => !string.IsNullOrEmpty(isin))
@@ -98,12 +97,12 @@ public class LogoFetcherBackgroundService : BackgroundService
if (missingIsins.Count == 0)
{
_logger.LogDebug("All asset logos are downloaded and up to date.");
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] All asset logos are downloaded and up to date.");
return;
}
var batchToFetch = missingIsins.Take(60).ToList();
_logger.LogInformation("[{Channel}] Found {Count} missing logos on disk. Fetching bulk batch of {BatchSize} logos...", "AssetsChannel", missingIsins.Count, batchToFetch.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Found {Count} missing logos on disk. Fetching bulk batch of {BatchSize} logos...", missingIsins.Count, batchToFetch.Count);
int successCount = 0;
@@ -126,7 +125,7 @@ public class LogoFetcherBackgroundService : BackgroundService
}
else
{
_logger.LogWarning("[{Channel}] Logo not found on CDN for ISIN {Isin} (HTTP {StatusCode}). Saving SVG placeholder.", "AssetsChannel", isin, response.StatusCode);
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Logo not found on CDN for ISIN {Isin} (HTTP {StatusCode}). Saving SVG placeholder.", isin, response.StatusCode);
byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg);
await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken);
successCount++;
@@ -136,7 +135,7 @@ public class LogoFetcherBackgroundService : BackgroundService
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
_logger.LogWarning(ex, "[{Channel}] Exception while downloading logo for ISIN {Isin} from {Url}. Saving SVG placeholder.", "AssetsChannel", isin, targetUrl);
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Exception while downloading logo for ISIN {Isin} from {Url}. Saving SVG placeholder.", isin, targetUrl);
try
{
byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg);
@@ -148,23 +147,22 @@ public class LogoFetcherBackgroundService : BackgroundService
catch { }
}
// Kurze Pause gegen Rate Limiting
await Task.Delay(50, stoppingToken);
}
_logger.LogInformation("[{Channel}] Batch fetch complete. Successfully processed {SuccessCount}/{BatchSize} logos. Remaining missing: {Remaining}",
"AssetsChannel", successCount, batchToFetch.Count, missingIsins.Count - batchToFetch.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Batch fetch complete. Successfully processed {SuccessCount}/{BatchSize} logos. Remaining missing: {Remaining}",
successCount, batchToFetch.Count, missingIsins.Count - batchToFetch.Count);
if (successCount > 0)
{
try
{
await indexService.ReCreateIndexFileAsync(stoppingToken);
_logger.LogInformation("[{Channel}] Successfully updated index.json after logo batch fetch.", "AssetsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Successfully updated index.json after logo batch fetch.");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to update index.json after logo batch fetch.", "AssetsChannel");
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Failed to update index.json after logo batch fetch.");
}
}
}
+130 -31
View File
@@ -1,75 +1,94 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Entities;
using FinlyticAssets.Services;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Models;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticAssets.Util;
/// <summary>
/// Represents a managed MQTT client acting as a server-side RPC provider within the asset microservice.
/// It subscribes to request topics, processes incoming JSON payloads via the database and index service,
/// and publishes the requested asset entities or logo files back to the response topic.
/// Also implements <see cref="IHostedService"/> to manage its own lifecycle connections.
/// </summary>
public class AssetsMqttClient(
ILogger<AssetsMqttClient> logger,
IServiceScopeFactory scopeFactory,
IConfiguration configuration) : ManagedMqttClient(logger), IHostedService
public class AssetsMqttClient : ManagedMqttClient, IHostedService
{
private readonly ILogger<AssetsMqttClient> _logger;
private readonly IServiceScopeFactory _scopeFactory;
private readonly IConfiguration _configuration;
public AssetsMqttClient(
ILogger<AssetsMqttClient> logger,
IServiceScopeFactory scopeFactory,
IConfiguration configuration) : base(logger)
{
_logger = logger;
_scopeFactory = scopeFactory;
_configuration = configuration;
}
/// <summary>
/// Starts the MQTT client and connects to the configured broker.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task representing the asynchronous start operation.</returns>
public async Task StartAsync(CancellationToken cancellationToken)
{
var config = new MqttConfiguration()
{
Host = configuration["MQTT:Host"] ?? configuration["MQTT__Host"]!,
Port = Convert.ToInt32(configuration["MQTT:Port"] ?? configuration["MQTT__Port"]!),
ClientId = $"{(configuration["MQTT:ClientId"] ?? configuration["MQTT__ClientId"]!)}_{Guid.NewGuid()}"
Host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost",
Port = Convert.ToInt32(_configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883"),
ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticAssets")}_{Guid.NewGuid()}"
};
_logger.LogInformation("Starting Assets MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId);
await ConnectAsync(config);
}
/// <summary>
/// Gracefully stops and disconnects the MQTT client.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task representing the asynchronous stop operation.</returns>
public async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Stopping Assets MQTT client.");
await DisconnectAsync();
}
/// <summary>
/// Invoked automatically once the connection to the MQTT broker is successfully established or restored.
/// Registers subscriptions for asset validation, search, and logo download requests.
/// </summary>
/// <returns>A <see cref="Task"/> representing the asynchronous subscription operation.</returns>
protected override async Task OnConnectedAsync()
{
_logger.LogInformation("Assets MQTT Client connected. Subscribing to topics...");
await SubscribeAsync("services/request/assets_Get/#");
await SubscribeAsync("services/request/assets_Search/#");
await SubscribeAsync("services/request/assets_GetDiscovery/#");
await SubscribeAsync("services/request/assets_GetDerivatives/#");
await SubscribeAsync("services/request/assets_FetchLogo/#");
await SubscribeAsync("services/request/assets_settings_GetAll/#");
await SubscribeAsync("services/request/assets_settings_Update/#");
await SubscribeAsync("services/request/health_Ping/#");
await SubscribeAsync("services/config/updated/#");
FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
{
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticAssets", StringComparison.OrdinalIgnoreCase))
{
await PublishAsync("finlytic/logs/FinlyticAssets", logDto);
}
};
}
/// <summary>
/// Processes incoming messages on the subscribed topics, executes the corresponding service methods,
/// and publishes the result to the response topic while preserving the correlation ID.
/// Processes incoming messages on the subscribed topics.
/// </summary>
/// <param name="topic">The MQTT topic on which the message was received.</param>
/// <param name="payload">The incoming message as a UTF-8 encoded JSON string.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous message processing operation.</returns>
protected override async Task OnMessageReceivedAsync(string topic, string payload)
{
if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase))
@@ -90,9 +109,21 @@ public class AssetsMqttClient(
return;
}
if (topic.StartsWith("services/request/assets_settings_GetAll", StringComparison.OrdinalIgnoreCase))
{
await HandleSettingsGetAllAsync(correlationId);
return;
}
if (topic.StartsWith("services/request/assets_settings_Update", StringComparison.OrdinalIgnoreCase))
{
await HandleSettingsUpdateAsync(payload, correlationId);
return;
}
try
{
using var scope = scopeFactory.CreateScope();
using var scope = _scopeFactory.CreateScope();
var dbService = scope.ServiceProvider.GetRequiredService<IAssetsDbService>();
var indexService = scope.ServiceProvider.GetRequiredService<IAssetsIndexService>();
@@ -129,26 +160,90 @@ public class AssetsMqttClient(
}
}
private async Task HandleSettingsGetAllAsync(string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AssetsMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
try
{
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/assets_settings_GetAll/{correlationId}";
await PublishAsync(responseTopic, settings);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticAssets] [Settings_GetAll] Failed to retrieve settings.");
}
}
private async Task HandleSettingsUpdateAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AssetsMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
try
{
Dictionary<string, object?>? updates = null;
try
{
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
}
catch
{
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
if (list != null)
{
updates = new Dictionary<string, object?>();
foreach (var item in list) updates[item.Key] = item.Value;
}
}
if (updates != null && updates.Count > 0)
{
await settingsService.UpdateSettingsAsync(updates);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
}
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/assets_settings_Update/{correlationId}";
await PublishAsync(responseTopic, currentSettings);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticAssets] [Settings_Update] Failed to update settings.");
}
}
private async Task HandleConfigUpdatedAsync(string topic, string payload)
{
if (!topic.EndsWith("FinlyticAssets", StringComparison.OrdinalIgnoreCase))
return;
logger.LogInformation("[{Channel}] [AssetsMqttClient] Received config update event for FinlyticAssets.", "AssetsChannel");
try
{
var updatePayload = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.ServiceConfigUpdatePayload);
if (updatePayload?.Settings != null && updatePayload.Settings.Count > 0)
{
using var scope = scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsDb.UpdateSettingsFromDictionary(updatePayload.Settings);
logger.LogInformation("[{Channel}] [AssetsMqttClient] Persisted {Count} updated settings to FinlyticAssets database.", "AssetsChannel", updatePayload.Settings.Count);
using var scope = _scopeFactory.CreateScope();
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
var dict = updatePayload.Settings.ToDictionary(k => k.Key, v => (object?)v.Value);
await settings.UpdateSettingsAsync(dict);
}
}
catch (Exception ex)
{
logger.LogError(ex, "[{Channel}] [AssetsMqttClient] Error processing MQTT config update event.", "AssetsChannel");
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AssetsMqttClient>>();
await finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsMqttClient] Error processing MQTT config update event.");
}
}
@@ -162,7 +257,9 @@ public class AssetsMqttClient(
{
string respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic, new FinlyticCore.Dtos.ServiceHealthResponse("FinlyticAssets", "Online", DateTime.UtcNow, "Connected"));
logger.LogInformation("[{Channel}] [AssetsMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", "AssetsChannel", correlationId);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AssetsMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[AssetsMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
}
}
@@ -236,7 +333,9 @@ public class AssetsMqttClient(
}
catch (Exception ex)
{
logger.LogError(ex, "[{Channel}] Error parsing GetDerivativesRequest payload.", "AssetsChannel");
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<AssetsMqttClient>>();
await finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsMqttClient] Error parsing GetDerivativesRequest payload.");
}
return [];
+22
View File
@@ -0,0 +1,22 @@
using FinlyticCore.Models.Settings;
namespace FinlyticAssets.Util;
public static class SettingKeys
{
// --- Logging-Kanäle ---
public static readonly SettingKey<bool> AssetsChannel = new("Logging.Channel.Assets", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
// --- Asset Scanning ---
public static readonly SettingKey<bool> EnableAutoScan = new("Scanner.EnableAutoScan", true);
public static readonly SettingKey<int> ScanIntervalHours = new("Scanner.ScanIntervalHours", 12);
public static readonly SettingKey<int> MaxConcurrentScans = new("Scanner.MaxConcurrentScans", 5);
public static readonly SettingKey<bool> EnableDerivativeScanning = new("Scanner.EnableDerivativeScanning", true);
// --- Logos & Media ---
public static readonly SettingKey<bool> AutoFetchLogos = new("Media.AutoFetchLogos", true);
public static readonly SettingKey<int> LogoFetchBatchSize = new("Media.LogoFetchBatchSize", 25);
public static readonly SettingKey<string> LogoStorageDirectory = new("Media.LogoStorageDirectory", "data/logos");
}
@@ -6,6 +6,7 @@ using System.Text.Json.Serialization;
using System.Threading.Tasks;
using FinlyticBackend.Util;
using FinlyticCore.Dtos;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Util;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Cors;
@@ -58,38 +59,18 @@ public class AdminSettingsController : ControllerBase
private readonly WebMqttClient _mqttClient;
private readonly ILogger<AdminSettingsController> _logger;
// In-memory static store for default UI configuration templates
private static readonly ConcurrentDictionary<string, List<ServiceConfigItem>> _inMemorySettings = new(StringComparer.OrdinalIgnoreCase);
static AdminSettingsController()
private static readonly Dictionary<string, string> ServiceRpcPrefixes = new(StringComparer.OrdinalIgnoreCase)
{
// Default-Templates für Microservice-Konfigurationen initialisieren
_inMemorySettings["FinlyticAnalyzer"] = new List<ServiceConfigItem>
{
new("MinSignalScore", "75.0", "double", "Mindest-Score für KI-Trade-Proposals (0-100)", DateTime.UtcNow),
new("VixPanicThreshold", "30.0", "double", "VIX-Wert ab dem Panik-Modus aktiviert wird", DateTime.UtcNow),
new("ProposalTtlMinutes", "180", "int", "Gültigkeitsdauer von Trade-Proposals in Minuten", DateTime.UtcNow)
};
["FinlyticFundamentals"] = "fundamentals",
["FinlyticNews"] = "news",
["FinlyticTechnicalAnalysis"] = "ta",
["FinlyticSentiment"] = "sentiment",
["FinlyticAnalyzer"] = "analyzer",
["FinlyticTrades"] = "trades",
["FinlyticAssets"] = "assets"
};
_inMemorySettings["FinlyticNews"] = new List<ServiceConfigItem>
{
new("ScrapeIntervalMinutes", "15", "int", "Intervall für das Scraping neuer Nachrichten", DateTime.UtcNow),
new("FinBertBatchSize", "8", "int", "Batch-Größe für die Sentiment-Analyse", DateTime.UtcNow)
};
_inMemorySettings["FinlyticTechnicalAnalysis"] = new List<ServiceConfigItem>
{
new("EmaShortPeriod", "20", "int", "Kurze Periode für EMA-Berechnungen", DateTime.UtcNow),
new("EmaLongPeriod", "50", "int", "Lange Periode für EMA-Berechnungen", DateTime.UtcNow),
new("RsiPeriod", "14", "int", "Standard-Periode für RSI-Berechnung", DateTime.UtcNow)
};
_inMemorySettings["FinlyticTrades"] = new List<ServiceConfigItem>
{
new("ExportFeedbackIntervalHours", "6", "int", "Intervall für den Parquet/JSON Feedback-Export", DateTime.UtcNow),
new("DefaultLeverageLimit", "10", "decimal", "Standardmäßiger Maximalhebel für Derivate", DateTime.UtcNow)
};
}
private static readonly ConcurrentDictionary<string, List<ServiceConfigItem>> _inMemorySettings = new(StringComparer.OrdinalIgnoreCase);
public AdminSettingsController(
WebMqttClient mqttClient,
@@ -100,11 +81,57 @@ public class AdminSettingsController : ControllerBase
}
/// <summary>
/// Retrieves all service configurations grouped by service name.
/// Retrieves recent buffered in-memory logs for a specific service.
/// </summary>
[HttpGet("logs/{serviceName}")]
public IActionResult GetServiceLogs(string serviceName)
{
if (BackendMqttBridge.ServiceLogsRingBuffer.TryGetValue(serviceName, out var queue))
{
return Ok(queue.ToList());
}
return Ok(new List<FinlyticCore.Dtos.Logging.LogMessageDto>());
}
/// <summary>
/// Retrieves all service configurations grouped by service name via live MQTT RPC queries.
/// </summary>
[HttpGet]
public IActionResult GetAllSettings()
public async Task<IActionResult> GetAllSettings()
{
if (_mqttClient.IsConnected)
{
var fetchTasks = ServiceRpcPrefixes.Select(async kvp =>
{
var serviceName = kvp.Key;
var prefix = kvp.Value;
try
{
var liveSettings = await _mqttClient.SendRpcRequestAsync<List<DynamicSettingDto>, string>(
$"{prefix}_settings_GetAll",
"",
TimeSpan.FromSeconds(2));
if (liveSettings != null && liveSettings.Count > 0)
{
_inMemorySettings[serviceName] = liveSettings.Select(d => new ServiceConfigItem(
Key: d.Key,
Value: d.Value?.ToString() ?? "",
DataType: d.Type,
Description: d.Description,
UpdatedAt: d.UpdatedAt ?? DateTime.UtcNow
)).ToList();
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[AdminSettings] Could not fetch live settings from {ServiceName} via MQTT.", serviceName);
}
});
await Task.WhenAll(fetchTasks);
}
var grouped = _inMemorySettings.ToDictionary(
g => g.Key,
g => g.Value.Select(item => new ServiceConfigItemResponseDto(
@@ -198,11 +225,37 @@ public class AdminSettingsController : ControllerBase
}
/// <summary>
/// Retrieves settings for a specific service.
/// Retrieves settings for a specific service via live MQTT RPC.
/// </summary>
[HttpGet("{serviceName}")]
public IActionResult GetServiceSettings(string serviceName)
public async Task<IActionResult> GetServiceSettings(string serviceName)
{
if (ServiceRpcPrefixes.TryGetValue(serviceName, out var prefix) && _mqttClient.IsConnected)
{
try
{
var liveSettings = await _mqttClient.SendRpcRequestAsync<List<DynamicSettingDto>, string>(
$"{prefix}_settings_GetAll",
"",
TimeSpan.FromSeconds(2));
if (liveSettings != null && liveSettings.Count > 0)
{
_inMemorySettings[serviceName] = liveSettings.Select(d => new ServiceConfigItem(
Key: d.Key,
Value: d.Value?.ToString() ?? "",
DataType: d.Type,
Description: d.Description,
UpdatedAt: d.UpdatedAt ?? DateTime.UtcNow
)).ToList();
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[AdminSettings] Could not fetch live settings for {ServiceName} via MQTT.", serviceName);
}
}
if (_inMemorySettings.TryGetValue(serviceName, out var list))
{
var dtos = list.Select(item => new ServiceConfigItemResponseDto(
@@ -220,11 +273,10 @@ public class AdminSettingsController : ControllerBase
}
/// <summary>
/// Broadcasts configuration settings to the targeted microservice via MQTT.
/// Does NOT write to Backend database. The microservice persists updated settings directly into its own database.
/// Broadcasts configuration settings to the targeted microservice via MQTT RPC and updates in-memory cache.
/// </summary>
[HttpPut("{serviceName}")]
public async Task<IActionResult> UpdateServiceSettings(string serviceName, [FromBody] Dictionary<string, string> updatedValues)
public async Task<IActionResult> UpdateServiceSettings(string serviceName, [FromBody] Dictionary<string, object?> updatedValues)
{
if (updatedValues == null || !updatedValues.Any())
{
@@ -233,40 +285,32 @@ public class AdminSettingsController : ControllerBase
_logger.LogInformation("[AdminSettings] Transmitting {Count} config settings to microservice '{ServiceName}' via MQTT", updatedValues.Count, serviceName);
// In-Memory Template-Store aktualisieren
if (_inMemorySettings.TryGetValue(serviceName, out var existingList))
{
foreach (var (key, value) in updatedValues)
{
var idx = existingList.FindIndex(item => item.Key.Equals(key, StringComparison.OrdinalIgnoreCase));
if (idx >= 0)
{
var old = existingList[idx];
existingList[idx] = old with { Value = value, UpdatedAt = DateTime.UtcNow };
}
else
{
existingList.Add(new ServiceConfigItem(key, value, "string", $"Setting for {serviceName}", DateTime.UtcNow));
}
}
}
else
{
var newList = updatedValues.Select(kv => new ServiceConfigItem(kv.Key, kv.Value, "string", $"Setting for {serviceName}", DateTime.UtcNow)).ToList();
_inMemorySettings[serviceName] = newList;
}
// MQTT Config-Update Event senden
bool mqttPublished = false;
try
{
if (_mqttClient.IsConnected)
if (_mqttClient.IsConnected && ServiceRpcPrefixes.TryGetValue(serviceName, out var prefix))
{
var updated = await _mqttClient.SendRpcRequestAsync<List<DynamicSettingDto>, Dictionary<string, object?>>(
$"{prefix}_settings_Update",
updatedValues,
TimeSpan.FromSeconds(3));
if (updated != null && updated.Count > 0)
{
_inMemorySettings[serviceName] = updated.Select(d => new ServiceConfigItem(
Key: d.Key,
Value: d.Value?.ToString() ?? "",
DataType: d.Type,
Description: d.Description,
UpdatedAt: d.UpdatedAt ?? DateTime.UtcNow
)).ToList();
}
string topic = $"services/config/updated/{serviceName}";
var payload = new ServiceConfigUpdatePayload(
ServiceName: serviceName,
Timestamp: DateTime.UtcNow,
Settings: updatedValues
Settings: updatedValues.ToDictionary(kv => kv.Key, kv => kv.Value?.ToString() ?? "")
);
await _mqttClient.PublishAsync(topic, payload);
+49
View File
@@ -0,0 +1,49 @@
using System;
using System.Threading.Tasks;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging;
namespace FinlyticBackend.Hubs;
/// <summary>
/// SignalR Hub that streams live log messages from microservices to connected web UI clients.
/// </summary>
public class LogStreamHub : Hub
{
private readonly ILogger<LogStreamHub> _logger;
public LogStreamHub(ILogger<LogStreamHub> logger)
{
_logger = logger;
}
public async Task JoinServiceLogs(string serviceName)
{
if (!string.IsNullOrWhiteSpace(serviceName))
{
await Groups.AddToGroupAsync(Context.ConnectionId, serviceName);
_logger.LogInformation("[LogStreamHub] Client {ConnectionId} joined log stream for {ServiceName}", Context.ConnectionId, serviceName);
}
}
public async Task LeaveServiceLogs(string serviceName)
{
if (!string.IsNullOrWhiteSpace(serviceName))
{
await Groups.RemoveFromGroupAsync(Context.ConnectionId, serviceName);
_logger.LogInformation("[LogStreamHub] Client {ConnectionId} left log stream for {ServiceName}", Context.ConnectionId, serviceName);
}
}
public override async Task OnConnectedAsync()
{
_logger.LogInformation("[LogStreamHub] Client connected: {ConnectionId}", Context.ConnectionId);
await base.OnConnectedAsync();
}
public override async Task OnDisconnectedAsync(Exception? exception)
{
_logger.LogInformation("[LogStreamHub] Client disconnected: {ConnectionId}", Context.ConnectionId);
await base.OnDisconnectedAsync(exception);
}
}
+4
View File
@@ -191,6 +191,10 @@ app.MapHub<FavoritesPriceHub>("/hubs/favorites-prices", options =>
{
options.Transports = HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents;
});
app.MapHub<LogStreamHub>("/hubs/logs", options =>
{
options.Transports = HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents;
});
app.MapGet("/health", () => Results.Ok(new { status = "Healthy", service = "FinlyticBackend", timestamp = DateTime.UtcNow }));
+29 -1
View File
@@ -7,6 +7,7 @@ using System.Threading.Tasks;
using FinlyticBackend.Database;
using FinlyticBackend.Hubs;
using FinlyticBackend.Services;
using FinlyticCore.Dtos.Logging;
using FinlyticCore.Dtos.News;
using FinlyticCore.Models;
using FinlyticCore.Models.Auth;
@@ -28,12 +29,14 @@ public class BackendMqttBridge : ManagedMqttClient, IHostedService
{
public static readonly ConcurrentDictionary<string, JsonElement> FundamentalsCache = new(StringComparer.OrdinalIgnoreCase);
public static readonly ConcurrentDictionary<string, JsonElement> TechnicalsCache = new(StringComparer.OrdinalIgnoreCase);
public static readonly ConcurrentDictionary<string, ConcurrentQueue<LogMessageDto>> ServiceLogsRingBuffer = new(StringComparer.OrdinalIgnoreCase);
private readonly IConfiguration _configuration;
private readonly IServiceScopeFactory _scopeFactory;
private readonly IHubContext<TradeRealtimeHub, ITradeClient> _hubContext;
private readonly IHubContext<TradeHub> _tradeHubContext;
private readonly IHubContext<NewsHub> _newsHubContext;
private readonly IHubContext<LogStreamHub> _logHubContext;
private readonly IFirebaseNotificationService _firebaseService;
private readonly ILogger<BackendMqttBridge> _logger;
@@ -43,6 +46,7 @@ public class BackendMqttBridge : ManagedMqttClient, IHostedService
IHubContext<TradeRealtimeHub, ITradeClient> hubContext,
IHubContext<TradeHub> tradeHubContext,
IHubContext<NewsHub> newsHubContext,
IHubContext<LogStreamHub> logHubContext,
IFirebaseNotificationService firebaseService,
ILogger<BackendMqttBridge> logger) : base(logger)
{
@@ -51,6 +55,7 @@ public class BackendMqttBridge : ManagedMqttClient, IHostedService
_hubContext = hubContext;
_tradeHubContext = tradeHubContext;
_newsHubContext = newsHubContext;
_logHubContext = logHubContext;
_firebaseService = firebaseService;
_logger = logger;
}
@@ -95,6 +100,8 @@ public class BackendMqttBridge : ManagedMqttClient, IHostedService
await SubscribeAsync("finlytic/technicalanalysis/#");
await SubscribeAsync("finlytic/ta/#");
// Real-time Logs
await SubscribeAsync("finlytic/logs/#");
}
/// <inheritdoc />
@@ -104,7 +111,11 @@ public class BackendMqttBridge : ManagedMqttClient, IHostedService
try
{
if (topic.StartsWith("finlytic/trades/proposed/", StringComparison.OrdinalIgnoreCase) ||
if (topic.StartsWith("finlytic/logs/", StringComparison.OrdinalIgnoreCase))
{
await HandleLogMessageAsync(payloadStr);
}
else if (topic.StartsWith("finlytic/trades/proposed/", StringComparison.OrdinalIgnoreCase) ||
topic.StartsWith("finlytic/trades/update", StringComparison.OrdinalIgnoreCase))
{
await HandleTradeProposalAsync(payloadStr);
@@ -136,6 +147,23 @@ public class BackendMqttBridge : ManagedMqttClient, IHostedService
}
}
private async Task HandleLogMessageAsync(string payloadStr)
{
var logDto = JsonSerializer.Deserialize<LogMessageDto>(payloadStr, FinlyticJsonSerializerContext.Default.LogMessageDto);
if (logDto == null) return;
string serviceKey = logDto.ServiceName;
var queue = ServiceLogsRingBuffer.GetOrAdd(serviceKey, _ => new ConcurrentQueue<LogMessageDto>());
queue.Enqueue(logDto);
// Keep buffer capped at 250 entries
while (queue.Count > 250 && queue.TryDequeue(out _)) { }
// Broadcast to SignalR clients
await _logHubContext.Clients.Group(serviceKey).SendAsync("ReceiveLogMessage", logDto);
await _logHubContext.Clients.All.SendAsync("ReceiveLogMessage", logDto);
}
private async Task HandleTradeProposalAsync(string payloadStr)
{
var proposal = JsonSerializer.Deserialize<TradeProposalDto>(payloadStr);
+18 -1
View File
@@ -61,10 +61,27 @@ public class WebMqttClient : ManagedMqttClient, IHostedService
await SubscribeAsync("services/response/assets_GetDiscovery/#");
await SubscribeAsync("services/response/assets_GetDerivatives/#");
await SubscribeAsync("services/response/trades_Get/#");
await SubscribeAsync("services/response/trades_Close/#");
await SubscribeAsync("services/response/trades_Reject/#");
await SubscribeAsync("services/response/trades_Accept/#");
await SubscribeAsync("services/response/analyzer_TriggerManual/#");
await SubscribeAsync("services/response/health_Ping/#");
// Settings RPC response channels for all microservices
await SubscribeAsync("services/response/fundamentals_settings_GetAll/#");
await SubscribeAsync("services/response/fundamentals_settings_Update/#");
await SubscribeAsync("services/response/news_settings_GetAll/#");
await SubscribeAsync("services/response/news_settings_Update/#");
await SubscribeAsync("services/response/ta_settings_GetAll/#");
await SubscribeAsync("services/response/ta_settings_Update/#");
await SubscribeAsync("services/response/sentiment_settings_GetAll/#");
await SubscribeAsync("services/response/sentiment_settings_Update/#");
await SubscribeAsync("services/response/analyzer_settings_GetAll/#");
await SubscribeAsync("services/response/analyzer_settings_Update/#");
await SubscribeAsync("services/response/trades_settings_GetAll/#");
await SubscribeAsync("services/response/trades_settings_Update/#");
await SubscribeAsync("services/response/assets_settings_GetAll/#");
await SubscribeAsync("services/response/assets_settings_Update/#");
}
protected override Task OnMessageReceivedAsync(string topic, string payload)
+524 -214
View File
@@ -7,112 +7,24 @@ using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Yahoo;
using Microsoft.Extensions.Logging;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using Microsoft.Playwright;
namespace FinlyticCore.Services.Yahoo;
/// <summary>
/// Thread-sicherer Client für den Zugriff auf die internen Yahoo Finance APIs.
/// Verwaltet automatisch den erforderlichen Cookie- (A3) und Crumb-Token-Authentifizierungs-Flow.
/// </summary>
public interface IYahooFinanceClient
{
/// <summary>
/// Stellt sicher, dass die aktuelle Session über ein gültiges Cookie und einen Crumb-Token verfügt.
/// </summary>
/// <param name="forceRefresh">Erzwingt das Erneuern des Authentifizierungs-Tokens, selbst wenn die Frist noch nicht abgelaufen ist.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Crumb-Token oder <c>null</c>, wenn die Authentifizierung fehlgeschlagen ist.</returns>
Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>
/// Sucht nach Tickern, Namen, ISINs oder Firmen über die Yahoo Finance Such-API.
/// erfordert keine Cookie/Crumb-Authentifizierung.
/// </summary>
/// <param name="query">Der Suchbegriff (z. B. "Apple", "US0378331005", "AAPL").</param>
/// <param name="quotesCount">Die maximale Anzahl an Treffern für Wertpapiere/Aktien.</param>
/// <param name="newsCount">Die maximale Anzahl an News-Treffern.</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Suchergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooSearchResponseDto?> SearchAsync(
string query,
int quotesCount = 10,
int newsCount = 0,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft Fundamentaldaten und Unternehmens-Metadaten für ein bestimmtes Symbol über den quoteSummary-Endpunkt ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL", "MSFT").</param>
/// <param name="modules">Die abzufragenden Yahoo-Module (z. B. "assetProfile", "financialData").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Die Abfrageergebnisse als DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
IEnumerable<string> modules,
CancellationToken cancellationToken = default);
/// <summary>
/// Hilfsmethode zum Abrufen aller vordefinierten Standard-Module für ein Tickersymbol.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das vollständige QuoteSummary-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteSummaryResponseDto?> GetFullQuoteSummaryAsync(
string symbol,
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft historische Chart- und Kursdaten (OHLCV) für ein Symbol ab.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "AAPL").</param>
/// <param name="range">Der Abfragezeitraum (z. B. "1d", "1m", "1y", "5y").</param>
/// <param name="interval">Das Intervall der Datenpunkte (z. B. "1m", "5m", "1d", "1wk").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Chart-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
string range = "1y",
string interval = "1d",
CancellationToken cancellationToken = default);
/// <summary>
/// Ruft schnelle Realtime-Preise für eine Liste von Tickersymbolen ab.
/// </summary>
/// <param name="symbols">Eine Liste von Tickersymbolen (z. B. <c>["AAPL", "MSFT", "^GSPC"]</c>).</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Das Quote-Ergebnis-DTO oder <c>null</c> bei Fehlern.</returns>
Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
CancellationToken cancellationToken = default);
/// <summary>
/// Bequeme Hilfsmethode, um den aktuellen regulären Marktpreis für ein einzelnes Tickersymbol abzufragen.
/// </summary>
/// <param name="symbol">Das Tickersymbol (z. B. "^VIX", "AAPL").</param>
/// <param name="cancellationToken">Ein Token zum Abbrechen der asynchronen Operation.</param>
/// <returns>Der aktuelle Preis als <see cref="decimal"/> oder <c>null</c>, wenn kein Preis ermittelt werden konnte.</returns>
Task<decimal?> GetLivePriceAsync(
string symbol,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Managed thread-safe HTTP client for Yahoo Finance APIs.
/// Implements the two-step Cookie (A3) & Crumb token authentication flow.
/// </summary>
public class YahooFinanceClient
{
private const string DefaultUserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
private readonly HttpClient _httpClient;
private readonly CookieContainer _cookieContainer;
private readonly ILogger<YahooFinanceClient>? _logger;
private readonly IFinlyticLogger<YahooFinanceClient>? _finlyticLogger;
private readonly ISettingsService? _settingsService;
private readonly IPlaywrightExecutionService? _playwrightService;
private readonly SemaphoreSlim _authLock = new(1, 1);
private string? _crumb;
private string? _rawCookieHeader;
private DateTime _lastAuthTime = DateTime.MinValue;
/// <summary>
@@ -133,9 +45,15 @@ public class YahooFinanceClient
"calendarEvents"
};
public YahooFinanceClient(ILogger<YahooFinanceClient>? logger = null, HttpClient? httpClient = null)
public YahooFinanceClient(
IFinlyticLogger<YahooFinanceClient>? finlyticLogger = null,
ISettingsService? settingsService = null,
IPlaywrightExecutionService? playwrightService = null,
HttpClient? httpClient = null)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_settingsService = settingsService;
_playwrightService = playwrightService;
_cookieContainer = new CookieContainer();
if (httpClient != null)
@@ -147,71 +65,108 @@ public class YahooFinanceClient
var handler = new HttpClientHandler
{
CookieContainer = _cookieContainer,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
UseCookies = true,
AllowAutoRedirect = true
};
_httpClient = new HttpClient(handler);
}
if (!_httpClient.DefaultRequestHeaders.Contains("User-Agent"))
{
_httpClient.DefaultRequestHeaders.Add("User-Agent", DefaultUserAgent);
_httpClient = new HttpClient(handler);
_httpClient.DefaultRequestHeaders.Add("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
}
}
/// <summary>
/// Executes the Cookie (A3) &amp; Crumb token authentication flow.
/// 1. GET https://fc.yahoo.com (sets session A3 cookie)
/// 2. GET https://query1.finance.yahoo.com/v1/test/getcrumb (returns crumb string)
/// Ensures that an active Yahoo session (Cookie + dynamic Crumb token) is initialized.
/// Uses persistent DB caching and only refreshes when the crumb is invalid or forceRefresh is true.
/// </summary>
public async Task<string?> EnsureAuthenticatedAsync(bool forceRefresh = false,
CancellationToken cancellationToken = default)
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) && (DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
{
return _crumb;
}
await _authLock.WaitAsync(cancellationToken);
try
{
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb) &&
(DateTime.UtcNow - _lastAuthTime).TotalHours < 12)
// 1. Check in-memory crumb
if (!forceRefresh && !string.IsNullOrWhiteSpace(_crumb))
{
return _crumb;
}
_logger?.LogInformation("[YahooFinanceClient] Authenticating session (Cookie + Crumb)...");
// 1. Send GET request to fc.yahoo.com to obtain session cookie A3
using (var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com"))
// 2. Check persistent DB cache via SettingsService
if (!forceRefresh && _settingsService != null)
{
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
// CookieContainer automatically intercepts and stores 'A3' cookie
}
// 2. Send GET request to getcrumb to obtain the dynamic crumb token
using (var crumbRequest =
new HttpRequestMessage(HttpMethod.Get, "https://query1.finance.yahoo.com/v1/test/getcrumb"))
{
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
if (!crumbResponse.IsSuccessStatusCode)
try
{
_logger?.LogWarning("[YahooFinanceClient] Failed to fetch crumb token. Status: {Status}",
crumbResponse.StatusCode);
return null;
var cachedCrumb = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCrumb, cancellationToken);
var cachedCookies = await _settingsService.GetSettingAsync(CoreSettingKeys.YahooAuthCookie, cancellationToken);
if (!string.IsNullOrWhiteSpace(cachedCrumb) && !string.IsNullOrWhiteSpace(cachedCookies))
{
RestoreCookies(cachedCookies);
_crumb = cachedCrumb;
_rawCookieHeader = cachedCookies;
_lastAuthTime = DateTime.UtcNow;
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Restored cached Yahoo session & crumb from database ({Crumb}).", _crumb);
return _crumb;
}
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Error restoring cached Yahoo session from DB.");
}
}
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
_crumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authenticating fresh session with Yahoo (Cookie + Crumb)...");
// 3. Send GET request to fc.yahoo.com to obtain session cookie A3
try
{
using var initRequest = new HttpRequestMessage(HttpMethod.Get, "https://fc.yahoo.com");
initRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
initRequest.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8");
using var initResponse = await _httpClient.SendAsync(initRequest, cancellationToken);
}
catch { }
// 4. Send GET request to getcrumb to obtain dynamic crumb token
var crumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
if (!string.IsNullOrWhiteSpace(crumb))
{
_crumb = crumb;
_lastAuthTime = DateTime.UtcNow;
_logger?.LogInformation("[YahooFinanceClient] Acquired Crumb token successfully: {Crumb}", _crumb);
await PersistSessionAsync(_crumb, cancellationToken);
return _crumb;
}
// 5. FALLBACK: Playwright Browser Authentication (Bypasses EU Consent Wall and 429)
if (_playwrightService != null)
{
var (browserCrumb, browserCookies) = await AuthenticateViaPlaywrightAsync(cancellationToken);
if (!string.IsNullOrWhiteSpace(browserCrumb))
{
_crumb = browserCrumb;
_lastAuthTime = DateTime.UtcNow;
if (!string.IsNullOrWhiteSpace(browserCookies))
{
RestoreCookies(browserCookies);
_rawCookieHeader = browserCookies;
}
await PersistSessionAsync(_crumb, cancellationToken);
return _crumb;
}
}
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Failed to fetch crumb token from all endpoints.");
return null;
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
if (_finlyticLogger != null)
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Cookie & Crumb authentication.");
return null;
}
finally
@@ -220,10 +175,259 @@ public class YahooFinanceClient
}
}
private async Task<string?> FetchCrumbWithHttpClientAsync(CancellationToken cancellationToken)
{
string[] crumbUrls = new[]
{
"https://query1.finance.yahoo.com/v1/test/getcrumb",
"https://query2.finance.yahoo.com/v1/test/getcrumb"
};
foreach (var url in crumbUrls)
{
try
{
using var crumbRequest = new HttpRequestMessage(HttpMethod.Get, url);
crumbRequest.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
crumbRequest.Headers.Add("Accept", "*/*");
crumbRequest.Headers.Add("Origin", "https://finance.yahoo.com");
crumbRequest.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
crumbRequest.Headers.Add("Cookie", _rawCookieHeader);
}
using var crumbResponse = await _httpClient.SendAsync(crumbRequest, cancellationToken);
var crumbText = await crumbResponse.Content.ReadAsStringAsync(cancellationToken);
if (crumbResponse.IsSuccessStatusCode && !string.IsNullOrWhiteSpace(crumbText))
{
var cleanCrumb = crumbText.Trim('"', ' ', '\t', '\r', '\n');
if (!cleanCrumb.Contains("<html", StringComparison.OrdinalIgnoreCase) && cleanCrumb.Length < 100)
{
return cleanCrumb;
}
}
else
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] Endpoint '{Url}' returned status {Status}. Response Body: {Body}",
url, crumbResponse.StatusCode, crumbText);
}
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during FetchCrumb on '{Url}'", url);
}
}
return null;
}
private async Task<(string? crumb, string? cookieStr)> AuthenticateViaPlaywrightAsync(CancellationToken cancellationToken)
{
if (_playwrightService == null) return (null, null);
try
{
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Launching Playwright browser to acquire valid EU Yahoo session and Crumb...");
return await _playwrightService.ExecuteInContextAsync(async context =>
{
var page = await context.NewPageAsync();
try
{
await page.GotoAsync("https://finance.yahoo.com/quote/AAPL/", new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30_000
});
// Handle EU Consent if redirected
var url = page.Url;
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
{
var selectors = new[]
{
"button[name='agree']",
"button[value='agree']",
"button.accept-all",
"button.btn.primary",
"button.btn.secondary.accept-all",
"form[action*='consent'] button[type='submit']",
"button:has-text('Alle akzeptieren')",
"button:has-text('Accept all')",
"button:has-text('Akzeptieren')",
"button:has-text('Agree')"
};
foreach (var sel in selectors)
{
var btn = page.Locator(sel);
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
{
await btn.First.ClickAsync();
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15_000 });
break;
}
}
}
// 1. Extract cookies from BrowserContext
var cookies = await context.CookiesAsync();
var pairs = new List<string>();
foreach (var c in cookies)
{
if (c.Domain.Contains("yahoo.com", StringComparison.OrdinalIgnoreCase))
{
pairs.Add($"{c.Name}={c.Value}");
}
}
var extractedCookies = string.Join(";", pairs);
// 2. Fetch Crumb from inside the authenticated page
string? extractedCrumb = null;
try
{
extractedCrumb = await page.EvaluateAsync<string>(@"async () => {
try {
const res = await fetch('/v1/test/getcrumb');
if (res.ok) {
return await res.text();
}
} catch {}
return null;
}");
}
catch { }
// 3. If in-page fetch was empty, use the extracted cookies with HttpClient
if (string.IsNullOrWhiteSpace(extractedCrumb) && !string.IsNullOrWhiteSpace(extractedCookies))
{
RestoreCookies(extractedCookies);
_rawCookieHeader = extractedCookies;
extractedCrumb = await FetchCrumbWithHttpClientAsync(cancellationToken);
}
if (!string.IsNullOrWhiteSpace(extractedCrumb))
{
extractedCrumb = extractedCrumb.Trim('"', ' ', '\t', '\r', '\n');
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Playwright successfully acquired Yahoo Crumb token: {Crumb}", extractedCrumb);
}
return (extractedCrumb, extractedCookies);
}
finally
{
await page.CloseAsync();
}
}, cancellationToken: cancellationToken);
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Playwright-based authentication failed.");
return (null, null);
}
}
private async Task PersistSessionAsync(string crumb, CancellationToken cancellationToken)
{
if (_settingsService == null || string.IsNullOrWhiteSpace(crumb)) return;
try
{
var serializedCookies = SerializeCookies();
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCrumb, crumb, cancellationToken);
if (!string.IsNullOrWhiteSpace(serializedCookies))
{
_rawCookieHeader = serializedCookies;
await _settingsService.SetSettingAsync(CoreSettingKeys.YahooAuthCookie, serializedCookies, cancellationToken);
}
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Persisted valid Yahoo session & crumb to database.");
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Failed to persist Yahoo session to database.");
}
}
private string SerializeCookies()
{
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
return _rawCookieHeader;
}
try
{
var cookies = _cookieContainer.GetAllCookies();
var pairs = new List<string>();
foreach (System.Net.Cookie cookie in cookies)
{
pairs.Add($"{cookie.Name}={cookie.Value}");
}
return string.Join(";", pairs);
}
catch
{
return string.Empty;
}
}
private void RestoreCookies(string serializedCookies)
{
if (string.IsNullOrWhiteSpace(serializedCookies)) return;
_rawCookieHeader = serializedCookies;
try
{
var parts = serializedCookies.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var uris = new[]
{
new Uri("https://yahoo.com"),
new Uri("https://finance.yahoo.com"),
new Uri("https://query1.finance.yahoo.com"),
new Uri("https://query2.finance.yahoo.com"),
new Uri("https://fc.yahoo.com")
};
foreach (var part in parts)
{
var eqIdx = part.IndexOf('=');
if (eqIdx > 0 && eqIdx < part.Length - 1)
{
var name = part.Substring(0, eqIdx).Trim();
var val = part.Substring(eqIdx + 1).Trim();
foreach (var uri in uris)
{
try
{
_cookieContainer.Add(uri, new System.Net.Cookie(name, val));
}
catch { }
}
}
}
}
catch
{
// Ignore cookie restore errors
}
}
/// <summary>
/// Searches for tickers, names, ISINs, or companies via the Yahoo Finance search API.
/// URL: https://query2.finance.yahoo.com/v1/finance/search?q={query}&amp;quotesCount={quotesCount}&amp;newsCount={newsCount}
/// Note: Does not require Cookie/Crumb authentication.
/// </summary>
public async Task<YahooSearchResponseDto?> SearchAsync(
string query,
@@ -233,32 +437,55 @@ public class YahooFinanceClient
{
if (string.IsNullOrWhiteSpace(query)) return null;
try
string[] endpoints = new[]
{
var url =
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
$"https://query2.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}",
$"https://query1.finance.yahoo.com/v1/finance/search?q={Uri.EscapeDataString(query)}&quotesCount={quotesCount}&newsCount={newsCount}"
};
if (!response.IsSuccessStatusCode)
foreach (var url in endpoints)
{
try
{
_logger?.LogWarning("[YahooFinanceClient] Search for '{Query}' failed with status {Status}", query,
response.StatusCode);
return null;
}
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
return JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
}
catch (Exception ex)
{
_logger?.LogError(ex, "[YahooFinanceClient] Exception during Search for query '{Query}'", query);
return null;
using var response = await _httpClient.SendAsync(request, cancellationToken);
var json = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooSearchResponseDto>(json, GetJsonOptions());
if (dto != null)
{
return dto;
}
}
else
{
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] Search endpoint '{Url}' for '{Query}' returned status {Status}. Response Body: {Body}",
url, query, response.StatusCode, json);
}
}
catch (Exception ex)
{
if (_finlyticLogger != null)
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.YahooClientChannel, ex, "[YahooFinanceClient] Exception during Search endpoint '{Url}'", url);
}
}
return null;
}
/// <summary>
/// Retrieves fundamentals and company metadata using the quoteSummary endpoint.
/// URL: https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}?crumb={crumb}&amp;modules={modules}
/// </summary>
public async Task<YahooQuoteSummaryResponseDto?> GetQuoteSummaryAsync(
string symbol,
@@ -270,22 +497,49 @@ public class YahooFinanceClient
var moduleList = string.Join(",", modules);
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query2.finance.yahoo.com/v10/finance/quoteSummary/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
string[] baseUrls = new[]
{
_logger?.LogWarning("[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}",
symbol, response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
"https://query2.finance.yahoo.com/v10/finance/quoteSummary",
"https://query1.finance.yahoo.com/v10/finance/quoteSummary"
};
foreach (var baseUrl in baseUrls)
{
var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?crumb={Uri.EscapeDataString(crumb)}&modules={Uri.EscapeDataString(moduleList)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Origin", "https://finance.yahoo.com");
request.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(responseContent, GetJsonOptions());
return (false, dto);
}
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] GetQuoteSummary for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}",
symbol, response.StatusCode, url, responseContent);
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.TooManyRequests)
{
return (true, null);
}
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooQuoteSummaryResponseDto>(json, GetJsonOptions());
return (false, dto);
return (false, null);
}, cancellationToken);
}
@@ -300,7 +554,6 @@ public class YahooFinanceClient
/// <summary>
/// Retrieves historical OHLCV chart data for a given symbol.
/// URL: https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?range={range}&amp;interval={interval}&amp;crumb={crumb}
/// </summary>
public async Task<YahooChartResponseDto?> GetChartAsync(
string symbol,
@@ -312,28 +565,54 @@ public class YahooFinanceClient
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query1.finance.yahoo.com/v8/finance/chart/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
string[] baseUrls = new[]
{
_logger?.LogWarning("[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}", symbol,
response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
"https://query1.finance.yahoo.com/v8/finance/chart",
"https://query2.finance.yahoo.com/v8/finance/chart"
};
foreach (var baseUrl in baseUrls)
{
var url = $"{baseUrl}/{Uri.EscapeDataString(symbol)}?range={Uri.EscapeDataString(range)}&interval={Uri.EscapeDataString(interval)}&crumb={Uri.EscapeDataString(crumb)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Origin", "https://finance.yahoo.com");
request.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(responseContent, GetJsonOptions());
return (false, dto);
}
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] GetChart for '{Symbol}' failed with status {Status}. URL: {Url}. Response Body: {Body}",
symbol, response.StatusCode, url, responseContent);
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.TooManyRequests)
{
return (true, null);
}
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooChartResponseDto>(json, GetJsonOptions());
return (false, dto);
return (false, null);
}, cancellationToken);
}
/// <summary>
/// Retrieves quick real-time price quotes for one or more symbols.
/// URL: https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}&amp;crumb={crumb}
/// </summary>
public async Task<YahooQuoteResponseDto?> GetQuotesAsync(
IEnumerable<string> symbols,
@@ -345,21 +624,49 @@ public class YahooFinanceClient
var symbolsParam = string.Join(",", symbolList);
return await ExecuteWithRetryAsync(async (crumb) =>
{
var url =
$"https://query1.finance.yahoo.com/v7/finance/quote?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
using var response = await _httpClient.GetAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode)
string[] baseUrls = new[]
{
_logger?.LogWarning("[YahooFinanceClient] GetQuotes failed with status {Status}", response.StatusCode);
return (
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden, null);
"https://query1.finance.yahoo.com/v7/finance/quote",
"https://query2.finance.yahoo.com/v7/finance/quote"
};
foreach (var baseUrl in baseUrls)
{
var url = $"{baseUrl}?symbols={Uri.EscapeDataString(symbolsParam)}&crumb={Uri.EscapeDataString(crumb)}";
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Origin", "https://finance.yahoo.com");
request.Headers.Add("Referer", "https://finance.yahoo.com/");
if (!string.IsNullOrWhiteSpace(_rawCookieHeader))
{
request.Headers.Add("Cookie", _rawCookieHeader);
}
using var response = await _httpClient.SendAsync(request, cancellationToken);
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(responseContent, GetJsonOptions());
return (false, dto);
}
if (_finlyticLogger != null)
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
"[YahooFinanceClient] GetQuotes failed with status {Status}. URL: {Url}. Response Body: {Body}",
response.StatusCode, url, responseContent);
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.TooManyRequests)
{
return (true, null);
}
}
var json = await response.Content.ReadAsStringAsync(cancellationToken);
var dto = JsonSerializer.Deserialize<YahooQuoteResponseDto>(json, GetJsonOptions());
return (false, dto);
return (false, null);
}, cancellationToken);
}
@@ -386,26 +693,29 @@ public class YahooFinanceClient
CancellationToken cancellationToken) where T : class
{
var crumb = await EnsureAuthenticatedAsync(false, cancellationToken);
if (!string.IsNullOrEmpty(crumb))
{
var (isAuthError, result) = await action(crumb);
if (!isAuthError && result != null)
{
return result;
}
if (!isAuthError)
{
return result;
}
}
// Re-authenticate when auth error (401/403/429) or empty crumb occurs
if (_finlyticLogger != null)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel, "[YahooFinanceClient] Authentication error or invalid crumb encountered (401/403/429). Force re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (isAuthError, result) = await action(crumb);
if (!isAuthError && result != null)
{
return result;
}
if (isAuthError)
{
_logger?.LogInformation(
"[YahooFinanceClient] Authentication error encountered (401/403). Re-authenticating...");
crumb = await EnsureAuthenticatedAsync(true, cancellationToken);
if (string.IsNullOrEmpty(crumb)) return null;
var (_, retryResult) = await action(crumb);
return retryResult;
}
return result;
var (_, retryResult) = await action(crumb);
return retryResult;
}
private static JsonSerializerOptions GetJsonOptions()
@@ -413,7 +723,7 @@ public class YahooFinanceClient
return new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
}
}
+377 -260
View File
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
@@ -9,7 +10,6 @@ using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Playwright;
namespace FinlyticCore.Clients;
@@ -18,81 +18,113 @@ public interface IYahooFinanceHtmlClient
{
Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
string isinOrSymbol,
bool includeProfile = true,
CancellationToken cancellationToken = default);
}
public interface IYahooFinanceHtmlClient<TDbContext> : IYahooFinanceHtmlClient
where TDbContext : DbContext
{
}
public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHtmlClient<TDbContext>
where TDbContext : DbContext
public class YahooFinanceHtmlClient : IYahooFinanceHtmlClient
{
private readonly IPlaywrightExecutionService _playwrightService;
private readonly IFinlyticLogger<TContextClass, TDbContext> _finlyticLogger;
private readonly string _serviceName;
private readonly IFinlyticLogger<YahooFinanceHtmlClient> _finlyticLogger;
private const string _serviceName = "YahooFinanceHtmlClient";
public YahooFinanceHtmlClient(
IPlaywrightExecutionService playwrightService,
IFinlyticLogger<TContextClass, TDbContext> finlyticLogger)
IFinlyticLogger<YahooFinanceHtmlClient> finlyticLogger)
{
_playwrightService = playwrightService;
_finlyticLogger = finlyticLogger;
_serviceName = typeof(TContextClass).Name;
}
public async Task<YahooQuoteSummaryModulesDto?> ScrapeQuoteSummaryModulesAsync(
string isinOrSymbol,
bool includeProfile = true,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(isinOrSymbol)) return null;
var symbol = isinOrSymbol.Trim().ToUpperInvariant();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [YahooFinanceHtmlClient] Starting parallel Playwright HTML scrape for symbol '{symbol}'...");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Starting parallel fast Playwright HTML DOM scrape for symbol '{symbol}' (IncludeProfile: {includeProfile})...");
return await _playwrightService.ExecuteInContextAsync(async context =>
{
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var analysisData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
ProfileExtractionResult? profileResult = null;
var summaryUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/";
var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/key-statistics/";
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
var financialsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/financials/";
var analysisUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/analysis/";
var profileUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(symbol)}/profile/";
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
var profileTask = ScrapeProfilePageAsync(context, profileUrl, cancellationToken);
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
var analysisTask = ScrapePagePairsAsync(context, analysisUrl, cancellationToken);
// 1. Initial Page: Authenticate session and pass Cookie Consent once for the entire context
var initialPage = await context.NewPageAsync();
var summaryData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
try
{
await Task.WhenAll(statsTask, profileTask, financialsTask, analysisTask);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [1/2] Loading Summary & passing Consent: {summaryUrl}");
await initialPage.GotoAsync(summaryUrl, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 15_000
});
keyStatsData = await statsTask;
profileResult = await profileTask;
financialsData = await financialsTask;
analysisData = await analysisTask;
await HandleConsentAsync(initialPage);
await WaitForContentAsync(initialPage);
summaryData = await ExtractKeyValuePairsFromPageAsync(initialPage, summaryUrl);
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-fatal error during parallel page scraping for {symbol}.");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping initial Summary page for {symbol}.");
}
finally
{
await initialPage.CloseAsync();
}
// 2. Parallel Sub-Pages (Stats, Financials, and conditionally Profile)
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] [2/2] Fetching Sub-Pages in parallel (IncludeProfile: {includeProfile})...");
var statsTask = ScrapePagePairsAsync(context, statsUrl, cancellationToken);
var financialsTask = ScrapePagePairsAsync(context, financialsUrl, cancellationToken);
Task<ProfileExtractionResult?> profileTask = includeProfile
? ScrapeProfilePageAsync(context, profileUrl, cancellationToken)!
: Task.FromResult<ProfileExtractionResult?>(null);
var keyStatsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var financialsData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
ProfileExtractionResult? profileResult = null;
try
{
await Task.WhenAll(statsTask, financialsTask, profileTask);
keyStatsData = await statsTask;
financialsData = await financialsTask;
profileResult = await profileTask;
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error during parallel sub-page scrape for {symbol}.");
}
// Merge summary data into key stats
foreach (var (k, v) in summaryData)
{
if (!keyStatsData.ContainsKey(k))
{
keyStatsData[k] = v;
}
}
var profileDict = profileResult?.ProfileDict ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var officers = profileResult?.Officers ?? new List<YahooCompanyOfficerDto>();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel, $"[{_serviceName}] Scrape complete for '{symbol}'. Officers: {officers.Count}, Stats Keys: {keyStatsData.Count}");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Fast HTML DOM scrape complete for '{symbol}'. Summary: {summaryData.Count}, Stats: {keyStatsData.Count}, Financials: {financialsData.Count}, Profile: {profileDict.Count}, Officers: {officers.Count}");
return BuildModulesDto(
keyStatsData,
profileDict,
financialsData,
analysisData,
new Dictionary<string, string>(),
officers,
profileResult?.Sector,
profileResult?.Industry,
@@ -115,47 +147,15 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 20_000
Timeout = 10_000
});
await HandleConsentAsync(page);
var extracted = await page.EvaluateAsync<Dictionary<string, string>>(@"() => {
const results = {};
const cleanKey = (str) => {
return str.toLowerCase()
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
.replace(/\s*\d+\s*$/, '')
.replace(/\s+/g, ' ')
.trim();
};
document.querySelectorAll('table tr').forEach(tr => {
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
if (cells.length >= 2 && cells[0] && cells[1]) {
const key = cleanKey(cells[0]);
const val = cells[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-') {
results[key] = val;
}
}
});
return results;
}");
if (extracted != null)
{
foreach (var (k, v) in extracted)
{
targetDict[k] = v;
}
}
await WaitForContentAsync(page);
targetDict = await ExtractKeyValuePairsFromPageAsync(page, url);
}
catch (Exception ex)
{
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping URL {url}");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping URL {url}");
}
finally
{
@@ -165,7 +165,7 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
return targetDict;
}
private async Task<ProfileExtractionResult> ScrapeProfilePageAsync(
private async Task<ProfileExtractionResult?> ScrapeProfilePageAsync(
IBrowserContext context,
string url,
CancellationToken cancellationToken)
@@ -184,15 +184,15 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 20_000
Timeout = 10_000
});
await HandleConsentAsync(page);
await WaitForContentAsync(page);
var metaInfo = await page.EvaluateAsync<ProfileMetaJsResult>(@"() => {
var jsonStr = await page.EvaluateAsync<string>(@"() => {
let sector = null, industry = null, employees = null, description = null;
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary');
const descEl = document.querySelector('section[data-testid=""description""] p, div[data-testid=""description""] p, p.business-summary, section[data-testid=""asset-profile""] p');
if (descEl) description = descEl.innerText.trim();
const profileSec = document.querySelector('section[data-testid=""asset-profile""], div.asset-profile-container, main');
@@ -227,40 +227,51 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
}
});
return { sector, industry, employees, description, officers };
return JSON.stringify({ sector, industry, employees, description, officers });
}");
if (metaInfo != null)
if (!string.IsNullOrWhiteSpace(jsonStr))
{
sector = metaInfo.Sector;
industry = metaInfo.Industry;
fullTimeEmployees = metaInfo.Employees;
description = metaInfo.Description;
var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
var metaInfo = JsonSerializer.Deserialize<ProfileMetaJsResult>(jsonStr, options);
if (metaInfo.Officers != null)
if (metaInfo != null)
{
foreach (var off in metaInfo.Officers)
sector = metaInfo.Sector;
industry = metaInfo.Industry;
fullTimeEmployees = metaInfo.Employees;
description = metaInfo.Description;
if (metaInfo.Officers != null)
{
if (!string.IsNullOrWhiteSpace(off.Name))
foreach (var off in metaInfo.Officers)
{
companyOfficers.Add(new YahooCompanyOfficerDto(
Name: off.Name,
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
Title: off.Title,
YearBorn: off.YearBorn,
FiscalYear: null,
TotalPay: ParseYahooValue(off.Pay),
ExercisedValue: ParseYahooValue(off.Exercised),
UnexercisedValue: null
));
if (!string.IsNullOrWhiteSpace(off.Name))
{
companyOfficers.Add(new YahooCompanyOfficerDto(
Name: off.Name,
Age: off.YearBorn.HasValue ? (DateTime.UtcNow.Year - off.YearBorn.Value) : null,
Title: off.Title,
YearBorn: off.YearBorn,
FiscalYear: null,
TotalPay: ParseYahooValue(off.Pay),
ExercisedValue: ParseYahooValue(off.Exercised),
UnexercisedValue: null
));
}
}
}
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] HTML Profile extracted -> Sector: '{sector}', Industry: '{industry}', Employees: {fullTimeEmployees}, Officers: {companyOfficers.Count}, Desc length: {description?.Length ?? 0}");
}
}
profileDict = await ExtractKeyValuePairsFromPageAsync(page, url);
}
catch (Exception ex)
{
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Non-critical error scraping Profile page {url}");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error scraping Profile data from {url}");
}
finally
{
@@ -270,21 +281,151 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
return new ProfileExtractionResult(profileDict, companyOfficers, sector, industry, fullTimeEmployees, description);
}
private static async Task HandleConsentAsync(IPage page)
private static async Task WaitForContentAsync(IPage page)
{
try
{
if (page.Url.Contains("consent.yahoo.com"))
await page.WaitForSelectorAsync("table tr, ul li, div[data-testid], section, main", new PageWaitForSelectorOptions
{
var consentBtn = page.Locator("button[name='agree'], button[value='agree'], button.accept-all, form[action*='consent'] button");
if (await consentBtn.CountAsync() > 0)
State = WaitForSelectorState.Attached,
Timeout = 2_500
});
}
catch { }
}
private async Task<Dictionary<string, string>> ExtractKeyValuePairsFromPageAsync(IPage page, string pageUrl)
{
var targetDict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
try
{
var jsonStr = await page.EvaluateAsync<string>(@"() => {
const results = {};
const cleanKey = (str) => {
return str.toLowerCase()
.replace(/\(ttm\)|\(mrq\)|\(fye\)/g, '')
.replace(/\s*\d+\s*$/, '')
.replace(/\s+/g, ' ')
.trim();
};
// 1. Standard HTML Table Rows (Financials, Balance Sheet, Key Stats)
document.querySelectorAll('table tr').forEach(tr => {
const cells = Array.from(tr.querySelectorAll('td, th')).map(c => c.innerText.trim());
if (cells.length >= 2 && cells[0] && cells[1]) {
const key = cleanKey(cells[0]);
const val = cells[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
results[key] = val;
}
}
});
// 2. Modern Yahoo Finance List / Div pairs (Summary Table, Quote Statistics, Flex containers)
document.querySelectorAll('li, div[class*=""container""], div[class*=""row""], section div, div[data-testid]').forEach(el => {
const children = Array.from(el.querySelectorAll(':scope > span, :scope > div, :scope > p')).map(s => s.innerText.trim()).filter(Boolean);
if (children.length === 2) {
const key = cleanKey(children[0]);
const val = children[1].replace(/\s+/g, ' ').trim();
if (key && val && val !== 'N/A' && val !== '--' && val !== '-' && key.length < 70 && val.length < 90) {
if (!results[key]) {
results[key] = val;
}
}
}
});
return JSON.stringify(results);
}");
if (!string.IsNullOrWhiteSpace(jsonStr))
{
var parsed = JsonSerializer.Deserialize<Dictionary<string, string>>(jsonStr);
if (parsed != null)
{
await consentBtn.First.ClickAsync();
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 10_000 });
foreach (var (k, v) in parsed)
{
targetDict[k] = v;
}
}
}
var sampleKeys = targetDict.Keys.Take(6).ToList();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Extracted {targetDict.Count} items from HTML of '{pageUrl}'. Sample keys: [{string.Join(", ", sampleKeys)}]");
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex, $"[{_serviceName}] Error extracting key-value pairs from HTML of '{pageUrl}'");
}
return targetDict;
}
private async Task HandleConsentAsync(IPage page)
{
try
{
var url = page.Url;
if (url.Contains("consent.yahoo.com", StringComparison.OrdinalIgnoreCase) ||
url.Contains("guce.yahoo.com", StringComparison.OrdinalIgnoreCase))
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Detected EU Cookie Consent redirect: '{url}'. Searching for Accept/Reject buttons...");
var selectors = new[]
{
"button[name='agree']",
"button[value='agree']",
"button.accept-all",
"button.btn.primary",
"button.btn.secondary.accept-all",
"button[name='reject']",
"button[value='reject']",
"button.reject-all",
"form[action*='consent'] button[type='submit']",
"button:has-text('Alle akzeptieren')",
"button:has-text('Accept all')",
"button:has-text('Alle ablehnen')",
"button:has-text('Reject all')",
"button:has-text('Akzeptieren')",
"button:has-text('Ablehnen')",
"button:has-text('Agree')",
"button:has-text('I agree')"
};
foreach (var sel in selectors)
{
var btn = page.Locator(sel);
if (await btn.CountAsync() > 0 && await btn.First.IsVisibleAsync())
{
var btnText = await btn.First.InnerTextAsync();
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Found consent button '{btnText.Trim()}' with selector '{sel}'. Clicking...");
// Fast click and wait for DOMContentLoaded on finance.yahoo.com
await btn.First.ClickAsync();
try
{
await page.WaitForURLAsync(u => u.Contains("finance.yahoo.com", StringComparison.OrdinalIgnoreCase),
new PageWaitForURLOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 5_000 });
}
catch { }
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.HtmlScrapperChannel,
$"[{_serviceName}] Successfully passed consent wall. Current URL: '{page.Url}'");
break;
}
}
}
}
catch { /* Fallback */ }
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.HtmlScrapperChannel, ex,
$"[{_serviceName}] Exception in HandleConsentAsync.");
}
}
private YahooQuoteSummaryModulesDto BuildModulesDto(
@@ -305,10 +446,8 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
var assetProfile = new YahooAssetProfileDto(
Address1: null, Address2: null, City: null, State: null, Zip: null, Country: null, Phone: null, Website: null,
Industry: industry ?? GetString(allStats, "industry"),
IndustryKey: null, IndustryDisp: null,
Sector: sector ?? GetString(allStats, "sector"),
SectorKey: null, SectorDisp: null,
Industry: industry, IndustryKey: null, IndustryDisp: null,
Sector: sector, SectorKey: null, SectorDisp: null,
LongBusinessSummary: description,
FullTimeEmployees: fullTimeEmployees,
CompanyOfficers: companyOfficers.Count > 0 ? companyOfficers : null,
@@ -316,100 +455,104 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
GovernanceEpochDate: null, CompensationAsOfEpochDate: null
);
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
PriceToBook: GetVal(allStats, "price/book", "price / book"),
EnterpriseValue: GetVal(allStats, "enterprise value"),
ForwardPE: GetVal(allStats, "forward p/e"),
ProfitMargins: GetVal(allStats, "profit margin"),
FloatShares: GetVal(allStats, "float"),
SharesOutstanding: GetVal(allStats, "shares outstanding"),
SharesShort: GetVal(allStats, "shares short"),
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
SharesShortPreviousMonthDate: null, DateShortInterest: null,
SharesPercentSharesOut: GetVal(allStats, "% of shares outstanding"),
HeldPercentInsiders: GetVal(allStats, "% held by insiders"),
HeldPercentInstitutions: GetVal(allStats, "% held by institutions"),
ShortRatio: GetVal(allStats, "short ratio"),
ShortPercentOfFloat: GetVal(allStats, "short % of float"),
Beta: GetVal(allStats, "beta (5y monthly)", "beta"),
Category: null,
BookValue: GetVal(allStats, "book value per share", "book value"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price / sales"),
LastFiscalYearEnd: GetVal(allStats, "last fiscal year end"),
NextFiscalYearEnd: GetVal(allStats, "next fiscal year end"),
MostRecentQuarter: GetVal(allStats, "most recent quarter"),
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth"),
NetIncomeToCommon: GetVal(allStats, "net income avi to common"),
TrailingEps: GetVal(allStats, "diluted eps"),
ForwardEps: GetVal(allStats, "forward eps"),
PegRatio: GetVal(allStats, "peg ratio", "peg ratio (5yr expected)"),
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue"),
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda"),
FiftyTwoWeekChange: GetVal(allStats, "52-week change"),
SandP52WeekChange: GetVal(allStats, "s&p500 52-week change")
var financialData = new YahooFinancialDataDto(
CurrentPrice: GetVal(allStats, "previous close", "current price", "price", "regular market price"),
TargetHighPrice: GetVal(allStats, "target high price", "target high"),
TargetLowPrice: GetVal(allStats, "target low price", "target low"),
TargetMeanPrice: GetVal(allStats, "1y target est", "target mean price", "target est"),
TargetMedianPrice: GetVal(allStats, "target median price"),
RecommendationMean: GetVal(allStats, "recommendation mean"),
RecommendationKey: allStats.GetValueOrDefault("recommendation") ?? allStats.GetValueOrDefault("recommendation key"),
NumberOfAnalystOpinions: GetVal(allStats, "number of analyst opinions", "analyst opinions"),
TotalCash: GetVal(allStats, "total cash", "total cash (mrq)"),
TotalCashPerShare: GetVal(allStats, "total cash per share", "total cash per share (mrq)"),
Ebitda: GetVal(allStats, "ebitda"),
TotalDebt: GetVal(allStats, "total debt", "total debt (mrq)"),
QuickRatio: GetVal(allStats, "quick ratio"),
CurrentRatio: GetVal(allStats, "current ratio", "current ratio (mrq)"),
TotalRevenue: GetVal(allStats, "total revenue", "revenue", "revenue (ttm)"),
DebtToEquity: GetVal(allStats, "total debt/equity", "total debt/equity (mrq)", "debt to equity"),
RevenuePerShare: GetVal(allStats, "revenue per share", "revenue per share (ttm)"),
ReturnOnAssets: GetVal(allStats, "return on assets", "return on assets (ttm)"),
ReturnOnEquity: GetVal(allStats, "return on equity", "return on equity (ttm)"),
GrossProfits: GetVal(allStats, "gross profit", "gross profit (ttm)", "gross profits"),
FreeCashflow: GetVal(allStats, "levered free cash flow", "levered free cash flow (ttm)", "free cash flow"),
OperatingCashflow: GetVal(allStats, "operating cash flow", "operating cash flow (ttm)"),
RevenueGrowth: GetVal(allStats, "quarterly revenue growth", "quarterly revenue growth (yoy)", "revenue growth"),
GrossMargins: GetVal(allStats, "gross margin", "gross margins"),
EbitdaMargins: GetVal(allStats, "ebitda margin", "ebitda margins"),
OperatingMargins: GetVal(allStats, "operating margin", "operating margin (ttm)", "operating margins"),
ProfitMargins: GetVal(allStats, "profit margin", "profit margins"),
FinancialCurrency: null
);
var financialData = new YahooFinancialDataDto(
CurrentPrice: GetVal(allStats, "current price", "price"),
TargetHighPrice: GetVal(allStats, "target high", "high target"),
TargetLowPrice: GetVal(allStats, "target low", "low target"),
TargetMeanPrice: GetVal(allStats, "target mean", "target est"),
TargetMedianPrice: GetVal(allStats, "target median"),
RecommendationMean: GetVal(allStats, "recommendation mean"),
RecommendationKey: GetString(allStats, "recommendation key"),
NumberOfAnalystOpinions: GetVal(allStats, "number of analysts"),
TotalCash: GetVal(allStats, "total cash"),
TotalCashPerShare: GetVal(allStats, "total cash per share"),
Ebitda: GetVal(allStats, "ebitda"),
TotalDebt: GetVal(allStats, "total debt"),
QuickRatio: GetVal(allStats, "quick ratio"),
CurrentRatio: GetVal(allStats, "current ratio"),
TotalRevenue: GetVal(allStats, "revenue", "total revenue"),
DebtToEquity: GetVal(allStats, "total debt/equity"),
RevenuePerShare: GetVal(allStats, "revenue per share"),
ReturnOnAssets: GetVal(allStats, "return on assets"),
ReturnOnEquity: GetVal(allStats, "return on equity"),
GrossProfits: GetVal(allStats, "gross profit"),
FreeCashflow: GetVal(allStats, "levered free cash flow"),
OperatingCashflow: GetVal(allStats, "operating cash flow"),
RevenueGrowth: GetVal(allStats, "quarterly revenue growth"),
GrossMargins: GetVal(allStats, "gross margin"),
EbitdaMargins: GetVal(allStats, "ebitda margin"),
OperatingMargins: GetVal(allStats, "operating margin"),
ProfitMargins: GetVal(allStats, "profit margin"),
FinancialCurrency: "USD"
var defaultKeyStatistics = new YahooDefaultKeyStatisticsDto(
PriceToBook: GetVal(allStats, "price/book", "price to book", "kbv", "kurs-buchwert-verhältnis"),
EnterpriseValue: GetVal(allStats, "enterprise value", "unternehmenswert"),
ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
ProfitMargins: GetVal(allStats, "profit margin", "gewinnmarge"),
FloatShares: GetVal(allStats, "float", "streubesitz"),
SharesOutstanding: GetVal(allStats, "shares outstanding", "ausstehende aktien"),
SharesShort: GetVal(allStats, "shares short", "leerverkaufte aktien"),
SharesShortPriorMonth: GetVal(allStats, "shares short (prior month)"),
SharesShortPreviousMonthDate: null,
DateShortInterest: null,
SharesPercentSharesOut: GetVal(allStats, "shares % of shares outstanding", "short % of shares outstanding"),
HeldPercentInsiders: GetVal(allStats, "% held by insiders", "insider anteil"),
HeldPercentInstitutions: GetVal(allStats, "% held by institutions", "institutioneller anteil"),
ShortRatio: GetVal(allStats, "short ratio"),
ShortPercentOfFloat: GetVal(allStats, "short % of float", "short percent of float"),
Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
Category: null,
BookValue: GetVal(allStats, "book value per share", "book value per share (mrq)", "book value", "buchwert"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "price to sales", "kuv"),
LastFiscalYearEnd: null,
NextFiscalYearEnd: null,
MostRecentQuarter: null,
EarningsQuarterlyGrowth: GetVal(allStats, "quarterly earnings growth", "quarterly earnings growth (yoy)", "earnings growth"),
NetIncomeToCommon: GetVal(allStats, "net income avi to common", "net income avi to common (ttm)", "net income avail. to common", "net income"),
TrailingEps: GetVal(allStats, "diluted eps", "diluted eps (ttm)", "trailing eps", "eps (ttm)", "gewinn je aktie"),
ForwardEps: GetVal(allStats, "forward eps"),
PegRatio: GetVal(allStats, "peg ratio (5 yr expected)", "peg ratio (5yr expected)", "peg ratio", "peg-verhältnis"),
EnterpriseToRevenue: GetVal(allStats, "enterprise value/revenue", "ev/revenue"),
EnterpriseToEbitda: GetVal(allStats, "enterprise value/ebitda", "ev/ebitda"),
FiftyTwoWeekChange: GetVal(allStats, "52 week change", "52-week change", "52-wochen-änderung"),
SandP52WeekChange: GetVal(allStats, "s&p 500 52-week change", "s&p500 52-week change", "s&p 500 52 week change")
);
var summaryDetail = new YahooSummaryDetailDto(
MaxAge: 86400, PriceHint: null,
PreviousClose: GetVal(allStats, "previous close"),
Open: GetVal(allStats, "open"),
DayLow: GetVal(allStats, "day low"),
DayHigh: GetVal(allStats, "day high"),
RegularMarketPreviousClose: GetVal(allStats, "previous close"),
RegularMarketOpen: GetVal(allStats, "open"),
RegularMarketDayLow: GetVal(allStats, "day low"),
RegularMarketDayHigh: GetVal(allStats, "day high"),
DividendRate: GetVal(allStats, "forward dividend & yield", "dividend rate"),
DividendYield: GetVal(allStats, "dividend yield", "forward annual dividend yield", "trailing annual dividend yield"),
ExDividendDate: GetVal(allStats, "ex-dividend date"),
PayoutRatio: GetVal(allStats, "payout ratio"),
FiveYearAvgDividendYield: GetVal(allStats, "5 year avg dividend yield"),
Beta: GetVal(allStats, "beta"),
TrailingPE: GetVal(allStats, "trailing p/e"),
ForwardPE: GetVal(allStats, "forward p/e"),
Volume: GetVal(allStats, "volume"),
RegularMarketVolume: GetVal(allStats, "volume"),
AverageVolume: GetVal(allStats, "avg. volume", "average volume"),
AverageVolume10days: GetVal(allStats, "avg. volume (10 day)"),
AverageDailyVolume10Day: GetVal(allStats, "avg. volume (10 day)"),
Bid: GetVal(allStats, "bid"), Ask: GetVal(allStats, "ask"),
BidSize: null, AskSize: null,
MarketCap: GetVal(allStats, "market cap (intraday)", "market cap"),
FiftyTwoWeekLow: GetVal(allStats, "52 week low"),
FiftyTwoWeekHigh: GetVal(allStats, "52 week high"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales"),
Currency: "USD"
MaxAge: null,
PriceHint: null,
PreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
Open: GetVal(allStats, "open", "eröffnung"),
DayLow: null,
DayHigh: null,
RegularMarketPreviousClose: GetVal(allStats, "previous close", "schlusskurs vortag"),
RegularMarketOpen: GetVal(allStats, "open", "eröffnung"),
RegularMarketDayLow: null,
RegularMarketDayHigh: null,
DividendRate: GetVal(allStats, "forward annual dividend rate", "trailing annual dividend rate", "forward dividend & yield", "dividend rate", "dividende"),
DividendYield: GetVal(allStats, "forward annual dividend yield", "trailing annual dividend yield", "dividend yield", "dividendenrendite"),
ExDividendDate: null,
PayoutRatio: GetVal(allStats, "payout ratio", "ausschüttungsquote"),
FiveYearAvgDividendYield: GetVal(allStats, "5 year average dividend yield"),
Beta: GetVal(allStats, "beta", "beta (5y monthly)"),
TrailingPE: GetVal(allStats, "pe ratio (ttm)", "trailing p/e", "p/e ratio", "pe", "trailing pe", "kgv (ttm)", "kgv"),
ForwardPE: GetVal(allStats, "forward p/e", "forward pe", "forward kgv", "kgv (roll.)"),
Volume: GetVal(allStats, "volume", "volumen"),
RegularMarketVolume: GetVal(allStats, "volume", "volumen"),
AverageVolume: GetVal(allStats, "avg vol (3 month)", "avg. volume", "average volume", "durchschnittsvolumen"),
AverageVolume10days: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
AverageDailyVolume10Day: GetVal(allStats, "avg vol (10 day)", "avg volume (10 day)"),
Bid: GetVal(allStats, "bid", "geld"),
Ask: GetVal(allStats, "ask", "brief"),
BidSize: null,
AskSize: null,
MarketCap: GetVal(allStats, "market cap", "market capitalization", "market cap (intraday)", "marktkapitalisierung"),
FiftyTwoWeekLow: GetVal(allStats, "52 week low", "52-week low", "52 wochen tief"),
FiftyTwoWeekHigh: GetVal(allStats, "52 week high", "52-week high", "52 wochen hoch"),
PriceToSalesTrailing12Months: GetVal(allStats, "price/sales", "kuv"),
Currency: null
);
return new YahooQuoteSummaryModulesDto(
@@ -428,66 +571,59 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
);
}
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] keys)
private static YahooValueDto? GetVal(Dictionary<string, string> dict, params string[] candidateKeys)
{
foreach (var k in keys)
foreach (var candidate in candidateKeys)
{
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
return ParseYahooValue(val);
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match.Value))
return ParseYahooValue(match.Value);
if (dict.TryGetValue(candidate, out var valStr) && !string.IsNullOrWhiteSpace(valStr))
{
var parsed = ParseYahooValue(valStr);
if (parsed != null) return parsed;
}
}
return null;
}
private static string? GetString(Dictionary<string, string> dict, params string[] keys)
private static YahooValueDto? ParseYahooValue(string? raw)
{
foreach (var k in keys)
{
if (dict.TryGetValue(k, out var val) && !string.IsNullOrWhiteSpace(val))
return val.Trim();
if (string.IsNullOrWhiteSpace(raw) || raw == "N/A" || raw == "--" || raw == "-") return null;
var match = dict.FirstOrDefault(kvp => kvp.Key.Equals(k, StringComparison.OrdinalIgnoreCase) || kvp.Key.StartsWith(k, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(match.Value))
return match.Value.Trim();
}
return null;
}
/// <summary>
/// Parsen von Suffixen (M, B, T, K) und Prozentwerten gemäß den funktionierenden Regex-Regeln.
/// </summary>
public static YahooValueDto? ParseYahooValue(string? text)
{
if (string.IsNullOrWhiteSpace(text) || text == "N/A" || text == "---" || text == "--" || text == "-")
return null;
var trimmed = text.Trim();
bool isPercent = trimmed.EndsWith("%");
var clean = raw.Trim().Replace(" ", "").Replace("$", "").Replace("€", "").Replace("£", "");
var isPercent = clean.EndsWith("%");
if (isPercent) clean = clean.TrimEnd('%');
double multiplier = 1.0;
if (trimmed.EndsWith("T", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000_000.0;
else if (trimmed.EndsWith("B", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000_000.0;
else if (trimmed.EndsWith("M", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000_000.0;
else if (trimmed.EndsWith("K", StringComparison.OrdinalIgnoreCase)) multiplier = 1_000.0;
if (clean.EndsWith("T", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000_000.0; clean = clean[..^1]; }
else if (clean.EndsWith("B", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000_000.0; clean = clean[..^1]; }
else if (clean.EndsWith("M", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000_000.0; clean = clean[..^1]; }
else if (clean.EndsWith("K", StringComparison.OrdinalIgnoreCase)) { multiplier = 1_000.0; clean = clean[..^1]; }
// Beseitigt Einheiten und Tausenderpunkte, isoliert die reine Zahl mit Dezimalpunkt
var numPart = Regex.Replace(trimmed, @"[^\d.-]", "");
if (double.TryParse(numPart, NumberStyles.Any, CultureInfo.InvariantCulture, out double parsedVal))
if (double.TryParse(clean, NumberStyles.Any, CultureInfo.InvariantCulture, out var num))
{
double finalVal = isPercent ? (parsedVal / 100.0) : (parsedVal * multiplier);
return new YahooValueDto
{
Raw = finalVal,
Fmt = trimmed,
LongFmt = finalVal.ToString("N0", CultureInfo.InvariantCulture)
};
var finalRaw = num * multiplier;
if (isPercent) finalRaw /= 100.0;
return new YahooValueDto { Raw = finalRaw, Fmt = raw };
}
return null;
return new YahooValueDto { Raw = null, Fmt = raw };
}
private class ProfileMetaJsResult
{
public string? Sector { get; set; }
public string? Industry { get; set; }
public int? Employees { get; set; }
public string? Description { get; set; }
public List<JsOfficer>? Officers { get; set; }
}
private class JsOfficer
{
public string? Name { get; set; }
public string? Title { get; set; }
public string? Pay { get; set; }
public string? Exercised { get; set; }
public int? YearBorn { get; set; }
}
private record ProfileExtractionResult(
@@ -496,24 +632,5 @@ public class YahooFinanceHtmlClient<TContextClass, TDbContext> : IYahooFinanceHt
string? Sector,
string? Industry,
int? Employees,
string? Description
);
private class ProfileMetaJsResult
{
public string? Sector { get; set; }
public string? Industry { get; set; }
public int? Employees { get; set; }
public string? Description { get; set; }
public List<OfficerJsResult>? Officers { get; set; }
}
private class OfficerJsResult
{
public string? Name { get; set; }
public string? Title { get; set; }
public string? Pay { get; set; }
public string? Exercised { get; set; }
public int? YearBorn { get; set; }
}
string? Description);
}
@@ -0,0 +1,15 @@
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Entities.Settings;
using Microsoft.EntityFrameworkCore;
namespace FinlyticCore.Database;
/// <summary>
/// Einheitliches Interface für DbContexts, die dynamische Einstellungen verwalten.
/// </summary>
public interface ISettingsDbContext
{
DbSet<SettingEntity> DynamicSettings { get; }
Task<int> SaveChangesAsync(CancellationToken cancellationToken = default);
}
@@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
namespace FinlyticCore.Dtos.Fundamentals;
@@ -18,7 +18,7 @@ public record AssetHeaderDto
public string Description { get; init; } = string.Empty;
[JsonPropertyName("primaryTicker")]
public TickerInfoDto PrimaryTicker { get; init; }
public TickerInfoDto PrimaryTicker { get; init; } = new();
[JsonPropertyName("availableTickers")]
public List<TickerInfoDto> AvailableTickers { get; init; } = [];
@@ -16,7 +16,7 @@ public record CorporateEventDto
public string? Isin { get; init; }
[JsonPropertyName("ticker")]
public TickerInfoDto Ticker { get; init; }
public TickerInfoDto Ticker { get; init; } = new();
[JsonPropertyName("companyName")]
public string? CompanyName { get; init; }
@@ -9,7 +9,7 @@ namespace FinlyticCore.Dtos.Fundamentals;
public record FundamentalDataDto
{
[JsonPropertyName("ticker")]
public TickerInfoDto Ticker { get; init; }
public TickerInfoDto Ticker { get; init; } = new();
// --- Valuation & Multiples ---
[JsonPropertyName("marketCap")]
@@ -0,0 +1,13 @@
using System;
using System.Text.Json.Serialization;
namespace FinlyticCore.Dtos.Logging;
public record LogMessageDto(
[property: JsonPropertyName("timestamp")] DateTime Timestamp,
[property: JsonPropertyName("serviceName")] string ServiceName,
[property: JsonPropertyName("channel")] string Channel,
[property: JsonPropertyName("level")] string Level,
[property: JsonPropertyName("message")] string Message,
[property: JsonPropertyName("exception")] string? Exception = null
);
@@ -0,0 +1,14 @@
using System;
namespace FinlyticCore.Dtos.Settings;
/// <summary>
/// Repräsentiert eine dynamische Einstellung für das Web-UI und MQTT-RPC.
/// </summary>
public record DynamicSettingDto(
string Key,
object? Value,
string Type,
string Description = "",
DateTime? UpdatedAt = null
);
+2
View File
@@ -9,6 +9,8 @@
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Abstractions" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.9" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.9" />
<PackageReference Include="Microsoft.Playwright" Version="1.49.0" />
<PackageReference Include="MQTTnet" Version="5.1.0.1559" />
@@ -5,16 +5,27 @@ namespace FinlyticCore.Models.Settings;
/// </summary>
public static class CoreSettingKeys
{
// --- Logging-Kanäle ---
// --- Globale Logging-Kanäle ---
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HtmlScrapperChannel = new("Logging.Channel.HtmlScrapper", true);
public static readonly SettingKey<bool> YahooClientChannel = new("Logging.Channel.YahooClient", true);
public static readonly SettingKey<bool> FundamentalsChannel = new("Logging.Channel.Fundamentals", true);
public static readonly SettingKey<bool> TradeRepublicChannel = new("Logging.Channel.TradeRepublic", true);
public static readonly SettingKey<bool> PlaywrightChannel = new("Logging.Channel.Playwright", true);
public static readonly SettingKey<bool> SettingsChannel = new("Logging.Channel.Settings", true);
// --- Scraper & Feature-Toggles ---
public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true);
public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true);
public static readonly SettingKey<int> ScraperTimeoutSeconds = new("Scraper.TimeoutSeconds", 30);
public static readonly SettingKey<int> ScraperMaxRetries = new("Scraper.MaxRetries", 2);
// --- Trade Republic WebSocket Config ---
public static readonly SettingKey<int> TradeRepublicWsReconnectIntervalSeconds = new("TradeRepublic.WsReconnectIntervalSeconds", 5);
public static readonly SettingKey<int> TradeRepublicWsTimeoutSeconds = new("TradeRepublic.WsTimeoutSeconds", 15);
// --- Yahoo Auth Persistence ---
public static readonly SettingKey<string> YahooAuthCrumb = new("Yahoo.Auth.Crumb", "");
public static readonly SettingKey<string> YahooAuthCookie = new("Yahoo.Auth.Cookie", "");
}
@@ -1,17 +1,42 @@
using System;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Logging;
using FinlyticCore.Models.Settings;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services;
/// <summary>
/// Bietet kanalbasierte, dynamisch steuerbare Logging-Funktionalitäten über den <see cref="ISettingsService{TContext}"/>.
/// Globaler Broadcaster für strukturierte Logs in Echtzeit.
/// </summary>
public static class FinlyticLogBroadcaster
{
public static Func<LogMessageDto, Task>? OnLogPublished { get; set; }
public static void Broadcast(LogMessageDto dto)
{
if (OnLogPublished != null)
{
_ = Task.Run(async () =>
{
try
{
await OnLogPublished(dto);
}
catch
{
// Ignore broadcast errors to never disrupt execution
}
});
}
}
}
/// <summary>
/// Bietet kanalbasierte, dynamisch steuerbare Logging-Funktionalitäten über den <see cref="ISettingsService"/>.
/// </summary>
/// <typeparam name="TContextClass">Die aufrufende Klasse (für Log-Kategorien).</typeparam>
/// <typeparam name="TDbContext">Der DbContext des Services für den Zugriff auf die Settings.</typeparam>
public interface IFinlyticLogger<TContextClass, TDbContext> where TDbContext : DbContext
public interface IFinlyticLogger<TContextClass>
{
// --- Debug ---
Task LogDebugAsync(SettingKey<bool> channelKey, string message, params object[] args);
@@ -36,22 +61,49 @@ public interface IFinlyticLogger<TContextClass, TDbContext> where TDbContext : D
/// <summary>
/// Kanalbasierte Logger-Implementierung, die Einstellungen und Stummschaltungen
/// in Echtzeit aus dem <see cref="ISettingsService{TContext}"/> bezieht.
/// in Echtzeit aus dem <see cref="ISettingsService"/> bezieht.
/// </summary>
public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContextClass, TDbContext>
where TDbContext : DbContext
public class FinlyticLogger<TContextClass> : IFinlyticLogger<TContextClass>
{
private static readonly string ServiceName = typeof(TContextClass).Assembly.GetName().Name ?? "Finlytic";
private readonly ILogger<TContextClass> _logger;
private readonly ISettingsService<TDbContext> _settingsService;
private readonly ISettingsService _settingsService;
public FinlyticLogger(
ILogger<TContextClass> logger,
ISettingsService<TDbContext> settingsService)
ISettingsService settingsService)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_settingsService = settingsService ?? throw new ArgumentNullException(nameof(settingsService));
}
private void DispatchBroadcast(SettingKey<bool> channelKey, LogLevel level, string message, Exception? exception, params object[] args)
{
try
{
string formattedMsg = args != null && args.Length > 0 ? string.Format(message, args) : message;
FinlyticLogBroadcaster.Broadcast(new LogMessageDto(
Timestamp: DateTime.UtcNow,
ServiceName: ServiceName,
Channel: channelKey.Name,
Level: level.ToString(),
Message: formattedMsg,
Exception: exception?.ToString()
));
}
catch
{
FinlyticLogBroadcaster.Broadcast(new LogMessageDto(
Timestamp: DateTime.UtcNow,
ServiceName: ServiceName,
Channel: channelKey.Name,
Level: level.ToString(),
Message: message,
Exception: exception?.ToString()
));
}
}
#region Debug
public async Task LogDebugAsync(SettingKey<bool> channelKey, string message, params object[] args)
@@ -59,6 +111,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Debug))
{
_logger.LogDebug(message, args);
DispatchBroadcast(channelKey, LogLevel.Debug, message, null, args);
}
}
@@ -70,6 +123,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogDebug(exception, message, args);
else
_logger.LogDebug(message, args);
DispatchBroadcast(channelKey, LogLevel.Debug, message, exception, args);
}
}
@@ -82,6 +136,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Information))
{
_logger.LogInformation(message, args);
DispatchBroadcast(channelKey, LogLevel.Information, message, null, args);
}
}
@@ -93,6 +148,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogInformation(exception, message, args);
else
_logger.LogInformation(message, args);
DispatchBroadcast(channelKey, LogLevel.Information, message, exception, args);
}
}
@@ -105,6 +161,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Warning))
{
_logger.LogWarning(message, args);
DispatchBroadcast(channelKey, LogLevel.Warning, message, null, args);
}
}
@@ -116,6 +173,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogWarning(exception, message, args);
else
_logger.LogWarning(message, args);
DispatchBroadcast(channelKey, LogLevel.Warning, message, exception, args);
}
}
@@ -128,6 +186,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Error))
{
_logger.LogError(message, args);
DispatchBroadcast(channelKey, LogLevel.Error, message, null, args);
}
}
@@ -139,6 +198,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogError(exception, message, args);
else
_logger.LogError(message, args);
DispatchBroadcast(channelKey, LogLevel.Error, message, exception, args);
}
}
@@ -151,6 +211,7 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
if (await ShouldLogAsync(channelKey, LogLevel.Trace))
{
_logger.LogTrace(message, args);
DispatchBroadcast(channelKey, LogLevel.Trace, message, null, args);
}
}
@@ -162,14 +223,12 @@ public class FinlyticLogger<TContextClass, TDbContext> : IFinlyticLogger<TContex
_logger.LogCritical(exception, message, args);
else
_logger.LogCritical(message, args);
DispatchBroadcast(channelKey, LogLevel.Critical, message, exception, args);
}
}
#endregion
/// <summary>
/// Prüft, ob ein spezifischer Kanal und die aufrufende Klasse aktives Logging erlauben.
/// </summary>
private async Task<bool> ShouldLogAsync(SettingKey<bool> channelKey, LogLevel level)
{
ArgumentNullException.ThrowIfNull(channelKey);
@@ -1,9 +1,14 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using Microsoft.Playwright;
namespace FinlyticCore.Services.PlaywrightScrapper;
public interface IPlaywrightBrowserFactory : IAsyncDisposable
public interface IPlaywrightBrowserFactory : IAsyncDisposable, IDisposable
{
/// <summary>
/// Stellt sicher, dass die IBrowser-Instanz verbunden ist.
@@ -18,15 +23,15 @@ public interface IPlaywrightBrowserFactory : IAsyncDisposable
public class PlaywrightBrowserFactory : IPlaywrightBrowserFactory
{
private readonly ILogger<PlaywrightBrowserFactory> _logger;
private readonly IFinlyticLogger<PlaywrightBrowserFactory> _finlyticLogger;
private readonly SemaphoreSlim _browserLock = new(1, 1);
private IPlaywright? _playwright;
private IBrowser? _browser;
public PlaywrightBrowserFactory(ILogger<PlaywrightBrowserFactory> logger)
public PlaywrightBrowserFactory(IFinlyticLogger<PlaywrightBrowserFactory> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
}
public async Task<IBrowser> GetBrowserAsync(CancellationToken cancellationToken = default)
@@ -51,7 +56,7 @@ public class PlaywrightBrowserFactory : IPlaywrightBrowserFactory
}
});
_logger.LogInformation("[PlaywrightFactory] Shared Chromium Instance successfully launched.");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.PlaywrightChannel, "[PlaywrightFactory] Shared Chromium Instance successfully launched.");
return _browser;
}
finally
@@ -79,12 +84,41 @@ public class PlaywrightBrowserFactory : IPlaywrightBrowserFactory
}
};
public void Dispose()
{
try
{
if (_browser != null)
{
_browser.CloseAsync().GetAwaiter().GetResult();
_browser.DisposeAsync().GetAwaiter().GetResult();
}
}
catch
{
// Ignore any sync disposal timeouts
}
finally
{
_playwright?.Dispose();
_browserLock.Dispose();
GC.SuppressFinalize(this);
}
}
public async ValueTask DisposeAsync()
{
if (_browser != null)
{
await _browser.CloseAsync();
await _browser.DisposeAsync();
try
{
await _browser.CloseAsync();
await _browser.DisposeAsync();
}
catch
{
// Ignore disposal errors
}
}
_playwright?.Dispose();
+395 -42
View File
@@ -1,50 +1,54 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Database;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Entities.Settings;
using FinlyticCore.Models.Settings;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services;
public interface ISettingsService<TContext> where TContext : DbContext
public interface ISettingsService
{
// --- 1. Typsicherer Zugriff über SettingKey<T> (Empfohlen) ---
Task<T> GetSettingAsync<T>(SettingKey<T> key,
CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(SettingKey<T> key, T value,
CancellationToken cancellationToken = default);
Task<T> GetSettingAsync<T>(SettingKey<T> key, CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(SettingKey<T> key, T value, CancellationToken cancellationToken = default);
// --- 2. Dynamischer Zugriff über Enum-Key ---
Task<T> GetSettingAsync<TEnum, T>(TEnum enumKey, T defaultValue = default!,
CancellationToken cancellationToken = default) where TEnum : struct, Enum;
Task SetSettingAsync<TEnum, T>(TEnum enumKey, T value,
CancellationToken cancellationToken = default) where TEnum : struct, Enum;
Task<T> GetSettingAsync<TEnum, T>(TEnum enumKey, T defaultValue = default!, CancellationToken cancellationToken = default) where TEnum : struct, Enum;
Task SetSettingAsync<TEnum, T>(TEnum enumKey, T value, CancellationToken cancellationToken = default) where TEnum : struct, Enum;
// --- 3. Dynamischer Zugriff über String-Key ---
Task<T> GetSettingAsync<T>(string key, T defaultValue = default!,
CancellationToken cancellationToken = default);
Task<T> GetSettingAsync<T>(string key, T defaultValue = default!, CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(string key, T value, CancellationToken cancellationToken = default);
Task SetSettingAsync<T>(string key, T value,
CancellationToken cancellationToken = default);
// --- 4. Reflection-Erkennung & Bulk-Verwaltung für Web UI / MQTT ---
Task<List<DynamicSettingDto>> GetAllRegisteredSettingsAsync(IEnumerable<Type>? customKeyHolders = null, CancellationToken cancellationToken = default);
Task UpdateSettingsAsync(Dictionary<string, object?> updatedSettings, CancellationToken cancellationToken = default);
}
public class SettingsService<TContext> : ISettingsService<TContext> where TContext : DbContext
public class SettingsService : ISettingsService
{
private readonly Microsoft.Extensions.DependencyInjection.IServiceScopeFactory _scopeFactory;
private readonly ILogger<SettingsService<TContext>>? _logger;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<SettingsService>? _logger;
// Fast In-Memory Cache: Key Schema: "KeyName"
private readonly ConcurrentDictionary<string, string> _cache = new();
// Fast In-Memory Cache: Key Schema: "KeyName" -> JSON string
private readonly ConcurrentDictionary<string, string> _cache = new(StringComparer.OrdinalIgnoreCase);
public SettingsService(Microsoft.Extensions.DependencyInjection.IServiceScopeFactory scopeFactory, ILogger<SettingsService<TContext>>? logger = null)
public SettingsService(
IServiceScopeFactory scopeFactory,
ILogger<SettingsService>? logger = null)
{
_scopeFactory = scopeFactory;
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
_logger = logger;
}
@@ -52,11 +56,13 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
public Task<T> GetSettingAsync<T>(SettingKey<T> key, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(key);
return GetSettingInternalAsync(key.Name, key.DefaultValue, cancellationToken);
}
public Task SetSettingAsync<T>(SettingKey<T> key, T value, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(key);
return SetSettingInternalAsync(key.Name, value, cancellationToken);
}
@@ -96,11 +102,138 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
#endregion
#region Core Engine Logik
#region Bulk & Reflection Discovery
public async Task<List<DynamicSettingDto>> GetAllRegisteredSettingsAsync(
IEnumerable<Type>? customKeyHolders = null,
CancellationToken cancellationToken = default)
{
var holderTypes = new List<Type> { typeof(CoreSettingKeys) };
if (customKeyHolders != null)
{
holderTypes.AddRange(customKeyHolders);
}
var resultList = new List<DynamicSettingDto>();
var seenKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
// 1. Reflection auf allen SettingKey<T> Feldern
foreach (var type in holderTypes.Distinct())
{
var fields = type.GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy);
foreach (var field in fields)
{
var fieldType = field.FieldType;
if (fieldType.IsGenericType && fieldType.GetGenericTypeDefinition() == typeof(SettingKey<>))
{
var valType = fieldType.GetGenericArguments()[0];
var settingKeyObj = field.GetValue(null);
if (settingKeyObj == null) continue;
var nameProp = fieldType.GetProperty("Name");
var defaultProp = fieldType.GetProperty("DefaultValue");
var keyName = nameProp?.GetValue(settingKeyObj)?.ToString() ?? field.Name;
if (seenKeys.Contains(keyName)) continue;
seenKeys.Add(keyName);
var defVal = defaultProp?.GetValue(settingKeyObj);
var typeName = MapToSimpleTypeName(valType);
// Aktuellen Wert aus DB / Cache lesen
var currentRawValue = await GetSettingInternalObjectAsync(keyName, valType, defVal, cancellationToken);
resultList.Add(new DynamicSettingDto(
Key: keyName,
Value: currentRawValue,
Type: typeName,
Description: FormatDescriptionFromKey(keyName),
UpdatedAt: DateTime.UtcNow
));
}
}
}
// 2. Prüfen, ob in der DB weitere gespeicherte Settings existieren, die nicht im Code deklariert sind
try
{
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext != null)
{
var dbSettings = await dbContext.DynamicSettings.AsNoTracking().ToListAsync(cancellationToken);
foreach (var dbSetting in dbSettings)
{
if (!seenKeys.Contains(dbSetting.Key))
{
seenKeys.Add(dbSetting.Key);
var (inferredVal, inferredType) = InferJsonValueAndType(dbSetting.ValueJson);
resultList.Add(new DynamicSettingDto(
Key: dbSetting.Key,
Value: inferredVal,
Type: inferredType,
Description: FormatDescriptionFromKey(dbSetting.Key),
UpdatedAt: dbSetting.LastUpdatedUtc
));
}
}
}
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "[SettingsService] Error reading database settings during GetAllRegisteredSettingsAsync.");
}
return resultList.OrderBy(s => s.Key).ToList();
}
public async Task UpdateSettingsAsync(Dictionary<string, object?> updatedSettings, CancellationToken cancellationToken = default)
{
if (updatedSettings == null || updatedSettings.Count == 0) return;
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
foreach (var (key, rawValue) in updatedSettings)
{
if (string.IsNullOrWhiteSpace(key)) continue;
string jsonValue = NormalizeJsonValue(rawValue);
_cache[key] = jsonValue;
if (dbContext != null)
{
var entity = await dbContext.DynamicSettings.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
if (entity == null)
{
entity = new SettingEntity
{
Key = key,
ValueJson = jsonValue,
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.DynamicSettings.Add(entity);
}
else
{
entity.ValueJson = jsonValue;
entity.LastUpdatedUtc = DateTime.UtcNow;
}
}
}
if (dbContext != null)
{
await dbContext.SaveChangesAsync(cancellationToken);
}
}
#endregion
#region Internal Engine Logic
private async Task<T> GetSettingInternalAsync<T>(string key, T defaultValue, CancellationToken cancellationToken)
{
// 1. Zuerst im In-Memory Cache prüfen
if (_cache.TryGetValue(key, out var cachedJson))
{
return DeserializeValue(cachedJson, defaultValue);
@@ -108,17 +241,21 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
try
{
using var scope = _scopeFactory.CreateScope();
var dbContext = Microsoft.Extensions.DependencyInjection.ServiceProviderServiceExtensions.GetRequiredService<TContext>(scope.ServiceProvider);
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext == null)
{
var defaultJson = NormalizeJsonValue(defaultValue);
_cache[key] = defaultJson;
return defaultValue;
}
// 2. Aus DB der spezifischen TContext-Instanz laden
var entity = await dbContext.Set<SettingEntity>()
var entity = await dbContext.DynamicSettings
.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
// 3. Falls noch nicht vorhanden: In DB anlegen (Seed on Demand)
if (entity == null)
{
var defaultJson = JsonSerializer.Serialize(defaultValue);
var defaultJson = NormalizeJsonValue(defaultValue);
entity = new SettingEntity
{
Key = key,
@@ -126,35 +263,76 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.Set<SettingEntity>().Add(entity);
dbContext.DynamicSettings.Add(entity);
await dbContext.SaveChangesAsync(cancellationToken);
_cache[key] = defaultJson;
return defaultValue;
}
// In Cache legen & Wert zurückgeben
_cache[key] = entity.ValueJson;
return DeserializeValue(entity.ValueJson, defaultValue);
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "Setting '{Key}' could not be loaded or initialized in DB. Using default value in memory.", key);
_cache[key] = JsonSerializer.Serialize(defaultValue);
_logger?.LogWarning(ex, "[SettingsService] Setting '{Key}' could not be loaded or initialized in DB. Using default value.", key);
_cache[key] = NormalizeJsonValue(defaultValue);
return defaultValue;
}
}
private async Task<object?> GetSettingInternalObjectAsync(string key, Type valueType, object? defaultValue, CancellationToken cancellationToken)
{
if (_cache.TryGetValue(key, out var cachedJson))
{
return DeserializeObject(cachedJson, valueType, defaultValue);
}
try
{
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext == null) return defaultValue;
var entity = await dbContext.DynamicSettings.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
if (entity == null)
{
var defaultJson = NormalizeJsonValue(defaultValue);
entity = new SettingEntity
{
Key = key,
ValueJson = defaultJson,
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.DynamicSettings.Add(entity);
await dbContext.SaveChangesAsync(cancellationToken);
_cache[key] = defaultJson;
return defaultValue;
}
_cache[key] = entity.ValueJson;
return DeserializeObject(entity.ValueJson, valueType, defaultValue);
}
catch
{
return defaultValue;
}
}
private async Task SetSettingInternalAsync<T>(string key, T value, CancellationToken cancellationToken)
{
var jsonValue = JsonSerializer.Serialize(value);
var jsonValue = NormalizeJsonValue(value);
_cache[key] = jsonValue;
try
{
using var scope = _scopeFactory.CreateScope();
var dbContext = Microsoft.Extensions.DependencyInjection.ServiceProviderServiceExtensions.GetRequiredService<TContext>(scope.ServiceProvider);
await using var scope = _scopeFactory.CreateAsyncScope();
var dbContext = scope.ServiceProvider.GetService<ISettingsDbContext>();
if (dbContext == null) return;
var entity = await dbContext.Set<SettingEntity>()
var entity = await dbContext.DynamicSettings
.FirstOrDefaultAsync(s => s.Key == key, cancellationToken);
if (entity == null)
@@ -165,7 +343,7 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
ValueJson = jsonValue,
LastUpdatedUtc = DateTime.UtcNow
};
dbContext.Set<SettingEntity>().Add(entity);
dbContext.DynamicSettings.Add(entity);
}
else
{
@@ -177,17 +355,91 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "Setting '{Key}' could not be saved to DB.", key);
_logger?.LogWarning(ex, "[SettingsService] Setting '{Key}' could not be saved to DB.", key);
}
}
private static string NormalizeJsonValue(object? rawValue)
{
if (rawValue == null) return "null";
if (rawValue is JsonElement jsonElem)
{
if (jsonElem.ValueKind == JsonValueKind.String)
{
var str = jsonElem.GetString()?.Trim() ?? string.Empty;
var unquoted = str.Trim('\"', ' ');
if (bool.TryParse(unquoted, out var b)) return b ? "true" : "false";
if (long.TryParse(unquoted, out var l)) return l.ToString();
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d.ToString(CultureInfo.InvariantCulture);
return JsonSerializer.Serialize(str);
}
return jsonElem.GetRawText();
}
// Cache trotzdem aktualisieren
_cache[key] = jsonValue;
if (rawValue is string s)
{
var unquoted = s.Trim('\"', ' ');
if (bool.TryParse(unquoted, out var b)) return b ? "true" : "false";
if (long.TryParse(unquoted, out var l)) return l.ToString();
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d.ToString(CultureInfo.InvariantCulture);
return JsonSerializer.Serialize(s);
}
if (rawValue is bool bVal) return bVal ? "true" : "false";
if (rawValue is int or long or short or byte) return rawValue.ToString()!;
if (rawValue is double or float or decimal) return Convert.ToString(rawValue, CultureInfo.InvariantCulture)!;
return JsonSerializer.Serialize(rawValue);
}
private static T DeserializeValue<T>(string json, T defaultValue)
{
if (string.IsNullOrWhiteSpace(json)) return defaultValue;
try
{
var unquoted = json.Trim('\"', ' ');
if (typeof(T) == typeof(bool))
{
if (bool.TryParse(unquoted, out var b))
{
return (T)(object)b;
}
}
else if (typeof(T) == typeof(int))
{
if (int.TryParse(unquoted, out var i))
{
return (T)(object)i;
}
}
else if (typeof(T) == typeof(long))
{
if (long.TryParse(unquoted, out var l))
{
return (T)(object)l;
}
}
else if (typeof(T) == typeof(double))
{
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d))
{
return (T)(object)d;
}
}
else if (typeof(T) == typeof(string))
{
var trimmed = json.Trim();
if (trimmed.StartsWith("\"") && trimmed.EndsWith("\"") && trimmed.Length >= 2)
{
try { return (T)(object)(JsonSerializer.Deserialize<string>(trimmed) ?? trimmed.Trim('\"')); }
catch { return (T)(object)trimmed.Trim('\"'); }
}
return (T)(object)trimmed;
}
var result = JsonSerializer.Deserialize<T>(json);
return result ?? defaultValue;
}
@@ -197,5 +449,106 @@ public class SettingsService<TContext> : ISettingsService<TContext> where TConte
}
}
private static object? DeserializeObject(string json, Type valueType, object? defaultValue)
{
if (string.IsNullOrWhiteSpace(json)) return defaultValue;
try
{
var unquoted = json.Trim('\"', ' ');
if (valueType == typeof(bool))
{
if (bool.TryParse(unquoted, out var b)) return b;
}
else if (valueType == typeof(int))
{
if (int.TryParse(unquoted, out var i)) return i;
}
else if (valueType == typeof(long))
{
if (long.TryParse(unquoted, out var l)) return l;
}
else if (valueType == typeof(double))
{
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d;
}
else if (valueType == typeof(string))
{
var trimmed = json.Trim();
if (trimmed.StartsWith("\"") && trimmed.EndsWith("\"") && trimmed.Length >= 2)
{
try { return JsonSerializer.Deserialize<string>(trimmed) ?? trimmed.Trim('\"'); }
catch { return trimmed.Trim('\"'); }
}
return trimmed;
}
return JsonSerializer.Deserialize(json, valueType) ?? defaultValue;
}
catch
{
return defaultValue;
}
}
private static string MapToSimpleTypeName(Type type)
{
if (type == typeof(bool)) return "bool";
if (type == typeof(int) || type == typeof(short) || type == typeof(byte) || type == typeof(long)) return "int";
if (type == typeof(double) || type == typeof(float) || type == typeof(decimal)) return "double";
return "string";
}
private static (object? Value, string Type) InferJsonValueAndType(string json)
{
if (string.IsNullOrWhiteSpace(json)) return (string.Empty, "string");
try
{
var unquoted = json.Trim('\"', ' ');
if (bool.TryParse(unquoted, out var b)) return (b, "bool");
if (long.TryParse(unquoted, out var l)) return (l, "int");
if (double.TryParse(unquoted, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return (d, "double");
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
return root.ValueKind switch
{
JsonValueKind.True => (true, "bool"),
JsonValueKind.False => (false, "bool"),
JsonValueKind.Number when root.TryGetInt64(out var i) => (i, "int"),
JsonValueKind.Number => (root.GetDouble(), "double"),
JsonValueKind.String => (root.GetString(), "string"),
_ => (json, "string")
};
}
catch
{
return (json, "string");
}
}
private static string FormatDescriptionFromKey(string key)
{
if (key.StartsWith("Logging.Channel.", StringComparison.OrdinalIgnoreCase))
{
return $"Logging-Kanal für {key.Substring(16)} (Ein/Aus)";
}
if (key.StartsWith("Feature.", StringComparison.OrdinalIgnoreCase))
{
return $"Feature-Toggle für {key.Substring(8)}";
}
if (key.StartsWith("Cache.", StringComparison.OrdinalIgnoreCase))
{
return $"Cache-Konfiguration ({key.Substring(6)})";
}
if (key.StartsWith("Scraper.", StringComparison.OrdinalIgnoreCase))
{
return $"Scraper-Konfiguration ({key.Substring(8)})";
}
return key;
}
#endregion
}
@@ -4,8 +4,9 @@ using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Util;
using Microsoft.Extensions.Logging;
namespace FinlyticCore.Services.TradeRepublic;
@@ -15,7 +16,7 @@ namespace FinlyticCore.Services.TradeRepublic;
/// </summary>
public class TradeRepublicClient : ManagedWebSocket
{
private readonly ILogger<TradeRepublicClient> _logger;
private readonly IFinlyticLogger<TradeRepublicClient> _finlyticLogger;
private int _currentSub;
private readonly ConcurrentDictionary<int, TaskCompletionSource<ReceivedMessage>> _pendingRequests = new();
private readonly ConcurrentDictionary<int, Action<string>> _tickerSubscriptions = new();
@@ -26,10 +27,10 @@ public class TradeRepublicClient : ManagedWebSocket
/// <summary>
/// Initializes a new instance of the <see cref="TradeRepublicClient"/> class.
/// </summary>
/// <param name="logger">The logger instance.</param>
public TradeRepublicClient(ILogger<TradeRepublicClient> logger)
/// <param name="finlyticLogger">The logger instance.</param>
public TradeRepublicClient(IFinlyticLogger<TradeRepublicClient> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -57,7 +58,7 @@ public class TradeRepublicClient : ManagedWebSocket
var isConnected = res.Type == "connected";
if (isConnected)
{
_logger.LogInformation("[{Channel}] WebSocket connection to Trade Republic established.", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] WebSocket connection to Trade Republic established.");
}
return isConnected;
@@ -65,7 +66,7 @@ public class TradeRepublicClient : ManagedWebSocket
catch (Exception ex)
{
_pendingRequests.TryRemove(-1, out _);
_logger.LogWarning(ex, "[{Channel}] Failed or timed out establishing Trade Republic WebSocket connection.", "TradeRepublicChannel");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicClient] Failed or timed out establishing Trade Republic WebSocket connection.");
return false;
}
}
@@ -84,7 +85,7 @@ public class TradeRepublicClient : ManagedWebSocket
var tempSub = Interlocked.Increment(ref _currentSub);
var msg = $"sub {tempSub} {JsonSerializer.Serialize(request, typeof(TRequest), FinlyticJsonSerializerContext.Default)}";
_logger.LogDebug("[{Channel}] TR WS Sent (Request): {Message}", "TradeRepublicChannel", msg);
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] TR WS Sent (Request): {Message}", msg);
var tcs = new TaskCompletionSource<ReceivedMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
_pendingRequests.TryAdd(tempSub, tcs);
@@ -99,7 +100,7 @@ public class TradeRepublicClient : ManagedWebSocket
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Error waiting for Trade Republic response ID {SubId}", "TradeRepublicChannel", tempSub);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicClient] Error waiting for Trade Republic response ID {SubId}", tempSub);
return null;
}
finally
@@ -125,7 +126,6 @@ public class TradeRepublicClient : ManagedWebSocket
_tickerSubscriptions[tempSub] = jsonPayload =>
{
// Skip empty or non-JSON payloads (e.g. TR protocol ack messages)
if (string.IsNullOrWhiteSpace(jsonPayload) || (!jsonPayload.TrimStart().StartsWith('{') && !jsonPayload.TrimStart().StartsWith('[')))
return;
@@ -139,12 +139,12 @@ public class TradeRepublicClient : ManagedWebSocket
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to parse real-time ticker payload for {TickerId}", "TradeRepublicChannel", tickerId);
_ = _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicClient] Failed to parse real-time ticker payload for {TickerId}", tickerId);
}
};
_logger.LogInformation("[{Channel}] Subscribing to Trade Republic real-time ticker {TickerId} (Sub ID: {SubId})", "TradeRepublicChannel", tickerId, tempSub);
_logger.LogDebug("[{Channel}] TR WS Sent: {Message}", "TradeRepublicChannel", msg);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] Subscribing to Trade Republic real-time ticker {TickerId} (Sub ID: {SubId})", tickerId, tempSub);
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] TR WS Sent: {Message}", msg);
await SendAsync(msg);
return tempSub;
}
@@ -165,85 +165,81 @@ public class TradeRepublicClient : ManagedWebSocket
}
/// <inheritdoc />
protected override void OnMessageReceived(string message)
{
if (string.IsNullOrWhiteSpace(message)) return;
_logger.LogDebug("[{Channel}] TR WS Recv: {Message}", "TradeRepublicChannel", message);
var trimmed = message.Trim();
int subId;
string type;
string payload;
_logger.LogDebug("Trade republic response: " + message);
if (trimmed.Equals("connected", StringComparison.OrdinalIgnoreCase))
protected override void OnMessageReceived(string message)
{
subId = -1;
type = "connected";
payload = trimmed;
}
else
{
// Ziffern am Anfang zählen (Sub-ID)
var digitLen = 0;
while (digitLen < trimmed.Length && char.IsDigit(trimmed[digitLen]))
{
digitLen++;
}
if (string.IsNullOrWhiteSpace(message)) return;
// Keine Ziffer am Anfang (Reines System-Event/Error ohne ID)
if (digitLen == 0)
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
_ = _finlyticLogger.LogDebugAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicClient] TR WS Recv: {Message}", message);
if (!int.TryParse(trimmed.Substring(0, digitLen), out subId))
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
var trimmed = message.Trim();
var remainder = trimmed.Substring(digitLen).TrimStart();
int subId;
string type;
string payload;
// 2. FALL: "34 connected" oder "34connected"
if (remainder.StartsWith("connected", StringComparison.OrdinalIgnoreCase))
if (trimmed.Equals("connected", StringComparison.OrdinalIgnoreCase))
{
subId = -1; // Mapping auf deine interne -1 für InitAsync
subId = -1;
type = "connected";
payload = remainder;
}
else if (remainder.Length > 0)
{
// Standard Trade Republic Data Push (z.B. "22A {...}")
type = remainder[0].ToString();
payload = remainder.Substring(1).TrimStart();
payload = trimmed;
}
else
{
type = "ack";
payload = string.Empty;
// Ziffern am Anfang zählen (Sub-ID)
var digitLen = 0;
while (digitLen < trimmed.Length && char.IsDigit(trimmed[digitLen]))
{
digitLen++;
}
// Keine Ziffer am Anfang (Reines System-Event/Error ohne ID)
if (digitLen == 0)
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
if (!int.TryParse(trimmed.Substring(0, digitLen), out subId))
{
SystemMessageReceived?.Invoke(trimmed);
return;
}
var remainder = trimmed.Substring(digitLen).TrimStart();
// 2. FALL: "34 connected" oder "34connected"
if (remainder.StartsWith("connected", StringComparison.OrdinalIgnoreCase))
{
subId = -1;
type = "connected";
payload = remainder;
}
else if (remainder.Length > 0)
{
type = remainder[0].ToString();
payload = remainder.Substring(1).TrimStart();
}
else
{
type = "ack";
payload = string.Empty;
}
}
var received = new ReceivedMessage(subId, type, payload);
if (_pendingRequests.TryGetValue(subId, out var tcs))
{
tcs.TrySetResult(received);
}
if (_tickerSubscriptions.TryGetValue(subId, out var handler))
{
handler(payload);
}
UnhandledMessageReceived?.Invoke(received);
}
var received = new ReceivedMessage(subId, type, payload);
// Löst jetzt garantiert dein TaskCompletionSource(-1) in InitAsync auf!
if (_pendingRequests.TryGetValue(subId, out var tcs))
{
tcs.TrySetResult(received);
}
if (_tickerSubscriptions.TryGetValue(subId, out var handler))
{
handler(payload);
}
UnhandledMessageReceived?.Invoke(received);
}
}
/// <summary>
@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using System.Timers;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Models.Assets;
using Microsoft.Extensions.Logging;
using FinlyticCore.Models.Settings;
namespace FinlyticCore.Services.TradeRepublic;
@@ -16,72 +16,50 @@ public interface ITradeRepublicService
/// <summary>
/// Fetches asset metadata from Trade Republic by ISIN.
/// </summary>
/// <param name="isin">The ISIN to search for.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The Trade Republic search response, or null if not found/failed.</returns>
Task<TradeRepublicAssetResponse?> GetAsset(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the total count of available assets grouped by their types.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>An <see cref="AssetsCount"/> object containing the metrics.</returns>
Task<AssetsCount> GetAssetsCount(CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves a paginated chunk of assets filtered by a specific type.
/// </summary>
/// <param name="type">The type of assets to retrieve.</param>
/// <param name="page">The zero-based page index.</param>
/// <param name="pageSize">The number of elements per page.</param>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A <see cref="TradeRepublicAssetResponse"/> containing the elements, or null if the request fails.</returns>
Task<TradeRepublicAssetResponse?> GetAssets(AssetType type, int page, int pageSize, CancellationToken cancellationToken = default);
/// <summary>
/// Subscribes to the real-time ticker stream for a specific ISIN.
/// </summary>
/// <param name="isin">The ISIN.</param>
/// <param name="onTick">The callback action when a tick is received.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The subscription ID, or null if failed.</returns>
Task<int?> SubscribeRealtimeTickerAsync(string isin, Action<TradeRepublicTickerResponse> onTick, CancellationToken cancellationToken = default);
/// <summary>
/// Unsubscribes from a real-time ticker stream.
/// </summary>
/// <param name="subId">The subscription ID to unsubscribe.</param>
/// <returns>A task representing the async operation.</returns>
Task UnsubscribeRealtimeTickerAsync(int subId);
/// <summary>
/// Fetches stock details (company description, events, earnings, analyst ratings) for a specific ISIN.
/// </summary>
/// <param name="isin">The ISIN of the stock.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The stock details response, or null if failed.</returns>
Task<TradeRepublicStockDetailsResponse?> GetStockDetailsAsync(string isin, CancellationToken cancellationToken = default);
/// <summary>
/// Fetches derivative products (KnockOuts, Warrants, Factor Certificates) for an underlying ISIN.
/// </summary>
/// <param name="request">The derivative query parameters.</param>
/// <param name="cancellationToken">A cancellation token.</param>
/// <returns>The derivatives response, or null if failed.</returns>
Task<TradeRepublicDerivativesResponse?> GetDerivativesAsync(TradeRepublicDerivativesRequest request, CancellationToken cancellationToken = default);
}
public class TradeRepublicService : ITradeRepublicService, IDisposable
{
private readonly TradeRepublicClient _client;
private readonly ILogger<TradeRepublicService> _logger;
private readonly IFinlyticLogger<TradeRepublicService> _finlyticLogger;
private readonly System.Timers.Timer _inactivityTimer;
private readonly SemaphoreSlim _lock = new(1, 1);
public TradeRepublicService(TradeRepublicClient client, ILogger<TradeRepublicService> logger)
public TradeRepublicService(TradeRepublicClient client, IFinlyticLogger<TradeRepublicService> finlyticLogger)
{
_client = client;
_logger = logger;
_finlyticLogger = finlyticLogger;
_inactivityTimer = new System.Timers.Timer(TimeSpan.FromSeconds(461).TotalMilliseconds);
_inactivityTimer.AutoReset = false;
@@ -96,14 +74,14 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
_inactivityTimer.Stop();
if (!_client.IsConnected)
{
_logger.LogInformation("[{Channel}] Connecting to Trade Republic API WebSocket...", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Connecting to Trade Republic API WebSocket...");
bool connected = await _client.InitAsync();
if (!connected)
{
_logger.LogWarning("[{Channel}] Trade Republic WebSocket connection failed or timed out.", "TradeRepublicChannel");
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Trade Republic WebSocket connection failed or timed out.");
throw new InvalidOperationException("Trade Republic WebSocket is not connected.");
}
_logger.LogInformation("[{Channel}] Successfully connected to Trade Republic API.", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Successfully connected to Trade Republic API.");
}
_inactivityTimer.Start();
}
@@ -131,7 +109,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error while fetching asset metadata for ISIN {Isin}", "TradeRepublicChannel", isin);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicService] Error while fetching asset metadata for ISIN {Isin}", isin);
return null;
}
}
@@ -206,7 +184,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error while fetching stock details for ISIN {Isin}", "TradeRepublicChannel", isin);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicService] Error while fetching stock details for ISIN {Isin}", isin);
return null;
}
}
@@ -221,7 +199,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error while fetching derivatives for underlying {Underlying}", "TradeRepublicChannel", request.Underlying);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.TradeRepublicChannel, ex, "[TradeRepublicService] Error while fetching derivatives for underlying {Underlying}", request.Underlying);
return null;
}
}
@@ -232,7 +210,7 @@ public class TradeRepublicService : ITradeRepublicService, IDisposable
{
await _lock.WaitAsync();
if (!_client.IsConnected) return;
_logger.LogInformation("[{Channel}] Inactivity timer expired. Auto-disconnecting Trade Republic WebSocket.", "TradeRepublicChannel");
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.TradeRepublicChannel, "[TradeRepublicService] Inactivity timer expired. Auto-disconnecting Trade Republic WebSocket.");
await _client.DisconnectAsync();
}
catch { }
@@ -1,19 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Clients;
using FinlyticCore.Dtos.Fundamentals;
using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Util;
using Microsoft.Extensions.Logging;
using FinlyticCore.Utils;
using Microsoft.Extensions.Configuration;
namespace FinlyticFundamentals.Services;
namespace FinlyticCore.Services.Yahoo;
public interface IYahooFinanceScraper
{
@@ -31,28 +29,26 @@ public interface IYahooFinanceScraper
/// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab
/// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind.
/// </summary>
/// <param name="symbolOrIsin">Das Tickersymbol (z. B. "MSFT") oder die ISIN.</param>
/// <param name="forceHtmlScrape">Erzwingt sofortiges HTML-Scraping ohne API-Vorprüfung.</param>
/// <param name="cancellationToken">Abbruch-Token.</param>
/// <returns>Das aggregierte <see cref="YahooQuoteSummaryModulesDto"/> oder <c>null</c>.</returns>
Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
bool includeProfile = true,
CancellationToken cancellationToken = default);
}
public class YahooFinanceScraper : IYahooFinanceScraper
{
private const string _serviceName = nameof(YahooFinanceScraper);
private readonly YahooFinanceClient _yahooApiClient;
private readonly IYahooFinanceHtmlClient _htmlScraperClient;
private readonly Microsoft.Extensions.Configuration.IConfiguration _configuration;
private readonly IFinlyticLogger<YahooFinanceScraper, FundamentalsDbContext> _finlyticLogger;
private readonly IConfiguration _configuration;
private readonly IFinlyticLogger<YahooFinanceScraper> _finlyticLogger;
public YahooFinanceScraper(
YahooFinanceClient yahooApiClient,
IYahooFinanceHtmlClient htmlScraperClient,
Microsoft.Extensions.Configuration.IConfiguration configuration,
IFinlyticLogger<YahooFinanceScraper, FundamentalsDbContext> finlyticLogger)
IConfiguration configuration,
IFinlyticLogger<YahooFinanceScraper> finlyticLogger)
{
_yahooApiClient = yahooApiClient;
_htmlScraperClient = htmlScraperClient;
@@ -78,7 +74,7 @@ public class YahooFinanceScraper : IYahooFinanceScraper
// Crypto / Trade Republic interne ISINs (beginnend mit 'X', z. B. XF000BTC0017)
if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase))
{
var (cryptoSubtitle, cryptoName) = await FinlyticCore.Utils.CryptoSubtitleResolver.ResolveCryptoInfoAsync(
var (cryptoSubtitle, cryptoName) = await CryptoSubtitleResolver.ResolveCryptoInfoAsync(
cleanIsin, _configuration.GetConnectionString("DefaultConnection"), cancellationToken);
if (!string.IsNullOrWhiteSpace(cryptoSubtitle))
@@ -103,11 +99,14 @@ public class YahooFinanceScraper : IYahooFinanceScraper
}
}
}
catch { }
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] Crypto search failed for {cryptoSubtitle}");
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[YahooFinanceScraper] Resolved Crypto ISIN {Isin} to {Symbol} using Subtitle {Sub}",
cleanIsin, cryptoEur, cryptoSubtitle);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.FundamentalsChannel,
$"[{_serviceName}] Resolved Crypto ISIN {cleanIsin} to {cryptoEur} using Subtitle {cryptoSubtitle}");
return symbols
.OrderBy(s => s.priority)
@@ -130,7 +129,6 @@ public class YahooFinanceScraper : IYahooFinanceScraper
foreach (var q in validQuotes.Skip(1))
{
if (!symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase)))
{
symbols.Add((q.Symbol, q.Exchange ?? string.Empty, Math.Max(1, GetExchangePriority(q.Symbol, cleanIsin))));
@@ -138,7 +136,7 @@ public class YahooFinanceScraper : IYahooFinanceScraper
}
}
// 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind
// 2. Falls Ticker gefunden, mit Unternehmensname noch mehr internationale Exchangeticker suchen (z.B. APC.DE)
if (validQuotes.Count > 0)
{
var companyName = validQuotes[0].LongName ?? validQuotes[0].ShortName;
@@ -158,20 +156,28 @@ public class YahooFinanceScraper : IYahooFinanceScraper
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] Fehler beim Auflösen des Tickers für ISIN '{Isin}'", cleanIsin);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] Fehler beim Auflösen des Tickers für ISIN '{cleanIsin}'");
}
return symbols
var result = symbols
.OrderBy(s => s.priority)
.Select(s => new TickerInfoDto(){Ticker = s.symbol, Exchange = s.exchange})
.Select(s => new TickerInfoDto
{
Ticker = s.symbol,
Exchange = !string.IsNullOrWhiteSpace(s.exchange) ? s.exchange : "Unknown"
})
.DistinctBy(s => s.Ticker, StringComparer.OrdinalIgnoreCase)
.ToList();
return result;
}
/// <inheritdoc />
public async Task<YahooQuoteSummaryModulesDto?> GetQuoteSummaryModulesAsync(
string symbolOrIsin,
bool forceHtmlScrape = false,
bool includeProfile = true,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null;
@@ -197,26 +203,26 @@ public class YahooFinanceScraper : IYahooFinanceScraper
{
try
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Starte primären API-Abruf für '{Symbol}'...", symbol);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte primären API-Abruf für '{symbol}'...");
var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken);
apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault();
if (apiModules != null && HasSufficientData(apiModules))
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Erfolgreich Daten über API bezogen für '{Symbol}'.", symbol);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Erfolgreich Daten über API bezogen für '{symbol}'.");
return apiModules;
}
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] API lieferte unvollständige Daten für '{Symbol}'. Initiiere Fallback...", symbol);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] API lieferte unvollständige Daten für '{symbol}'. Initiiere Fallback...");
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] API-Abruf fehlgeschlagen für '{Symbol}'. Wechsle zu Scraper...", symbol);
await _finlyticLogger.LogWarningAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] API-Abruf fehlgeschlagen für '{symbol}'. Wechsle zu Scraper...");
}
}
@@ -226,15 +232,15 @@ public class YahooFinanceScraper : IYahooFinanceScraper
YahooQuoteSummaryModulesDto? htmlModules = null;
try
{
await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel,
"[YahooFinanceScraper] Starte HTML-Scraper Fallback für '{Symbol}'...", symbol);
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.YahooClientChannel,
$"[{_serviceName}] Starte HTML-Scraper Fallback für '{symbol}' (IncludeProfile: {includeProfile})...");
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken);
htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, includeProfile, cancellationToken);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.YahooClientChannel, ex,
"[YahooFinanceScraper] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{Symbol}'.", symbol);
await _finlyticLogger.LogErrorAsync(CoreSettingKeys.YahooClientChannel, ex,
$"[{_serviceName}] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{symbol}'.");
}
// -------------------------------------------------------------
@@ -246,9 +252,6 @@ public class YahooFinanceScraper : IYahooFinanceScraper
return MergeModules(apiModules, htmlModules);
}
/// <summary>
/// Prüft, ob das Modul-DTO die wesentlichen Fundamentalblöcke enthält.
/// </summary>
private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules)
{
return modules.SummaryDetail != null ||
@@ -256,19 +259,20 @@ public class YahooFinanceScraper : IYahooFinanceScraper
modules.DefaultKeyStatistics != null;
}
/// <summary>
/// Führt API- und Scraper-Daten zusammen, damit Lücken in API-Responses geschlossen werden.
/// </summary>
private static YahooQuoteSummaryModulesDto MergeModules(
YahooQuoteSummaryModulesDto primary,
YahooQuoteSummaryModulesDto secondary)
public static YahooQuoteSummaryModulesDto? MergeModules(
YahooQuoteSummaryModulesDto? primary,
YahooQuoteSummaryModulesDto? secondary)
{
if (primary == null && secondary == null) return null;
if (primary == null) return secondary;
if (secondary == null) return primary;
return new YahooQuoteSummaryModulesDto(
QuoteType: primary.QuoteType ?? secondary.QuoteType,
AssetProfile: primary.AssetProfile ?? secondary.AssetProfile,
FinancialData: primary.FinancialData ?? secondary.FinancialData,
DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics,
SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail,
FinancialData: MergeFinancialData(primary.FinancialData, secondary.FinancialData),
DefaultKeyStatistics: MergeDefaultKeyStatistics(primary.DefaultKeyStatistics, secondary.DefaultKeyStatistics),
SummaryDetail: MergeSummaryDetail(primary.SummaryDetail, secondary.SummaryDetail),
IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory,
IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly,
BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory,
@@ -279,6 +283,124 @@ public class YahooFinanceScraper : IYahooFinanceScraper
);
}
private static YahooFinancialDataDto? MergeFinancialData(YahooFinancialDataDto? a, YahooFinancialDataDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooFinancialDataDto(
CurrentPrice: a.CurrentPrice ?? b.CurrentPrice,
TargetHighPrice: a.TargetHighPrice ?? b.TargetHighPrice,
TargetLowPrice: a.TargetLowPrice ?? b.TargetLowPrice,
TargetMeanPrice: a.TargetMeanPrice ?? b.TargetMeanPrice,
TargetMedianPrice: a.TargetMedianPrice ?? b.TargetMedianPrice,
RecommendationMean: a.RecommendationMean ?? b.RecommendationMean,
RecommendationKey: !string.IsNullOrWhiteSpace(a.RecommendationKey) && a.RecommendationKey != "none" ? a.RecommendationKey : b.RecommendationKey,
NumberOfAnalystOpinions: a.NumberOfAnalystOpinions ?? b.NumberOfAnalystOpinions,
TotalCash: a.TotalCash ?? b.TotalCash,
TotalCashPerShare: a.TotalCashPerShare ?? b.TotalCashPerShare,
Ebitda: a.Ebitda ?? b.Ebitda,
TotalDebt: a.TotalDebt ?? b.TotalDebt,
QuickRatio: a.QuickRatio ?? b.QuickRatio,
CurrentRatio: a.CurrentRatio ?? b.CurrentRatio,
TotalRevenue: a.TotalRevenue ?? b.TotalRevenue,
DebtToEquity: a.DebtToEquity ?? b.DebtToEquity,
RevenuePerShare: a.RevenuePerShare ?? b.RevenuePerShare,
ReturnOnAssets: a.ReturnOnAssets ?? b.ReturnOnAssets,
ReturnOnEquity: a.ReturnOnEquity ?? b.ReturnOnEquity,
GrossProfits: a.GrossProfits ?? b.GrossProfits,
FreeCashflow: a.FreeCashflow ?? b.FreeCashflow,
OperatingCashflow: a.OperatingCashflow ?? b.OperatingCashflow,
RevenueGrowth: a.RevenueGrowth ?? b.RevenueGrowth,
GrossMargins: a.GrossMargins ?? b.GrossMargins,
EbitdaMargins: a.EbitdaMargins ?? b.EbitdaMargins,
OperatingMargins: a.OperatingMargins ?? b.OperatingMargins,
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
FinancialCurrency: a.FinancialCurrency ?? b.FinancialCurrency
);
}
private static YahooDefaultKeyStatisticsDto? MergeDefaultKeyStatistics(YahooDefaultKeyStatisticsDto? a, YahooDefaultKeyStatisticsDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooDefaultKeyStatisticsDto(
PriceToBook: a.PriceToBook ?? b.PriceToBook,
EnterpriseValue: a.EnterpriseValue ?? b.EnterpriseValue,
ForwardPE: a.ForwardPE ?? b.ForwardPE,
ProfitMargins: a.ProfitMargins ?? b.ProfitMargins,
FloatShares: a.FloatShares ?? b.FloatShares,
SharesOutstanding: a.SharesOutstanding ?? b.SharesOutstanding,
SharesShort: a.SharesShort ?? b.SharesShort,
SharesShortPriorMonth: a.SharesShortPriorMonth ?? b.SharesShortPriorMonth,
SharesShortPreviousMonthDate: a.SharesShortPreviousMonthDate ?? b.SharesShortPreviousMonthDate,
DateShortInterest: a.DateShortInterest ?? b.DateShortInterest,
SharesPercentSharesOut: a.SharesPercentSharesOut ?? b.SharesPercentSharesOut,
HeldPercentInsiders: a.HeldPercentInsiders ?? b.HeldPercentInsiders,
HeldPercentInstitutions: a.HeldPercentInstitutions ?? b.HeldPercentInstitutions,
ShortRatio: a.ShortRatio ?? b.ShortRatio,
ShortPercentOfFloat: a.ShortPercentOfFloat ?? b.ShortPercentOfFloat,
Beta: a.Beta ?? b.Beta,
Category: a.Category ?? b.Category,
BookValue: a.BookValue ?? b.BookValue,
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
LastFiscalYearEnd: a.LastFiscalYearEnd ?? b.LastFiscalYearEnd,
NextFiscalYearEnd: a.NextFiscalYearEnd ?? b.NextFiscalYearEnd,
MostRecentQuarter: a.MostRecentQuarter ?? b.MostRecentQuarter,
EarningsQuarterlyGrowth: a.EarningsQuarterlyGrowth ?? b.EarningsQuarterlyGrowth,
NetIncomeToCommon: a.NetIncomeToCommon ?? b.NetIncomeToCommon,
TrailingEps: a.TrailingEps ?? b.TrailingEps,
ForwardEps: a.ForwardEps ?? b.ForwardEps,
PegRatio: a.PegRatio ?? b.PegRatio,
EnterpriseToRevenue: a.EnterpriseToRevenue ?? b.EnterpriseToRevenue,
EnterpriseToEbitda: a.EnterpriseToEbitda ?? b.EnterpriseToEbitda,
FiftyTwoWeekChange: a.FiftyTwoWeekChange ?? b.FiftyTwoWeekChange,
SandP52WeekChange: a.SandP52WeekChange ?? b.SandP52WeekChange
);
}
private static YahooSummaryDetailDto? MergeSummaryDetail(YahooSummaryDetailDto? a, YahooSummaryDetailDto? b)
{
if (a == null) return b;
if (b == null) return a;
return new YahooSummaryDetailDto(
MaxAge: a.MaxAge ?? b.MaxAge,
PriceHint: a.PriceHint ?? b.PriceHint,
PreviousClose: a.PreviousClose ?? b.PreviousClose,
Open: a.Open ?? b.Open,
DayLow: a.DayLow ?? b.DayLow,
DayHigh: a.DayHigh ?? b.DayHigh,
RegularMarketPreviousClose: a.RegularMarketPreviousClose ?? b.RegularMarketPreviousClose,
RegularMarketOpen: a.RegularMarketOpen ?? b.RegularMarketOpen,
RegularMarketDayLow: a.RegularMarketDayLow ?? b.RegularMarketDayLow,
RegularMarketDayHigh: a.RegularMarketDayHigh ?? b.RegularMarketDayHigh,
DividendRate: a.DividendRate ?? b.DividendRate,
DividendYield: a.DividendYield ?? b.DividendYield,
ExDividendDate: a.ExDividendDate ?? b.ExDividendDate,
PayoutRatio: a.PayoutRatio ?? b.PayoutRatio,
FiveYearAvgDividendYield: a.FiveYearAvgDividendYield ?? b.FiveYearAvgDividendYield,
Beta: a.Beta ?? b.Beta,
TrailingPE: a.TrailingPE ?? b.TrailingPE,
ForwardPE: a.ForwardPE ?? b.ForwardPE,
Volume: a.Volume ?? b.Volume,
RegularMarketVolume: a.RegularMarketVolume ?? b.RegularMarketVolume,
AverageVolume: a.AverageVolume ?? b.AverageVolume,
AverageVolume10days: a.AverageVolume10days ?? b.AverageVolume10days,
AverageDailyVolume10Day: a.AverageDailyVolume10Day ?? b.AverageDailyVolume10Day,
Bid: a.Bid ?? b.Bid,
Ask: a.Ask ?? b.Ask,
BidSize: a.BidSize ?? b.BidSize,
AskSize: a.AskSize ?? b.AskSize,
MarketCap: a.MarketCap ?? b.MarketCap,
FiftyTwoWeekLow: a.FiftyTwoWeekLow ?? b.FiftyTwoWeekLow,
FiftyTwoWeekHigh: a.FiftyTwoWeekHigh ?? b.FiftyTwoWeekHigh,
PriceToSalesTrailing12Months: a.PriceToSalesTrailing12Months ?? b.PriceToSalesTrailing12Months,
Currency: a.Currency ?? b.Currency
);
}
private static bool IsIsin(string value)
{
return value.Length == 12 &&
@@ -311,4 +433,4 @@ public class YahooFinanceScraper : IYahooFinanceScraper
return 10;
}
}
}
@@ -18,6 +18,8 @@ namespace FinlyticCore.Util;
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(TradeProposalDto))]
[JsonSerializable(typeof(List<TradeProposalDto>))]
[JsonSerializable(typeof(FinlyticCore.Dtos.Logging.LogMessageDto))]
[JsonSerializable(typeof(List<FinlyticCore.Dtos.Logging.LogMessageDto>))]
[JsonSerializable(typeof(TradeAcceptanceDto))]
[JsonSerializable(typeof(List<TradeAcceptanceDto>))]
[JsonSerializable(typeof(CloseTradeRequest))]
+69 -19
View File
@@ -6,6 +6,8 @@ using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Models;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using Microsoft.Extensions.Logging;
using MQTTnet;
@@ -14,10 +16,13 @@ namespace FinlyticCore.Util;
/// <summary>
/// An abstract, resilient MQTT client wrapper designed for microservice architectures.
/// Handles automatic reconnection, structured JSON publishing, thread-safe subscription management, and synchronous Request-Reply (RPC).
/// Supports channel-controlled logging via <see cref="CoreSettingKeys.MqttChannel"/>.
/// </summary>
public abstract class ManagedMqttClient : IDisposable
{
private readonly ILogger<ManagedMqttClient> _logger;
private readonly ISettingsService? _settingsService;
private readonly IFinlyticLogger<ManagedMqttClient>? _finlyticLogger;
private readonly IMqttClient _mqttClient;
private CancellationTokenSource? _cts;
@@ -29,15 +34,58 @@ public abstract class ManagedMqttClient : IDisposable
/// </summary>
public bool IsConnected => _mqttClient.IsConnected;
protected ManagedMqttClient(ILogger<ManagedMqttClient> logger)
protected ManagedMqttClient(
ILogger<ManagedMqttClient> logger,
ISettingsService? settingsService = null,
IFinlyticLogger<ManagedMqttClient>? finlyticLogger = null)
{
_logger = logger;
_settingsService = settingsService;
_finlyticLogger = finlyticLogger;
_mqttClient = new MqttClientFactory().CreateMqttClient();
_mqttClient.ApplicationMessageReceivedAsync += HandleIncomingMessageAsync;
_mqttClient.DisconnectedAsync += HandleDisconnectAsync;
}
private async Task LogMqttInfoAsync(string message, params object[] args)
{
if (_finlyticLogger != null)
{
await _finlyticLogger.LogInfoAsync(CoreSettingKeys.MqttChannel, message, args);
}
else if (_settingsService != null)
{
if (await _settingsService.GetSettingAsync(CoreSettingKeys.MqttChannel))
{
_logger.LogInformation(message, args);
}
}
else
{
_logger.LogInformation(message, args);
}
}
private async Task LogMqttDebugAsync(string message, params object[] args)
{
if (_finlyticLogger != null)
{
await _finlyticLogger.LogDebugAsync(CoreSettingKeys.MqttChannel, message, args);
}
else if (_settingsService != null)
{
if (await _settingsService.GetSettingAsync(CoreSettingKeys.MqttChannel))
{
_logger.LogDebug(message, args);
}
}
else
{
_logger.LogDebug(message, args);
}
}
/// <summary>
/// Establishes a connection to the MQTT broker and initializes the background auto-reconnection loop.
/// </summary>
@@ -61,12 +109,12 @@ public abstract class ManagedMqttClient : IDisposable
var options = optionsBuilder.Build();
_logger.LogInformation("Connecting to MQTT broker at {Host}:{Port}...", config.Host, config.Port);
await LogMqttInfoAsync("Connecting to MQTT broker at {Host}:{Port}...", config.Host, config.Port);
try
{
await _mqttClient.ConnectAsync(options, _cts.Token);
_logger.LogInformation("Successfully connected to MQTT broker.");
await LogMqttInfoAsync("Successfully connected to MQTT broker.");
await OnConnectedAsync();
}
@@ -94,7 +142,7 @@ public abstract class ManagedMqttClient : IDisposable
{
Reason = MqttClientDisconnectOptionsReason.NormalDisconnection
});
_logger.LogInformation("MQTT connection gracefully closed.");
await LogMqttInfoAsync("MQTT connection gracefully closed.");
}
catch (Exception ex)
{
@@ -127,7 +175,7 @@ public abstract class ManagedMqttClient : IDisposable
.Build();
await _mqttClient.SubscribeAsync(subscribeOptions, CancellationToken.None);
_logger.LogDebug("Successfully subscribed to topic: {Topic} (NoLocal: {NoLocal})", topic, noLocal);
await LogMqttDebugAsync("Successfully subscribed to topic: {Topic} (NoLocal: {NoLocal})", topic, noLocal);
}
/// <summary>
@@ -180,15 +228,21 @@ public abstract class ManagedMqttClient : IDisposable
return _mqttClient.PublishAsync(message, CancellationToken.None);
}
/// <summary>
/// Sends a parameterless request to an RPC channel and asynchronously blocks until a matching response arrives.
/// </summary>
public Task<TResponse?> SendRpcRequestAsync<TResponse>(
string channel,
TimeSpan? timeout = null)
where TResponse : class
{
return SendRpcRequestAsync<TResponse, string>(channel, string.Empty, timeout);
}
/// <summary>
/// Sends a generic request payload to an RPC channel and asynchronously blocks until a matching response arrives.
/// Uses the topic conventions: <c>services/request/{channel}/{correlationId}</c> and <c>services/response/{channel}/{correlationId}</c>.
/// </summary>
/// <typeparam name="TResponse">The expected strongly-typed object type of the reply.</typeparam>
/// <typeparam name="TRequest">The type of the payload being transmitted.</typeparam>
/// <param name="channel">The target sub-channel or service name (e.g., "sentix", "assets").</param>
/// <param name="requestData">The object that will be serialized to JSON and sent.</param>
/// <param name="timeout">Optional. Maximum time to wait before returning null. Defaults to 10 seconds.</param>
public async Task<TResponse?> SendRpcRequestAsync<TResponse, TRequest>(
string channel,
TRequest requestData,
@@ -209,7 +263,7 @@ public abstract class ManagedMqttClient : IDisposable
// 2. Serialize and dispatch via the existing JSON helper
await PublishAsync(requestTopic, requestData);
_logger.LogInformation("RPC request published to '{Topic}' [CorrelationId: {Id}]", requestTopic, correlationId);
await LogMqttInfoAsync("RPC request published to '{Topic}' [CorrelationId: {Id}]", requestTopic, correlationId);
try
{
@@ -250,7 +304,7 @@ public abstract class ManagedMqttClient : IDisposable
{
var topic = e.ApplicationMessage.Topic;
var payload = Encoding.UTF8.GetString(e.ApplicationMessage.Payload);
_logger.LogInformation("MQTT message received on topic '{Topic}', length={Length}", topic, payload?.Length ?? 0);
await LogMqttDebugAsync("MQTT message received on topic '{Topic}', length={Length}", topic, payload?.Length ?? 0);
// Intercept message if it belongs to the RPC response convention
if (topic.StartsWith("services/response/"))
@@ -282,7 +336,6 @@ public abstract class ManagedMqttClient : IDisposable
private async Task HandleDisconnectAsync(MqttClientDisconnectedEventArgs e)
{
// Prevent trigger during deliberate connection shutdowns
if (_cts == null || _cts.IsCancellationRequested)
return;
@@ -296,19 +349,19 @@ public abstract class ManagedMqttClient : IDisposable
try
{
_logger.LogInformation("Reconnect attempt {Attempt} in {Delay}s...", attempt, delaySeconds);
await LogMqttInfoAsync("Reconnect attempt {Attempt} in {Delay}s...", attempt, delaySeconds);
await Task.Delay(TimeSpan.FromSeconds(delaySeconds), _cts.Token);
await _mqttClient.ReconnectAsync(_cts.Token);
if (_mqttClient.IsConnected)
{
_logger.LogInformation("MQTT client reconnected successfully after {Attempt} attempt(s).", attempt);
await LogMqttInfoAsync("MQTT client reconnected successfully after {Attempt} attempt(s).", attempt);
await OnConnectedAsync();
return;
}
}
catch (OperationCanceledException) { return; /* Expected on application shutdown */ }
catch (OperationCanceledException) { return; }
catch (Exception ex)
{
_logger.LogWarning(ex, "Reconnect attempt {Attempt} to the MQTT broker failed.", attempt);
@@ -318,15 +371,12 @@ public abstract class ManagedMqttClient : IDisposable
/// <summary>
/// Fired automatically whenever a connection or reconnection is successfully established.
/// Ideal place to trigger <see cref="SubscribeAsync"/> operations.
/// </summary>
protected abstract Task OnConnectedAsync();
/// <summary>
/// Fired whenever a new message lands on a registered subscription channel.
/// </summary>
/// <param name="topic">The specific topic where the message was broadcasted.</param>
/// <param name="payload">The deserialized UTF-8 payload string.</param>
protected abstract Task OnMessageReceivedAsync(string topic, string payload);
/// <summary>
@@ -1,10 +1,12 @@
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using FinlyticFundamentals.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticFundamentals.Database;
public class FundamentalsDbContext : DbContext
public class FundamentalsDbContext : DbContext, ISettingsDbContext
{
public FundamentalsDbContext(DbContextOptions<FundamentalsDbContext> options) : base(options)
{
@@ -23,7 +25,7 @@ public class FundamentalsDbContext : DbContext
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key);
entity.HasIndex(e => e.Key).IsUnique();
});
modelBuilder.Entity<AssetDataEntity>(entity =>
@@ -65,13 +67,16 @@ public class FundamentalsDbContext : DbContext
modelBuilder.Entity<FundamentalDataEntity>(entity =>
{
entity.HasKey(e => e.Isin);
entity.HasKey(e => e.Id);
entity.OwnsOne(e => e.Ticker, t =>
{
t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty);
t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty);
t.HasIndex(p => p.Ticker);
});
entity.HasIndex(e => new { e.AssetDataIsin });
});
modelBuilder.Entity<KeyExecutiveEntity>(entity =>
@@ -93,3 +98,13 @@ public class FundamentalsDbContext : DbContext
});
}
}
public class FundamentalsDbContextFactory : IDesignTimeDbContextFactory<FundamentalsDbContext>
{
public FundamentalsDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<FundamentalsDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=fundamentals;Username=postgres;Password=postgres");
return new FundamentalsDbContext(optionsBuilder.Options);
}
}
@@ -6,7 +6,7 @@ namespace FinlyticFundamentals.Entities;
public class FundamentalDataEntity
{
[Key]
public string Isin { get; set; } = string.Empty;
public Guid Id { get; set; } = Guid.NewGuid();
public TickerEntity Ticker { get; set; } = new();
@@ -0,0 +1,430 @@
// <auto-generated />
using System;
using FinlyticFundamentals.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticFundamentals.Migrations
{
[DbContext(typeof(FundamentalsDbContext))]
[Migration("20260815183935_AddDynamicSettings")]
partial class AddDynamicSettings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.Property<string>("Isin")
.HasColumnType("text");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.HasKey("Isin");
b.ToTable("AssetData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("Date")
.HasColumnType("timestamp with time zone");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AssetDataIsin");
b.ToTable("AssetEvents");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.Property<string>("Isin")
.HasColumnType("text");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ConsensusRating")
.HasColumnType("text");
b.Property<decimal?>("CurrentRatio")
.HasColumnType("numeric");
b.Property<decimal?>("DebtToEquity")
.HasColumnType("numeric");
b.Property<decimal?>("DilutedEps")
.HasColumnType("numeric");
b.Property<decimal?>("Ebitda")
.HasColumnType("numeric");
b.Property<decimal?>("EnterpriseValue")
.HasColumnType("numeric");
b.Property<decimal?>("EvToEbitda")
.HasColumnType("numeric");
b.Property<decimal?>("FiftyTwoWeekHigh")
.HasColumnType("numeric");
b.Property<decimal?>("FiftyTwoWeekLow")
.HasColumnType("numeric");
b.Property<decimal?>("ForwardDividendYield")
.HasColumnType("numeric");
b.Property<decimal?>("ForwardPe")
.HasColumnType("numeric");
b.Property<decimal?>("FreeCashFlow")
.HasColumnType("numeric");
b.Property<decimal?>("GrossProfit")
.HasColumnType("numeric");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<decimal?>("MarketCap")
.HasColumnType("numeric");
b.Property<decimal?>("NetIncome")
.HasColumnType("numeric");
b.Property<decimal?>("OperatingCashFlow")
.HasColumnType("numeric");
b.Property<decimal?>("OperatingIncome")
.HasColumnType("numeric");
b.Property<decimal?>("PayoutRatio")
.HasColumnType("numeric");
b.Property<decimal?>("PegRatio")
.HasColumnType("numeric");
b.Property<decimal?>("PercentHeldByInsiders")
.HasColumnType("numeric");
b.Property<decimal?>("PercentHeldByInstitutions")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetHigh")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetLow")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetMean")
.HasColumnType("numeric");
b.Property<decimal?>("PriceToBook")
.HasColumnType("numeric");
b.Property<decimal?>("PriceToSales")
.HasColumnType("numeric");
b.Property<decimal?>("ReturnOnAssets")
.HasColumnType("numeric");
b.Property<decimal?>("ReturnOnEquity")
.HasColumnType("numeric");
b.Property<decimal?>("RevenueGrowthYoY")
.HasColumnType("numeric");
b.Property<decimal?>("ShortPercentOfFloat")
.HasColumnType("numeric");
b.Property<decimal?>("ShortRatio")
.HasColumnType("numeric");
b.Property<decimal?>("TotalCash")
.HasColumnType("numeric");
b.Property<decimal?>("TotalDebt")
.HasColumnType("numeric");
b.Property<decimal?>("TotalRevenue")
.HasColumnType("numeric");
b.Property<decimal?>("TrailingPe")
.HasColumnType("numeric");
b.HasKey("Isin");
b.HasIndex("AssetDataIsin");
b.ToTable("FundamentalData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Payment")
.IsRequired()
.HasColumnType("text");
b.Property<int>("SortOrder")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0);
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AssetDataIsin", "SortOrder");
b.ToTable("KeyExecutives");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 =>
{
b1.Property<string>("AssetDataEntityIsin")
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("PrimaryTickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("PrimaryTicker");
b1.HasKey("AssetDataEntityIsin");
b1.ToTable("AssetData");
b1.WithOwner()
.HasForeignKey("AssetDataEntityIsin");
});
b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 =>
{
b1.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b1.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.HasColumnType("text")
.HasColumnName("Exchange");
b1.Property<string>("Ticker")
.IsRequired()
.HasColumnType("text")
.HasColumnName("Ticker");
b1.HasKey("Id");
b1.HasIndex("AssetDataIsin");
b1.HasIndex("Ticker");
b1.ToTable("Tickers", (string)null);
b1.WithOwner()
.HasForeignKey("AssetDataIsin");
});
b.Navigation("AvailableTickers");
b.Navigation("PrimaryTicker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("AssetEvents")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<Guid>("AssetEventEntityId")
.HasColumnType("uuid");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("TickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("AssetEventEntityId");
b1.ToTable("AssetEvents");
b1.WithOwner()
.HasForeignKey("AssetEventEntityId");
});
b.Navigation("AssetData");
b.Navigation("Ticker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("FundamentalData")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<string>("FundamentalDataEntityIsin")
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("TickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("FundamentalDataEntityIsin");
b1.ToTable("FundamentalData");
b1.WithOwner()
.HasForeignKey("FundamentalDataEntityIsin");
});
b.Navigation("AssetData");
b.Navigation("Ticker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("KeyExecutives")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("AssetData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.Navigation("AssetEvents");
b.Navigation("FundamentalData");
b.Navigation("KeyExecutives");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticFundamentals.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings");
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings");
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key");
}
}
}
@@ -0,0 +1,433 @@
// <auto-generated />
using System;
using FinlyticFundamentals.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticFundamentals.Migrations
{
[DbContext(typeof(FundamentalsDbContext))]
[Migration("20260816103109_MakeFundamentalDataPerTicker")]
partial class MakeFundamentalDataPerTicker
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.Property<string>("Isin")
.HasColumnType("text");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.HasKey("Isin");
b.ToTable("AssetData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<DateTime>("Date")
.HasColumnType("timestamp with time zone");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AssetDataIsin");
b.ToTable("AssetEvents");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("ConsensusRating")
.HasColumnType("text");
b.Property<decimal?>("CurrentRatio")
.HasColumnType("numeric");
b.Property<decimal?>("DebtToEquity")
.HasColumnType("numeric");
b.Property<decimal?>("DilutedEps")
.HasColumnType("numeric");
b.Property<decimal?>("Ebitda")
.HasColumnType("numeric");
b.Property<decimal?>("EnterpriseValue")
.HasColumnType("numeric");
b.Property<decimal?>("EvToEbitda")
.HasColumnType("numeric");
b.Property<decimal?>("FiftyTwoWeekHigh")
.HasColumnType("numeric");
b.Property<decimal?>("FiftyTwoWeekLow")
.HasColumnType("numeric");
b.Property<decimal?>("ForwardDividendYield")
.HasColumnType("numeric");
b.Property<decimal?>("ForwardPe")
.HasColumnType("numeric");
b.Property<decimal?>("FreeCashFlow")
.HasColumnType("numeric");
b.Property<decimal?>("GrossProfit")
.HasColumnType("numeric");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<decimal?>("MarketCap")
.HasColumnType("numeric");
b.Property<decimal?>("NetIncome")
.HasColumnType("numeric");
b.Property<decimal?>("OperatingCashFlow")
.HasColumnType("numeric");
b.Property<decimal?>("OperatingIncome")
.HasColumnType("numeric");
b.Property<decimal?>("PayoutRatio")
.HasColumnType("numeric");
b.Property<decimal?>("PegRatio")
.HasColumnType("numeric");
b.Property<decimal?>("PercentHeldByInsiders")
.HasColumnType("numeric");
b.Property<decimal?>("PercentHeldByInstitutions")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetHigh")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetLow")
.HasColumnType("numeric");
b.Property<decimal?>("PriceTargetMean")
.HasColumnType("numeric");
b.Property<decimal?>("PriceToBook")
.HasColumnType("numeric");
b.Property<decimal?>("PriceToSales")
.HasColumnType("numeric");
b.Property<decimal?>("ReturnOnAssets")
.HasColumnType("numeric");
b.Property<decimal?>("ReturnOnEquity")
.HasColumnType("numeric");
b.Property<decimal?>("RevenueGrowthYoY")
.HasColumnType("numeric");
b.Property<decimal?>("ShortPercentOfFloat")
.HasColumnType("numeric");
b.Property<decimal?>("ShortRatio")
.HasColumnType("numeric");
b.Property<decimal?>("TotalCash")
.HasColumnType("numeric");
b.Property<decimal?>("TotalDebt")
.HasColumnType("numeric");
b.Property<decimal?>("TotalRevenue")
.HasColumnType("numeric");
b.Property<decimal?>("TrailingPe")
.HasColumnType("numeric");
b.HasKey("Id");
b.HasIndex("AssetDataIsin");
b.ToTable("FundamentalData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Payment")
.IsRequired()
.HasColumnType("text");
b.Property<int>("SortOrder")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasDefaultValue(0);
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("AssetDataIsin", "SortOrder");
b.ToTable("KeyExecutives");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 =>
{
b1.Property<string>("AssetDataEntityIsin")
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("PrimaryTickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("PrimaryTicker");
b1.HasKey("AssetDataEntityIsin");
b1.ToTable("AssetData");
b1.WithOwner()
.HasForeignKey("AssetDataEntityIsin");
});
b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 =>
{
b1.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b1.Property<string>("AssetDataIsin")
.IsRequired()
.HasColumnType("text");
b1.Property<string>("Exchange")
.IsRequired()
.HasColumnType("text")
.HasColumnName("Exchange");
b1.Property<string>("Ticker")
.IsRequired()
.HasColumnType("text")
.HasColumnName("Ticker");
b1.HasKey("Id");
b1.HasIndex("AssetDataIsin");
b1.HasIndex("Ticker");
b1.ToTable("Tickers", (string)null);
b1.WithOwner()
.HasForeignKey("AssetDataIsin");
});
b.Navigation("AvailableTickers");
b.Navigation("PrimaryTicker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("AssetEvents")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<Guid>("AssetEventEntityId")
.HasColumnType("uuid");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("TickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("AssetEventEntityId");
b1.ToTable("AssetEvents");
b1.WithOwner()
.HasForeignKey("AssetEventEntityId");
});
b.Navigation("AssetData");
b.Navigation("Ticker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("FundamentalData")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<Guid>("FundamentalDataEntityId")
.HasColumnType("uuid");
b1.Property<string>("Exchange")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("TickerExchange");
b1.Property<string>("Ticker")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("text")
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("FundamentalDataEntityId");
b1.HasIndex("Ticker");
b1.ToTable("FundamentalData");
b1.WithOwner()
.HasForeignKey("FundamentalDataEntityId");
});
b.Navigation("AssetData");
b.Navigation("Ticker")
.IsRequired();
});
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
{
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
.WithMany("KeyExecutives")
.HasForeignKey("AssetDataIsin")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("AssetData");
});
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
{
b.Navigation("AssetEvents");
b.Navigation("FundamentalData");
b.Navigation("KeyExecutives");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,68 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticFundamentals.Migrations
{
/// <inheritdoc />
public partial class MakeFundamentalDataPerTicker : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData");
migrationBuilder.DropColumn(
name: "Isin",
table: "FundamentalData");
migrationBuilder.AddColumn<Guid>(
name: "Id",
table: "FundamentalData",
type: "uuid",
nullable: false,
defaultValueSql: "gen_random_uuid()");
migrationBuilder.AddPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData",
column: "Id");
migrationBuilder.CreateIndex(
name: "IX_FundamentalData_Ticker",
table: "FundamentalData",
column: "Ticker");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData");
migrationBuilder.DropIndex(
name: "IX_FundamentalData_Ticker",
table: "FundamentalData");
migrationBuilder.DropColumn(
name: "Id",
table: "FundamentalData");
migrationBuilder.AddColumn<string>(
name: "Isin",
table: "FundamentalData",
type: "text",
nullable: false,
defaultValue: "");
migrationBuilder.AddPrimaryKey(
name: "PK_FundamentalData",
table: "FundamentalData",
column: "Isin");
}
}
}
@@ -47,7 +47,8 @@ namespace FinlyticFundamentals.Migrations
b.HasKey("Id");
b.HasIndex("Key");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
@@ -96,8 +97,9 @@ namespace FinlyticFundamentals.Migrations
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
{
b.Property<string>("Isin")
.HasColumnType("text");
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("AssetDataIsin")
.IsRequired()
@@ -211,7 +213,7 @@ namespace FinlyticFundamentals.Migrations
b.Property<decimal?>("TrailingPe")
.HasColumnType("numeric");
b.HasKey("Isin");
b.HasKey("Id");
b.HasIndex("AssetDataIsin");
@@ -370,8 +372,8 @@ namespace FinlyticFundamentals.Migrations
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
{
b1.Property<string>("FundamentalDataEntityIsin")
.HasColumnType("text");
b1.Property<Guid>("FundamentalDataEntityId")
.HasColumnType("uuid");
b1.Property<string>("Exchange")
.IsRequired()
@@ -387,12 +389,14 @@ namespace FinlyticFundamentals.Migrations
.HasDefaultValue("")
.HasColumnName("Ticker");
b1.HasKey("FundamentalDataEntityIsin");
b1.HasKey("FundamentalDataEntityId");
b1.HasIndex("Ticker");
b1.ToTable("FundamentalData");
b1.WithOwner()
.HasForeignKey("FundamentalDataEntityIsin");
.HasForeignKey("FundamentalDataEntityId");
});
b.Navigation("AssetData");
+17 -11
View File
@@ -1,20 +1,26 @@
using FinlyticCore.Clients;
using FinlyticCore.Database;
using FinlyticCore.Services;
using FinlyticCore.Services.PlaywrightScrapper;
using FinlyticCore.Services.TradeRepublic;
using FinlyticCore.Services.Yahoo;
using Microsoft.EntityFrameworkCore;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Services;
using FinlyticFundamentals.Util;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
var builder = Host.CreateApplicationBuilder(args);
// Register DB Context
// Register DB Context & ISettingsDbContext
builder.Services.AddDbContext<FundamentalsDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection"))
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning)));
builder.Services.AddScoped<ISettingsDbContext>(sp => sp.GetRequiredService<FundamentalsDbContext>());
// Register HTTP Clients
builder.Services.AddHttpClient<IYahooFinanceScraper, YahooFinanceScraper>()
@@ -27,20 +33,21 @@ builder.Services.AddHttpClient<IYahooFinanceScraper, YahooFinanceScraper>()
builder.Services.AddSingleton<IPlaywrightBrowserFactory, PlaywrightBrowserFactory>();
builder.Services.AddSingleton<IPlaywrightExecutionService, PlaywrightExecutionService>();
builder.Services.AddTransient<IYahooFinanceHtmlClient, YahooFinanceHtmlClient<FundamentalsDbService, FundamentalsDbContext>>();
builder.Services.AddTransient<IYahooFinanceHtmlClient, YahooFinanceHtmlClient>();
// Register Application Services
builder.Services.AddSingleton<TradeRepublicClient>();
builder.Services.AddSingleton<ITradeRepublicService, TradeRepublicService>();
builder.Services.AddSingleton<FinlyticCore.Services.Yahoo.YahooFinanceClient>();
builder.Services.AddTransient<IFundamentalsDbService, FundamentalsDbService>();
builder.Services.AddScoped<TradeRepublicClient>();
builder.Services.AddScoped<ITradeRepublicService, TradeRepublicService>();
builder.Services.AddSingleton<YahooFinanceClient>();
builder.Services.AddScoped<IFundamentalsDbService, FundamentalsDbService>();
builder.Services.AddScoped<IYahooFinanceScraper, YahooFinanceScraper>();
builder.Services.AddScoped(typeof(ISettingsService<>), typeof(SettingsService<>));
builder.Services.AddScoped(typeof(IFinlyticLogger<,>), typeof(FinlyticLogger<,>));
builder.Services.AddSingleton<ISettingsService, SettingsService>();
builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<>));
// Register MQTT Client (as a Hosted Service)
builder.Services.AddHostedService<FundamentalsMqttClient>();
builder.Services.AddSingleton<FundamentalsMqttClient>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<FundamentalsMqttClient>());
var host = builder.Build();
@@ -52,7 +59,6 @@ using (var scope = host.Services.CreateScope())
var context = scope.ServiceProvider.GetRequiredService<FundamentalsDbContext>();
await context.Database.MigrateAsync();
Console.WriteLine("Database migrations successfully executed for FinlyticFundamentals.");
}
catch (Exception ex)
{
@@ -10,6 +10,7 @@ using FinlyticCore.Dtos.Yahoo;
using FinlyticCore.Models.Settings;
using FinlyticCore.Services;
using FinlyticCore.Services.TradeRepublic;
using FinlyticCore.Services.Yahoo;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Entities;
using FinlyticFundamentals.Util;
@@ -39,13 +40,13 @@ public class FundamentalsDbService : IFundamentalsDbService
private readonly IServiceScopeFactory _scopeFactory;
private readonly IYahooFinanceScraper _scraper;
private readonly ITradeRepublicService _tradeRepublicService;
private readonly IFinlyticLogger<FundamentalsDbService, FundamentalsDbContext> _finlyticLogger;
private readonly IFinlyticLogger<FundamentalsDbService> _finlyticLogger;
public FundamentalsDbService(
IServiceScopeFactory scopeFactory,
IYahooFinanceScraper scraper,
ITradeRepublicService tradeRepublicService,
IFinlyticLogger<FundamentalsDbService, FundamentalsDbContext> finlyticLogger)
IFinlyticLogger<FundamentalsDbService> finlyticLogger)
{
_scopeFactory = scopeFactory;
_scraper = scraper;
@@ -71,51 +72,60 @@ public class FundamentalsDbService : IFundamentalsDbService
{
using var scope = _scopeFactory.CreateScope();
var context = scope.ServiceProvider.GetRequiredService<FundamentalsDbContext>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService<FundamentalsDbContext>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
// 1. Dynamic Settings lesen
bool allowForceRefresh =
await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken);
bool enableHtmlFallback =
await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken);
bool forceHtmlFallback =
await settingsService.GetSettingAsync(SettingKeys.ForceHtmlFallback, cancellationToken);
int validityDays =
await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken);
bool effectiveForceRefresh = forceRefresh && allowForceRefresh;
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtmlFallback: {Html}",
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback);
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtml: {Html} | ForceHtml: {ForceHtml}",
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback, forceHtmlFallback);
// 2. Entitäten aus DB laden
var assetData = await context.AssetData
.Include(a => a.AvailableTickers)
.Include(a => a.KeyExecutives)
.Include(a => a.AssetEvents)
.Include(a => a.FundamentalData)
.FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken);
var fundamentalData = await context.FundamentalData
.FirstOrDefaultAsync(f => f.Isin == cleanIsin, cancellationToken);
string targetTicker = !string.IsNullOrWhiteSpace(requestedTicker)
? requestedTicker
: (assetData?.PrimaryTicker?.Ticker ?? string.Empty);
var fundamentalData = assetData?.FundamentalData?
.FirstOrDefault(f => !string.IsNullOrWhiteSpace(targetTicker) && string.Equals(f.Ticker.Ticker, targetTicker, StringComparison.OrdinalIgnoreCase))
?? (string.IsNullOrWhiteSpace(requestedTicker) ? assetData?.FundamentalData?.FirstOrDefault() : null);
// 3. Prüfen, was aktualisiert werden muss
bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name);
bool executivesMissing = assetData == null || assetData.KeyExecutives == null ||
assetData.KeyExecutives.Count == 0;
bool fundamentalsExpired = fundamentalData == null ||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
bool fundamentalsMissingOrExpired = fundamentalData == null ||
(fundamentalData.MarketCap == null && fundamentalData.TrailingPe == null) ||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
bool tickersCorruptOrMissing = assetData?.AvailableTickers == null ||
assetData.AvailableTickers.Count == 0 ||
assetData.AvailableTickers.Any(t => t.Ticker != null && t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase));
// Wenn ein expliziter Ticker übergeben wurde und sich vom gespeicherten unterscheidet,
// müssen Asset-Daten und Fundamentals mit dem neuen Ticker neu abgerufen werden.
bool tickerChanged = !string.IsNullOrWhiteSpace(requestedTicker)
&& assetData?.PrimaryTicker != null
&& !string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker,
StringComparison.OrdinalIgnoreCase);
bool shouldUpdate = assetDataMissing || executivesMissing || fundamentalsMissingOrExpired || tickersCorruptOrMissing || effectiveForceRefresh || forceHtmlFallback;
bool shouldUpdateAssetData = assetDataMissing || effectiveForceRefresh || tickerChanged;
bool shouldUpdateExecutives = executivesMissing || effectiveForceRefresh;
bool shouldUpdateFundamentals = fundamentalsExpired || effectiveForceRefresh || tickerChanged;
if (shouldUpdateAssetData || shouldUpdateExecutives || shouldUpdateFundamentals)
if (!shouldUpdate && fundamentalData != null)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[FundamentalsDbService] Returning valid cached fundamental data for ISIN {Isin} (Ticker: {Ticker}, Updated: {UpdatedUtc}). External API fetch skipped.",
cleanIsin, fundamentalData.Ticker.Ticker, fundamentalData.LastUpdatedUtc.ToString("o"));
}
else
{
// --- STEP 1: Trade Republic Details ---
TradeRepublicStockDetailsResponse? trDetails = null;
@@ -149,8 +159,10 @@ public class FundamentalsDbService : IFundamentalsDbService
TickerInfoDto activeQueryTicker;
if (!string.IsNullOrWhiteSpace(requestedTicker))
{
var matchDto = resolvedTickers.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchDto = resolvedTickers.FirstOrDefault(t =>
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(t =>
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
if (matchDto != null)
{
@@ -196,12 +208,18 @@ public class FundamentalsDbService : IFundamentalsDbService
yahooPrimaryTicker.Ticker, activeQueryTicker.Ticker, cleanIsin);
// --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper ---
// Profile (Sektor, Industrie, Vorstände) wird gescrapt, wenn weder in DB noch in TR Vorstände/Beschreibungen vorliegen
bool hasProfileInDb = assetData != null && !string.IsNullOrWhiteSpace(assetData.Description) && assetData.KeyExecutives != null && assetData.KeyExecutives.Count > 0;
bool hasCeoInTr = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
bool needProfile = !hasProfileInDb && !hasCeoInTr;
YahooQuoteSummaryModulesDto? modulesDto = null;
if (!string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) && activeQueryTicker.Ticker != cleanIsin)
{
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
activeQueryTicker.Ticker,
forceHtmlScrape: false,
forceHtmlScrape: forceHtmlFallback,
includeProfile: needProfile,
cancellationToken: cancellationToken);
}
else
@@ -210,51 +228,96 @@ public class FundamentalsDbService : IFundamentalsDbService
"[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", activeQueryTicker.Ticker);
}
// --- Update AssetDataEntity ---
if (shouldUpdateAssetData)
// Falls der Sekundär-Ticker (z. B. APC.DE) überhaupt keine Daten liefert, nutze den PrimaryTicker (z. B. AAPL) als Fallback
if (modulesDto == null &&
!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) &&
yahooPrimaryTicker.Ticker != activeQueryTicker.Ticker &&
yahooPrimaryTicker.Ticker != cleanIsin)
{
if (assetData == null)
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-FALLBACK-PRIMARY] Sekundär-Ticker '{Active}' lieferte keine Daten. Versuche PrimaryTicker '{Primary}'...",
activeQueryTicker.Ticker, yahooPrimaryTicker.Ticker);
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
yahooPrimaryTicker.Ticker,
forceHtmlScrape: forceHtmlFallback,
includeProfile: needProfile,
cancellationToken: cancellationToken);
}
// --- Update AssetDataEntity ---
if (assetData == null)
{
assetData = new AssetDataEntity
{
assetData = new AssetDataEntity
Isin = cleanIsin,
PrimaryTicker = new TickerEntity
{
Isin = cleanIsin,
PrimaryTicker = new TickerEntity
{
Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
},
KeyExecutives = new List<KeyExecutiveEntity>(),
AssetEvents = new List<AssetEventEntity>()
};
context.AssetData.Add(assetData);
}
Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
},
KeyExecutives = new List<KeyExecutiveEntity>(),
AssetEvents = new List<AssetEventEntity>()
};
context.AssetData.Add(assetData);
}
string trName = trDetails?.Company?.Name ?? string.Empty;
string trDescription = trDetails?.Company?.Description ?? string.Empty;
string trName = trDetails?.Company?.Name?.Trim() ?? string.Empty;
string trDescription = trDetails?.Company?.Description?.Trim() ?? string.Empty;
string fallbackName = modulesDto?.QuoteType?.ShortName
?? modulesDto?.QuoteType?.LongName
?? activeQueryTicker.Ticker;
string yahooName = modulesDto?.QuoteType?.LongName?.Trim()
?? modulesDto?.QuoteType?.ShortName?.Trim()
?? string.Empty;
string yahooDesc = modulesDto?.AssetProfile?.LongBusinessSummary?.Trim() ?? string.Empty;
assetData.Name = !string.IsNullOrWhiteSpace(trName) ? trName : fallbackName;
assetData.Description = !string.IsNullOrWhiteSpace(trDescription)
? trDescription
: (modulesDto?.AssetProfile?.LongBusinessSummary ?? string.Empty);
// Name nur aktualisieren, wenn ein echter Name vorliegt (Bestandsdaten niemals mit ISIN/Ticker überschreiben)
if (!string.IsNullOrWhiteSpace(trName))
{
assetData.Name = trName;
}
else if (!string.IsNullOrWhiteSpace(yahooName))
{
assetData.Name = yahooName;
}
else if (string.IsNullOrWhiteSpace(assetData.Name))
{
assetData.Name = !string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) ? activeQueryTicker.Ticker : cleanIsin;
}
// PrimaryTicker ist FEST der erste von Yahoo Finance
// Description nur aktualisieren, wenn neue Beschreibung vorhanden ist
if (!string.IsNullOrWhiteSpace(trDescription))
{
assetData.Description = trDescription;
}
else if (!string.IsNullOrWhiteSpace(yahooDesc))
{
assetData.Description = yahooDesc;
}
// PrimaryTicker aktualisieren falls vorhanden
if (!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) && yahooPrimaryTicker.Ticker != cleanIsin)
{
assetData.PrimaryTicker = new TickerEntity
{
Ticker = yahooPrimaryTicker.Ticker,
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
};
}
if (!resolvedTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
// AvailableTickers aktualisieren (nur echte Börsenticker, keine ISINs)
var validTickers = resolvedTickers
.Where(t => !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase))
.ToList();
if (validTickers.Count > 0)
{
if (!validTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
{
resolvedTickers.Insert(0, yahooPrimaryTicker);
validTickers.Insert(0, yahooPrimaryTicker);
}
assetData.AvailableTickers.Clear();
foreach (var a in resolvedTickers)
foreach (var a in validTickers)
{
assetData.AvailableTickers.Add(new TickerEntity
{
@@ -262,21 +325,19 @@ public class FundamentalsDbService : IFundamentalsDbService
Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker)
});
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}'",
assetData.Name, assetData.PrimaryTicker.Ticker);
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}' | AvailableTickers: {Count}",
assetData.Name, assetData.PrimaryTicker?.Ticker ?? "NULL", assetData.AvailableTickers.Count);
// --- Process Trade Republic Corporate Events ---
if (trDetails != null && (shouldUpdateAssetData || effectiveForceRefresh) && assetData != null)
if (trDetails != null && assetData != null)
{
// 1. Alte Events direkt in der DB löschen (bypasses Change Tracker)
await context.AssetEvents
.Where(e => e.AssetDataIsin == cleanIsin)
.ExecuteDeleteAsync(cancellationToken);
// 2. ALLE tracked AssetEventEntity-Einträge aus dem Change Tracker entfernen
foreach (var entry in context.ChangeTracker.Entries<AssetEventEntity>()
.Where(e => e.Entity.AssetDataIsin == cleanIsin)
.ToList())
@@ -284,7 +345,6 @@ public class FundamentalsDbService : IFundamentalsDbService
entry.State = EntityState.Detached;
}
// 3. Navigation-Collection zurücksetzen
assetData.AssetEvents = new List<AssetEventEntity>();
var trEventList = new List<TradeRepublicEventDto>();
@@ -323,18 +383,18 @@ public class FundamentalsDbService : IFundamentalsDbService
}
// --- Process Modules DTO (Executives & Fundamental Data) ---
if (modulesDto != null)
if (modulesDto != null || trDetails?.Company != null)
{
// Update KeyExecutives
if (shouldUpdateExecutives && assetData != null)
// Update KeyExecutives wenn Executives aus TR oder Yahoo vorliegen
var yahooOfficers = modulesDto?.AssetProfile?.CompanyOfficers;
bool hasTrOfficers = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
if (((yahooOfficers != null && yahooOfficers.Count > 0) || hasTrOfficers) && assetData != null)
{
// 1. Alte Executives direkt in der DB löschen (bypasses Change Tracker)
await context.KeyExecutives
.Where(e => e.AssetDataIsin == cleanIsin)
.ExecuteDeleteAsync(cancellationToken);
// 2. ALLE tracked KeyExecutiveEntity-Einträge aus dem Change Tracker entfernen
// (nicht nur die in der Navigation-Collection — der Tracker kann mehr halten)
foreach (var entry in context.ChangeTracker.Entries<KeyExecutiveEntity>()
.Where(e => e.Entity.AssetDataIsin == cleanIsin)
.ToList())
@@ -342,14 +402,12 @@ public class FundamentalsDbService : IFundamentalsDbService
entry.State = EntityState.Detached;
}
// 3. Navigation-Collection zurücksetzen
assetData.KeyExecutives = new List<KeyExecutiveEntity>();
// 4. Neue Executives aufbauen und direkt über den DbSet hinzufügen
if (modulesDto.AssetProfile?.CompanyOfficers != null)
if (yahooOfficers != null && yahooOfficers.Count > 0)
{
int sortIdx = 0;
foreach (var officer in modulesDto.AssetProfile.CompanyOfficers)
foreach (var officer in yahooOfficers)
{
if (!string.IsNullOrWhiteSpace(officer.Name))
{
@@ -367,6 +425,49 @@ public class FundamentalsDbService : IFundamentalsDbService
}
}
}
else if (hasTrOfficers && trDetails?.Company != null)
{
int sortIdx = 0;
if (!string.IsNullOrWhiteSpace(trDetails.Company.CeoName))
{
var ceo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CeoName,
Title = "CEO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(ceo);
assetData.KeyExecutives.Add(ceo);
}
if (!string.IsNullOrWhiteSpace(trDetails.Company.CfoName))
{
var cfo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CfoName,
Title = "CFO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(cfo);
assetData.KeyExecutives.Add(cfo);
}
if (!string.IsNullOrWhiteSpace(trDetails.Company.CooName))
{
var coo = new KeyExecutiveEntity
{
AssetDataIsin = cleanIsin,
Name = trDetails.Company.CooName,
Title = "COO",
Payment = string.Empty,
SortOrder = sortIdx++
};
context.KeyExecutives.Add(coo);
assetData.KeyExecutives.Add(coo);
}
}
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
"[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.",
@@ -374,16 +475,23 @@ public class FundamentalsDbService : IFundamentalsDbService
}
// Update FundamentalDataEntity
if (shouldUpdateFundamentals)
if (modulesDto != null && (modulesDto.SummaryDetail != null || modulesDto.DefaultKeyStatistics != null || modulesDto.FinancialData != null))
{
if (fundamentalData == null || !string.Equals(fundamentalData.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase))
{
fundamentalData = assetData?.FundamentalData?
.FirstOrDefault(f => string.Equals(f.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase));
}
if (fundamentalData == null)
{
fundamentalData = new FundamentalDataEntity
{
Isin = cleanIsin,
Id = Guid.NewGuid(),
AssetDataIsin = cleanIsin
};
context.FundamentalData.Add(fundamentalData);
assetData?.FundamentalData.Add(fundamentalData);
}
fundamentalData.Ticker = new TickerEntity
@@ -541,7 +649,7 @@ public class FundamentalsDbService : IFundamentalsDbService
: (assetData.PrimaryTicker != null ? new List<TickerEntity> { assetData.PrimaryTicker } : new List<TickerEntity>());
var tickerDtos = tickerEntities
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker))
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(assetData.Isin, StringComparison.OrdinalIgnoreCase))
.Select(a => new TickerInfoDto
{
Ticker = a.Ticker,
@@ -1,12 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Models;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticFundamentals.Database;
using FinlyticFundamentals.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
@@ -60,7 +61,17 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
await SubscribeAsync("services/request/fundamentals_Get/#");
await SubscribeAsync("services/request/events_GetAll/#");
await SubscribeAsync("services/request/events_GetByMonth/#");
await SubscribeAsync("services/request/fundamentals_settings_GetAll/#");
await SubscribeAsync("services/request/fundamentals_settings_Update/#");
await SubscribeAsync("services/request/health_Ping/#");
FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
{
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticFundamentals", StringComparison.OrdinalIgnoreCase))
{
await PublishAsync("finlytic/logs/FinlyticFundamentals", logDto);
}
};
}
/// <inheritdoc />
@@ -85,6 +96,14 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
{
await OnEventsGetByMonthAsync(payload, correlationId);
}
else if (topic.StartsWith("services/request/fundamentals_settings_GetAll", StringComparison.OrdinalIgnoreCase))
{
await OnSettingsGetAllAsync(correlationId);
}
else if (topic.StartsWith("services/request/fundamentals_settings_Update", StringComparison.OrdinalIgnoreCase))
{
await OnSettingsUpdateAsync(payload, correlationId);
}
else if (topic.StartsWith("services/request/health_Ping", StringComparison.OrdinalIgnoreCase))
{
await OnHealthPingAsync(topic, correlationId);
@@ -93,8 +112,8 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
private async Task OnFundamentalsGetAsync(string payload, string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
await using var scope = _scopeFactory.CreateAsyncScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient>>();
var dbService = scope.ServiceProvider.GetRequiredService<IFundamentalsDbService>();
if (string.IsNullOrWhiteSpace(payload))
@@ -129,8 +148,8 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
private async Task OnEventsGetAllAsync(string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
await using var scope = _scopeFactory.CreateAsyncScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient>>();
var dbService = scope.ServiceProvider.GetRequiredService<IFundamentalsDbService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC events_GetAll request [CorrelationId: {CorrelationId}]", correlationId);
@@ -152,8 +171,8 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
await using var scope = _scopeFactory.CreateAsyncScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient>>();
var dbService = scope.ServiceProvider.GetRequiredService<IFundamentalsDbService>();
try
@@ -175,12 +194,78 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService
}
}
private async Task OnSettingsGetAllAsync(string correlationId)
{
await using var scope = _scopeFactory.CreateAsyncScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
try
{
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/fundamentals_settings_GetAll/{correlationId}";
await PublishAsync(responseTopic, settings);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [Settings_GetAll] Failed to retrieve settings.");
}
}
private async Task OnSettingsUpdateAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
await using var scope = _scopeFactory.CreateAsyncScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
try
{
Dictionary<string, object?>? updates = null;
try
{
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
}
catch
{
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
if (list != null)
{
updates = new Dictionary<string, object?>();
foreach (var item in list)
{
updates[item.Key] = item.Value;
}
}
}
if (updates != null && updates.Count > 0)
{
await settingsService.UpdateSettingsAsync(updates);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
}
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/fundamentals_settings_Update/{correlationId}";
await PublishAsync(responseTopic, currentSettings);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [Settings_Update] Failed to update settings.");
}
}
private async Task OnHealthPingAsync(string topic, string correlationId)
{
if (topic.Contains("FinlyticFundamentals", StringComparison.OrdinalIgnoreCase) || !topic.Contains("/", StringComparison.OrdinalIgnoreCase))
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient, FundamentalsDbContext>>();
await using var scope = _scopeFactory.CreateAsyncScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<FundamentalsMqttClient>>();
var respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticFundamentals", "Online", DateTime.UtcNow, "Connected"));
+1
View File
@@ -13,6 +13,7 @@ public class SettingKeys
// --- Features & Toggles ---
public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true);
public static readonly SettingKey<bool> ForceHtmlFallback = new("Scraper.ForceHtmlFallback", false);
public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true);
public static readonly SettingKey<int> FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30);
}
+25 -1
View File
@@ -1,5 +1,8 @@
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using FinlyticNews.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticNews.Database;
@@ -7,7 +10,7 @@ namespace FinlyticNews.Database;
/// Entity Framework Core database context for the news microservice,
/// managing article sources, processed news, and matched assets.
/// </summary>
public class NewsDbContext : DbContext
public class NewsDbContext : DbContext, ISettingsDbContext
{
/// <summary>
/// Initializes a new instance of the <see cref="NewsDbContext"/> class.
@@ -17,6 +20,11 @@ public class NewsDbContext : DbContext
{
}
/// <summary>
/// Gets or sets the database set for dynamic settings.
/// </summary>
public DbSet<SettingEntity> DynamicSettings => Set<SettingEntity>();
/// <summary>
/// Gets or sets the database set for configured article sources.
/// </summary>
@@ -45,6 +53,12 @@ public class NewsDbContext : DbContext
{
base.OnModelCreating(modelBuilder);
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key).IsUnique();
});
// Configure unique index on SourceUrl for deduplication check (idempotency)
modelBuilder.Entity<NewsArticleEntity>()
.HasIndex(a => a.SourceUrl)
@@ -68,3 +82,13 @@ public class NewsDbContext : DbContext
.OnDelete(DeleteBehavior.Cascade);
}
}
public class NewsDbContextFactory : IDesignTimeDbContextFactory<NewsDbContext>
{
public NewsDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<NewsDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=news;Username=postgres;Password=postgres");
return new NewsDbContext(optionsBuilder.Options);
}
}
@@ -0,0 +1,205 @@
// <auto-generated />
using System;
using FinlyticNews.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticNews.Migrations
{
[DbContext(typeof(NewsDbContext))]
[Migration("20260815183946_AddDynamicSettings")]
partial class AddDynamicSettings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticNews.Entities.ArticleSourceEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("ArticleSources");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Isin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("NewsArticleId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("Isin");
b.HasIndex("NewsArticleId");
b.ToTable("MatchedAssets");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsArticleEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Author")
.HasColumnType("text");
b.Property<string>("ContentRaw")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Language")
.HasColumnType("text");
b.Property<DateTime>("PublishedAt")
.HasColumnType("timestamp with time zone");
b.Property<DateTime>("ScrapedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("SourceUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Status")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Summary")
.HasColumnType("text");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("SourceUrl")
.IsUnique();
b.HasIndex("Status");
b.HasIndex("PublishedAt", "ScrapedAt");
b.ToTable("NewsArticles");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("ArticleRetentionDays")
.HasColumnType("integer");
b.Property<int>("DefaultPageSize")
.HasColumnType("integer");
b.Property<string>("N8nWebhookUrl")
.IsRequired()
.HasColumnType("text");
b.Property<int>("PollingFrequencyMinutes")
.HasColumnType("integer");
b.Property<int>("ScrapingIntervalMinutes")
.HasColumnType("integer");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.HasOne("FinlyticNews.Entities.NewsArticleEntity", "NewsArticle")
.WithMany("MatchedAssets")
.HasForeignKey("NewsArticleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("NewsArticle");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsArticleEntity", b =>
{
b.Navigation("MatchedAssets");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,43 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticNews.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DynamicSettings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
ValueJson = table.Column<string>(type: "text", nullable: false),
ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DynamicSettings", x => x.Id);
});
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DynamicSettings");
}
}
}
@@ -22,6 +22,37 @@ namespace FinlyticNews.Migrations
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticNews.Entities.ArticleSourceEntity", b =>
{
b.Property<Guid>("Id")
+7
View File
@@ -1,3 +1,5 @@
using FinlyticCore.Database;
using FinlyticCore.Services;
using FinlyticNews.Database;
using FinlyticNews.Services;
using FinlyticNews.Util;
@@ -10,10 +12,15 @@ var builder = Host.CreateApplicationBuilder(args);
builder.Services.AddDbContext<NewsDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddScoped<ISettingsDbContext>(sp => sp.GetRequiredService<NewsDbContext>());
// Register standard HttpClient
builder.Services.AddHttpClient();
// Register Core Services
builder.Services.AddSingleton<ISettingsService, SettingsService>();
builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<>));
// Register Discovery Adapters
builder.Services.AddSingleton<ArticleDiscoveryAdapter, RssDiscoveryAdapter>();
builder.Services.AddSingleton<ArticleDiscoveryAdapter, FinanznachrichtenRssDiscoveryAdapter>();
@@ -1,21 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Services;
using FinlyticNews.Adapters.Discovery;
using Microsoft.Extensions.Logging;
using FinlyticNews.Util;
namespace FinlyticNews.Services;
/// <summary>
/// Defines operations for discovering article links from various news feeds and web pages.
/// </summary>
public interface IArticleDiscoveryService
{
/// <summary>
/// Discovers article links from a given source URL using configured adapters.
/// </summary>
/// <param name="url">The URL of the source news page or feed.</param>
/// <param name="adapterType">The type identifier of the adapter to use (e.g. "rss", "html").</param>
/// <param name="ct">The token to monitor for cancellation requests.</param>
/// <returns>A list of discovered absolute article URLs (optionally carrying ISINs), or null if the URL was invalid.</returns>
Task<List<DiscoveredArticle>?> DiscoverLinksAsync(string url, string adapterType, CancellationToken ct = default);
}
@@ -23,22 +20,16 @@ public interface IArticleDiscoveryService
public class ArticleDiscoveryService : IArticleDiscoveryService
{
private readonly IEnumerable<ArticleDiscoveryAdapter> _adapters;
private readonly ILogger<ArticleDiscoveryService> _logger;
private readonly IFinlyticLogger<ArticleDiscoveryService> _finlyticLogger;
private readonly HttpClient _httpClient;
/// <summary>
/// Initializes a new instance of the <see cref="ArticleDiscoveryService"/> class.
/// </summary>
/// <param name="adapters">Registered list of specialized adapters.</param>
/// <param name="logger">The application logging channel.</param>
/// <param name="httpClient">The HTTP client to fetch feeds.</param>
public ArticleDiscoveryService(
IEnumerable<ArticleDiscoveryAdapter> adapters,
ILogger<ArticleDiscoveryService> logger,
IFinlyticLogger<ArticleDiscoveryService> finlyticLogger,
HttpClient httpClient)
{
_adapters = adapters;
_logger = logger;
_finlyticLogger = finlyticLogger;
_httpClient = httpClient;
}
@@ -47,37 +38,35 @@ public class ArticleDiscoveryService : IArticleDiscoveryService
{
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
_logger.LogWarning("[{Channel}] Invalid source URL passed for discovery: {Url}", "NewsChannel", url);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[ArticleDiscoveryService] Invalid source URL passed for discovery: {Url}", url);
return null;
}
try
{
_logger.LogDebug("Fetching content from discovery source: {Url}", uri);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[ArticleDiscoveryService] Fetching content from discovery source: {Url}", uri);
var content = await _httpClient.GetStringAsync(uri, ct);
if (string.IsNullOrWhiteSpace(content)) return [];
var trimmedContent = content.TrimStart();
// 1. Zuerst gezielt nach registriertem Adapter suchen (z. B. finanznachrichten_rss)
var adapter = _adapters.FirstOrDefault(a =>
a.Name.Equals(adapterType, StringComparison.OrdinalIgnoreCase) ||
uri.Host.Contains(a.Name, StringComparison.OrdinalIgnoreCase));
if (adapter != null)
{
_logger.LogInformation("[{Channel}] Using specialized adapter {AdapterName} for source: {Url}", "NewsChannel", adapter.Name, url);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[ArticleDiscoveryService] Using specialized adapter {AdapterName} for source: {Url}", adapter.Name, url);
return adapter.ExtractUrls(content, url);
}
// 2. Fallback: Automatische Erkennung für generische RSS/Atom-Feeds
if (adapterType.Equals("rss", StringComparison.OrdinalIgnoreCase) ||
trimmedContent.StartsWith("<?xml", StringComparison.OrdinalIgnoreCase) ||
trimmedContent.StartsWith("<rss", StringComparison.OrdinalIgnoreCase) ||
trimmedContent.StartsWith("<feed", StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation("[{Channel}] Syndication (RSS) format detected for source: {Url}", "NewsChannel", url);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[ArticleDiscoveryService] Syndication (RSS) format detected for source: {Url}", url);
var rssAdapter = _adapters.FirstOrDefault(a => a.Name.Equals("rss", StringComparison.OrdinalIgnoreCase))
?? new RssDiscoveryAdapter();
@@ -85,12 +74,12 @@ public class ArticleDiscoveryService : IArticleDiscoveryService
return rssAdapter.ExtractUrls(content, url);
}
_logger.LogWarning("[{Channel}] No suitable discovery adapter found for type '{Type}' and URL: {Url}", "NewsChannel", adapterType, url);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[ArticleDiscoveryService] No suitable discovery adapter found for type '{Type}' and URL: {Url}", adapterType, url);
return [];
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to run article discovery on URL: {Url}", "NewsChannel", url);
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[ArticleDiscoveryService] Failed to run article discovery on URL: {Url}", url);
return [];
}
}
+16 -20
View File
@@ -1,14 +1,18 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticNews.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FinlyticNews.Services;
using FinlyticCore.Dtos.News;
/// <summary>
/// Defines integration operations with the external n8n AI workflow webhook.
/// </summary>
@@ -26,21 +30,18 @@ public class N8nService : IN8nService
private readonly HttpClient _httpClient;
private readonly IServiceScopeFactory _scopeFactory;
private readonly IConfiguration _configuration;
private readonly ILogger<N8nService> _logger;
private readonly IFinlyticLogger<N8nService> _finlyticLogger;
/// <summary>
/// Initializes a new instance of the <see cref="N8nService"/> class.
/// </summary>
public N8nService(
HttpClient httpClient,
IServiceScopeFactory scopeFactory,
IConfiguration configuration,
ILogger<N8nService> logger)
IFinlyticLogger<N8nService> finlyticLogger)
{
_httpClient = httpClient;
_scopeFactory = scopeFactory;
_configuration = configuration;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <inheritdoc />
@@ -48,7 +49,6 @@ public class N8nService : IN8nService
{
string? targetUrl = null;
// 1. Dynamic Settings Resolution (DB Scope -> AppSettings Fallback)
using (var scope = _scopeFactory.CreateScope())
{
var settingsDb = scope.ServiceProvider.GetService<ISettingsDbService>();
@@ -67,17 +67,16 @@ public class N8nService : IN8nService
if (string.IsNullOrWhiteSpace(targetUrl))
{
_logger.LogError("[{Channel}] N8nWebhookUrl is not configured in DB or application settings.", "NewsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, "[N8nService] N8nWebhookUrl is not configured in DB or application settings.");
return null;
}
_logger.LogInformation("[{Channel}] Posting article to n8n webhook pipeline at: {Url}", "NewsChannel", targetUrl);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[N8nService] Posting article to n8n webhook pipeline at: {Url}", targetUrl);
var payload = new N8nRequestPayload(content, filteredAssets ?? []);
try
{
// Zero-Allocation / Source-Generated Request Serialization
var jsonContent = JsonSerializer.Serialize(payload, FinlyticJsonSerializerContext.Default.N8nRequestPayload);
using var requestContent = new StringContent(jsonContent, System.Text.Encoding.UTF8, "application/json");
@@ -86,7 +85,7 @@ public class N8nService : IN8nService
if (!response.IsSuccessStatusCode)
{
var errorMsg = await response.Content.ReadAsStringAsync(ct);
_logger.LogError("[{Channel}] n8n webhook returned status code {StatusCode}. Error payload: {Error}", "NewsChannel", response.StatusCode, errorMsg);
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, "[N8nService] n8n webhook returned status code {StatusCode}. Error payload: {Error}", response.StatusCode, errorMsg);
return null;
}
@@ -95,18 +94,16 @@ public class N8nService : IN8nService
var root = doc.RootElement;
// 2. Robust n8n Array-Unwrapping (Handles [{ "json": { ... } }])
if (root.ValueKind == JsonValueKind.Array)
{
if (root.GetArrayLength() == 0)
{
_logger.LogWarning("[{Channel}] n8n webhook returned an empty array.", "NewsChannel");
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[N8nService] n8n webhook returned an empty array.");
return null;
}
root = root[0];
}
// 3. Dynamic Node Wrapper Unwrapping ("json", "output", "data", "body")
if (root.ValueKind == JsonValueKind.Object)
{
if (root.TryGetProperty("json", out var jsonChild) && jsonChild.ValueKind == JsonValueKind.Object)
@@ -119,13 +116,12 @@ public class N8nService : IN8nService
root = bodyChild;
}
// 4. Source-Generated Deserialization directly from JsonElement
var result = root.Deserialize(FinlyticJsonSerializerContext.Default.N8nResponsePayload);
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to communicate with or parse response from n8n webhook workflow.", "NewsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[N8nService] Failed to communicate with or parse response from n8n webhook workflow.");
return null;
}
}
+48 -116
View File
@@ -1,6 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Services;
using FinlyticNews.Database;
using FinlyticNews.Entities;
using FinlyticNews.Util;
using Microsoft.EntityFrameworkCore;
namespace FinlyticNews.Services;
@@ -39,22 +45,11 @@ public interface INewsDbService
/// </summary>
Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets);
/// <summary>
/// Retrieves articles ready for historical sync or sentiment processing.
/// </summary>
Task<List<NewsArticleEntity>> GetCompletedArticlesAsync(int limit, int offset, string? isin = null);
Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(string status);
/// <summary>
/// Fetches articles matching specific lifecycle statuses (e.g. "Pending", "Scraping" for Phase 1 retry).
/// </summary>
Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(params string[] statuses);
/// <summary>
/// Fetches public daily news for API endpoints, filtering out intermediate or failed lifecycle states by default.
/// </summary>
Task<List<NewsArticleEntity>> GetFilteredNewsAsync(
int limit = 20,
int offset = 0,
int limit,
int offset,
string? isin = null,
DateTime? date = null,
string? status = null,
@@ -67,15 +62,12 @@ public interface INewsDbService
public class NewsDbService : INewsDbService
{
private readonly NewsDbContext _context;
private readonly ILogger<NewsDbService> _logger;
private readonly IFinlyticLogger<NewsDbService> _finlyticLogger;
/// <summary>
/// Initializes a new instance of the <see cref="NewsDbService"/> class.
/// </summary>
public NewsDbService(NewsDbContext context, ILogger<NewsDbService> logger)
public NewsDbService(NewsDbContext context, IFinlyticLogger<NewsDbService> finlyticLogger)
{
_context = context;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <inheritdoc />
@@ -103,9 +95,6 @@ public class NewsDbService : INewsDbService
}
/// <inheritdoc />
/// <summary>
/// Lifecycle Step 1: Creates a new article in 'Pending' state as an immediate lock.
/// </summary>
public async Task<NewsArticleEntity> CreatePendingArticleAsync(
string url,
List<string>? discoveredIsins = null,
@@ -121,7 +110,7 @@ public class NewsDbService : INewsDbService
if (existingArticle != null)
{
_logger.LogDebug("[Lifecycle] Article URL already exists (Duplicate hit): {Url}", trimmedUrl);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Article URL already exists (Duplicate hit): {Url}", trimmedUrl);
return existingArticle;
}
@@ -146,7 +135,7 @@ public class NewsDbService : INewsDbService
Language = language,
ScrapedAt = DateTime.UtcNow,
PublishedAt = finalPublishedAt,
Status = "Pending" // 1. Pending State
Status = "Pending"
};
if (discoveredIsins != null && discoveredIsins.Count > 0)
@@ -167,11 +156,11 @@ public class NewsDbService : INewsDbService
{
_context.NewsArticles.Add(article);
await _context.SaveChangesAsync();
_logger.LogDebug("[Lifecycle] Registered new article with status 'Pending'. ID: {Id}, Url: {Url}", article.Id, trimmedUrl);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Registered new article with status 'Pending'. ID: {Id}, Url: {Url}", article.Id, trimmedUrl);
}
catch (DbUpdateConcurrencyException)
{
_logger.LogWarning("[{Channel}] Concurrency hit during insert for URL: {Url}. Fetching existing fallback.", "NewsChannel", trimmedUrl);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Concurrency hit during insert for URL: {Url}. Fetching existing fallback.", trimmedUrl);
return await _context.NewsArticles.FirstAsync(a => a.SourceUrl == trimmedUrl);
}
@@ -179,9 +168,6 @@ public class NewsDbService : INewsDbService
}
/// <inheritdoc />
/// <summary>
/// Lifecycle Step 2 & 5: Updates state (e.g. Pending -> Processing -> Scraping / Failed / Analyzed).
/// </summary>
public async Task UpdateArticleStatusAsync(Guid id, string status)
{
var rowsAffected = await _context.NewsArticles
@@ -190,11 +176,11 @@ public class NewsDbService : INewsDbService
if (rowsAffected == 0)
{
_logger.LogWarning("[{Channel}] Attempted status transition for non-existing article. ID: {Id}", "NewsChannel", id);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Attempted status transition for non-existing article. ID: {Id}", id);
}
else
{
_logger.LogDebug("[Lifecycle] Transitioned article {Id} to status '{Status}'", id, status);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Transitioned article {Id} to status '{Status}'", id, status);
}
}
@@ -207,18 +193,15 @@ public class NewsDbService : INewsDbService
if (rowsAffected == 0)
{
_logger.LogWarning("[{Channel}] Attempted URL update for non-existing article. ID: {Id}", "NewsChannel", id);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Attempted URL update for non-existing article. ID: {Id}", id);
}
else
{
_logger.LogDebug("[Lifecycle] Resolved redirect for article {Id} -> New URL: {Url}", id, resolvedUrl);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Resolved redirect for article {Id} -> New URL: {Url}", id, resolvedUrl);
}
}
/// <inheritdoc />
/// <summary>
/// Lifecycle Step 4: Persists n8n classification and transitions status to 'Completed'.
/// </summary>
public async Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets)
{
var article = await _context.NewsArticles
@@ -226,7 +209,7 @@ public class NewsDbService : INewsDbService
if (article == null)
{
_logger.LogWarning("[{Channel}] Article with ID {Id} not found for classification update.", "NewsChannel", id);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Article with ID {Id} not found for classification update.", id);
return null;
}
@@ -235,8 +218,6 @@ public class NewsDbService : INewsDbService
article.Summary = payload.Summary;
article.ContentRaw = payload.ContentRaw;
article.Language = payload.Language;
// 🎯 Step 4: Classification finished -> Transition to 'Completed' (triggers MQTT broadcast)
article.Status = "Completed";
if (DateTime.TryParse(payload.PublishedAt, out var publishedDate))
@@ -251,7 +232,6 @@ public class NewsDbService : INewsDbService
article.ScrapedAt = scrapedDate.ToUniversalTime();
}
// Clean up previous temporary assets
await _context.MatchedAssets.Where(m => m.NewsArticleId == id).ExecuteDeleteAsync();
article.MatchedAssets = new List<MatchedAssetEntity>();
@@ -267,113 +247,65 @@ public class NewsDbService : INewsDbService
}
await _context.SaveChangesAsync();
_logger.LogInformation("[Lifecycle] Article {Id} successfully classified and marked 'Completed'. Title: '{Title}'", article.Id, article.Title);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[Lifecycle] Article {Id} successfully classified and marked 'Completed'. Title: '{Title}'", article.Id, article.Title);
return article;
}
/// <inheritdoc />
public async Task<List<NewsArticleEntity>> GetCompletedArticlesAsync(int limit, int offset, string? isin = null)
public async Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(string status)
{
var query = _context.NewsArticles
return await _context.NewsArticles
.Include(a => a.MatchedAssets)
.Where(a => a.Status == "Completed" || a.Status == "Analyzed");
if (!string.IsNullOrWhiteSpace(isin))
{
var cleanIsin = isin.Trim();
query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin || m.Name == cleanIsin));
}
return await query
.Where(a => a.Status == status)
.OrderByDescending(a => a.PublishedAt)
.ThenByDescending(a => a.ScrapedAt)
.ThenByDescending(a => a.Id)
.Skip(offset)
.Take(limit)
.AsNoTracking()
.ToListAsync();
}
/// <inheritdoc />
/// <summary>
/// Lifecycle Helper: Retrieves articles by target lifecycle status (e.g. "Pending", "Scraping" for Phase 1 Retry).
/// </summary>
public async Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(params string[] statuses)
{
IQueryable<NewsArticleEntity> query = _context.NewsArticles
.Include(a => a.MatchedAssets)
.AsNoTracking();
if (statuses != null && statuses.Length > 0)
{
var cleanStatuses = statuses.Select(s => s.Trim()).ToList();
query = query.Where(a => cleanStatuses.Contains(a.Status));
}
else
{
query = query.Where(a => a.Status != "Failed" && a.Status != "Duplicate");
}
return await query.ToListAsync();
}
/// <inheritdoc />
/// <summary>
/// API Gateway Helper: Retrieves articles for UI rendering, excluding intermediate/failed states by default.
/// </summary>
public async Task<List<NewsArticleEntity>> GetFilteredNewsAsync(
int limit = 20,
int offset = 0,
int limit,
int offset,
string? isin = null,
DateTime? date = null,
string? status = null,
string? searchQuery = null)
{
IQueryable<NewsArticleEntity> query = _context.NewsArticles
var query = _context.NewsArticles
.Include(a => a.MatchedAssets)
.AsNoTracking();
.AsNoTracking()
.AsQueryable();
// 1. Status Filter
if (!string.IsNullOrWhiteSpace(status))
{
var targetStatus = status.Trim();
query = query.Where(a => a.Status == targetStatus);
}
else
{
// By default, only show fully processed articles to the API/UI
query = query.Where(a => a.Status == "Completed" || a.Status == "Analyzed");
}
// 2. Date Filter
if (date.HasValue)
{
// Erstelle ein exaktes UTC-Datum von 00:00:00 Uhr am gebuchten Tag
var targetDate = new DateTime(date.Value.Year, date.Value.Month, date.Value.Day, 0, 0, 0, DateTimeKind.Utc);
var nextDate = targetDate.AddDays(1);
query = query.Where(a => a.PublishedAt >= targetDate && a.PublishedAt < nextDate);
}
// 3. ISIN / Symbol Filter
if (!string.IsNullOrWhiteSpace(isin))
{
var cleanIsin = isin.Trim();
query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin || m.Name == cleanIsin));
query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin));
}
if (date.HasValue)
{
var startUtc = date.Value.Date.ToUniversalTime();
var endUtc = startUtc.AddDays(1);
query = query.Where(a => a.PublishedAt >= startUtc && a.PublishedAt < endUtc);
}
if (!string.IsNullOrWhiteSpace(status))
{
query = query.Where(a => a.Status == status);
}
// 4. Search Term
if (!string.IsNullOrWhiteSpace(searchQuery))
{
var q = searchQuery.Trim();
query = query.Where(a => EF.Functions.ILike(a.Title, $"%{q}%") || (a.Summary != null && EF.Functions.ILike(a.Summary, $"%{q}%")));
var cleanSearch = searchQuery.Trim().ToLower();
query = query.Where(a =>
a.Title.ToLower().Contains(cleanSearch) ||
a.Summary.ToLower().Contains(cleanSearch) ||
a.MatchedAssets.Any(m => m.Name.ToLower().Contains(cleanSearch)));
}
return await query
.OrderByDescending(a => a.PublishedAt)
.ThenByDescending(a => a.ScrapedAt)
.ThenByDescending(a => a.Id)
.Skip(offset)
.Take(limit)
.ToListAsync();
@@ -1,18 +1,22 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Models;
using FinlyticAssets.Util;
using FinlyticCore.Dtos.News;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticNews.Entities;
using FinlyticNews.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticNews.Services;
@@ -22,9 +26,6 @@ namespace FinlyticNews.Services;
/// </summary>
public class NewsScraperBackgroundService : BackgroundService
{
/// <summary>
/// Internal wrapper to associate compiled regex patterns with the unmodified AssetIndex record.
/// </summary>
private record CompiledAssetMatcher(
AssetIndex Asset,
string CoreName,
@@ -33,63 +34,63 @@ public class NewsScraperBackgroundService : BackgroundService
);
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<NewsScraperBackgroundService> _logger;
private readonly IFinlyticLogger<NewsScraperBackgroundService> _finlyticLogger;
private readonly NewsMqttClient _mqttClient;
private readonly int _intervalMinutes;
private readonly string _indexPath;
// In-Memory Cache for compiled asset matchers to prevent re-reading & re-compiling Regex
private List<CompiledAssetMatcher>? _cachedAssetMatchers;
private DateTime _lastIndexLoadTime = DateTime.MinValue;
public NewsScraperBackgroundService(
IServiceScopeFactory scopeFactory,
ILogger<NewsScraperBackgroundService> logger,
IFinlyticLogger<NewsScraperBackgroundService> finlyticLogger,
NewsMqttClient mqttClient,
IConfiguration configuration)
{
_scopeFactory = scopeFactory;
_logger = logger;
_finlyticLogger = finlyticLogger;
_mqttClient = mqttClient;
_intervalMinutes = configuration.GetValue<int>("ScrapingSettings:IntervalMinutes", 15);
_indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json");
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("[{Channel}] NewsScraperBackgroundService started. Interval: {Minutes} minutes.", "NewsChannel", _intervalMinutes);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] NewsScraperBackgroundService started.");
while (!stoppingToken.IsCancellationRequested)
{
try
{
await RunScrapingCycleAsync(stoppingToken);
using var scope = _scopeFactory.CreateScope();
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
bool enabled = await settings.GetSettingAsync(SettingKeys.EnableAutoScraping, stoppingToken);
if (enabled)
{
await RunScrapingCycleAsync(stoppingToken);
}
else
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Auto-scraping is disabled via settings.");
}
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogError(ex, "[{Channel}] An unhandled exception occurred during news scraping cycle.", "NewsChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] An unhandled exception occurred during news scraping cycle.");
}
int intervalMinutes = _intervalMinutes;
int intervalMinutes = 15;
try
{
using var scope = _scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetService<ISettingsDbService>();
if (settingsDb != null)
{
var settings = await settingsDb.GetSettingsAsync();
if (settings?.ScrapingIntervalMinutes > 0)
{
intervalMinutes = settings.ScrapingIntervalMinutes;
}
}
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
intervalMinutes = await settings.GetSettingAsync(SettingKeys.ScrapeIntervalMinutes, stoppingToken);
}
catch { /* Ignore settings DB lookup failures */ }
catch { }
var jitterSeconds = Random.Shared.Next(0, 300);
var jitterSeconds = Random.Shared.Next(0, 60);
var nextRunDelay = TimeSpan.FromMinutes(intervalMinutes) + TimeSpan.FromSeconds(jitterSeconds);
_logger.LogInformation("[{Channel}] Scraping cycle completed. Next cycle in {Delay} (interval: {Minutes}m).", "NewsChannel", nextRunDelay, intervalMinutes);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Scraping cycle completed. Next cycle in {Delay} (interval: {Minutes}m).", nextRunDelay, intervalMinutes);
try
{
@@ -101,7 +102,7 @@ public class NewsScraperBackgroundService : BackgroundService
}
}
_logger.LogInformation("[{Channel}] NewsScraperBackgroundService stopping.", "NewsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] NewsScraperBackgroundService stopping.");
}
private async Task RunScrapingCycleAsync(CancellationToken stoppingToken)
@@ -111,28 +112,28 @@ public class NewsScraperBackgroundService : BackgroundService
var discoveryService = scope.ServiceProvider.GetRequiredService<IArticleDiscoveryService>();
var scraperService = scope.ServiceProvider.GetRequiredService<IPlaywrightScraperService>();
var n8nService = scope.ServiceProvider.GetRequiredService<IN8nService>();
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
var maxArticlesPerFeed = await settings.GetSettingAsync(SettingKeys.MaxArticlesPerFeed, stoppingToken);
// Load pre-compiled asset index matchers for zero-latency pre-filtering
var assetMatchers = await GetOrLoadAssetMatchersAsync();
_logger.LogInformation("[{Channel}] Loaded {Count} asset index items for text pre-filtering.", "NewsChannel", assetMatchers.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Loaded {Count} asset index items for text pre-filtering.", assetMatchers.Count);
// 1. Scraping Retry Phase: query articles in status "Scraping" (failed Playwright runs)
var failedArticles = await dbService.GetArticlesByStatusAsync("Scraping");
if (failedArticles.Count > 0)
{
_logger.LogInformation("[{Channel}] Found {Count} articles in status 'Scraping' that failed to scrape previously. Retrying...", "NewsChannel", failedArticles.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Found {Count} articles in status 'Scraping' that failed to scrape previously. Retrying...", failedArticles.Count);
foreach (var article in failedArticles)
{
if (stoppingToken.IsCancellationRequested) break;
await ProcessSingleArticleAsync(article, dbService, scraperService, n8nService, assetMatchers, stoppingToken);
if (stoppingToken.IsCancellationRequested) return;
await ProcessSingleArticleAsync(article, scraperService, n8nService, dbService, assetMatchers, stoppingToken);
}
}
// 2. Link Discovery Phase: query RSS feeds and listing pages
var sources = await dbService.GetSourcesAsync();
if (sources.Count == 0)
{
_logger.LogWarning("[{Channel}] No article sources configured in database. Skipping cycle.", "NewsChannel");
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] No article sources configured in database. Skipping cycle.");
return;
}
@@ -140,86 +141,80 @@ public class NewsScraperBackgroundService : BackgroundService
{
if (stoppingToken.IsCancellationRequested) break;
_logger.LogInformation("[{Channel}] Starting article link discovery for source: {SourceName} ({Url})", "NewsChannel", source.Name, source.Source);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Starting article link discovery for source: {SourceName} ({Url})", source.Name, source.Source);
var discoveredArticles = await discoveryService.DiscoverLinksAsync(source.Source, source.Type, stoppingToken);
if (discoveredArticles == null || discoveredArticles.Count == 0)
{
_logger.LogDebug("No links discovered from source: {SourceName}", source.Name);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] No links discovered from source: {SourceName}", source.Name);
continue;
}
_logger.LogInformation("[{Channel}] Discovered {Count} potential article links from {SourceName}.", "NewsChannel", discoveredArticles.Count, source.Name);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Discovered {Count} potential article links from {SourceName}.", discoveredArticles.Count, source.Name);
foreach (var discovered in discoveredArticles)
var toProcess = discoveredArticles.Take(maxArticlesPerFeed > 0 ? maxArticlesPerFeed : 20);
foreach (var discovered in toProcess)
{
if (stoppingToken.IsCancellationRequested) break;
// Idempotency Check & Deduplication
var isDuplicate = await dbService.IsUrlDuplicateAsync(discovered.Url);
if (isDuplicate)
if (await dbService.IsUrlDuplicateAsync(discovered.Url))
{
_logger.LogDebug("Skipping duplicate article URL: {Url}", discovered.Url);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Skipping duplicate article URL: {Url}", discovered.Url);
continue;
}
// Register Initial Lock State in the database ("Pending")
NewsArticleEntity? article;
NewsArticleEntity article;
try
{
article = await dbService.CreatePendingArticleAsync(
discovered.Url,
discovered.Isins,
discovered.Title,
discovered.Summary,
discovered.PublishedAt,
discovered.Language);
discovered.Url,
discovered.Isins,
discovered.Title,
discovered.Summary,
discovered.PublishedAt,
discovered.Language
);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to register initial pending state for URL: {Url}. Skipping.", "NewsChannel", discovered.Url);
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to register initial pending state for URL: {Url}. Skipping.", discovered.Url);
continue;
}
if (article == null || article.Id == Guid.Empty)
if (article.Id == Guid.Empty)
{
_logger.LogWarning("[{Channel}] Created pending article has invalid/empty ID for URL: {Url}. Skipping.", "NewsChannel", discovered.Url);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Created pending article has invalid/empty ID for URL: {Url}. Skipping.", discovered.Url);
continue;
}
// Process single article pipeline
await ProcessSingleArticleAsync(article, dbService, scraperService, n8nService, assetMatchers, stoppingToken);
await ProcessSingleArticleAsync(article, scraperService, n8nService, dbService, assetMatchers, stoppingToken);
}
}
}
private async Task ProcessSingleArticleAsync(
NewsArticleEntity article,
INewsDbService dbService,
IPlaywrightScraperService scraperService,
IN8nService n8nService,
INewsDbService dbService,
List<CompiledAssetMatcher> assetMatchers,
CancellationToken stoppingToken)
{
try
{
// 3. Extraction with Headless Browser (Transitions to "Processing")
await dbService.UpdateArticleStatusAsync(article.Id, "Processing");
var (resolvedUrl, rawText) = await scraperService.ScrapeArticleAsync(article.SourceUrl);
if (string.IsNullOrWhiteSpace(rawText))
{
throw new InvalidOperationException("Scraping returned empty text body content.");
}
var (resolvedUrl, rawContent) = await scraperService.ScrapeArticleAsync(article.SourceUrl);
// Update resolved URL if redirect occurred
if (!string.Equals(resolvedUrl, article.SourceUrl, StringComparison.OrdinalIgnoreCase))
if (!string.IsNullOrWhiteSpace(resolvedUrl) &&
!resolvedUrl.Equals(article.SourceUrl, StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation("[{Channel}] Redirect detected. Initial: {OldUrl} -> Resolved: {NewUrl}", "NewsChannel", article.SourceUrl, resolvedUrl);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Redirect detected. Initial: {OldUrl} -> Resolved: {NewUrl}", article.SourceUrl, resolvedUrl);
if (await dbService.IsUrlDuplicateAsync(resolvedUrl))
{
_logger.LogInformation("[{Channel}] Redirected URL {ResolvedUrl} is a duplicate. Terminating processing.", "NewsChannel", resolvedUrl);
await dbService.UpdateArticleStatusAsync(article.Id, "Duplicate");
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Redirected URL {ResolvedUrl} is a duplicate. Terminating processing.", resolvedUrl);
await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
return;
}
@@ -227,108 +222,85 @@ public class NewsScraperBackgroundService : BackgroundService
article.SourceUrl = resolvedUrl;
}
// 4. Pre-filtering Assets (Optimized with Pre-Compiled Regex Patterns)
var title = article.Title ?? string.Empty;
var summary = article.Summary ?? string.Empty;
var preFilteredAssets = assetMatchers.Where(matcher =>
if (string.IsNullOrWhiteSpace(rawContent) || rawContent.Length < 60)
{
var asset = matcher.Asset;
// ISIN direct match
if (rawText.Contains(asset.Isin, StringComparison.OrdinalIgnoreCase) ||
title.Contains(asset.Isin, StringComparison.OrdinalIgnoreCase) ||
summary.Contains(asset.Isin, StringComparison.OrdinalIgnoreCase))
{
return true;
}
// Fast regex word boundary check on Full Name
if (matcher.WordRegex != null && (matcher.WordRegex.IsMatch(rawText) || matcher.WordRegex.IsMatch(title)))
{
return true;
}
// Fast regex word boundary check on Core Name
if (matcher.CoreName.Length >= 3 && matcher.CoreWordRegex != null &&
(matcher.CoreWordRegex.IsMatch(rawText) || matcher.CoreWordRegex.IsMatch(title)))
{
return true;
}
return false;
})
.Select(m => new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin))
.ToList();
if (preFilteredAssets.Count == 0)
{
_logger.LogInformation("[{Channel}] Pre-filtering: Article {Id} does not reference any known assets. Terminating pipeline.", "NewsChannel", article.Id);
await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
return;
}
_logger.LogInformation("[{Channel}] Pre-filtering matched {Count} assets for article {Id}.", "NewsChannel", preFilteredAssets.Count, article.Id);
var discoveredIsins = article.MatchedAssets.Select(m => m.Isin).Where(i => !string.IsNullOrEmpty(i)).ToList();
var preFilteredAssets = PreFilterAssets(rawContent, article.Title, assetMatchers, discoveredIsins);
// 5. Send to n8n Webhook Pipeline
var n8nResponse = await n8nService.AnalyzeArticleAsync(rawText, preFilteredAssets, stoppingToken);
if (n8nResponse == null)
if (preFilteredAssets.Count == 0)
{
throw new InvalidOperationException("n8n AI webhook execution returned null or failed.");
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Pre-filtering: Article {Id} does not reference any known assets. Terminating pipeline.", article.Id);
await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
return;
}
// 6. Map and Save Completed Classification
var matchedEntities = new List<MatchedAssetEntity>();
foreach (var n8nAsset in n8nResponse.MatchedAssets)
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Pre-filtering matched {Count} assets for article {Id}.", preFilteredAssets.Count, article.Id);
var n8nResponse = await n8nService.AnalyzeArticleAsync(rawContent, preFilteredAssets, stoppingToken);
if (n8nResponse == null)
{
var n8nCoreName = ExtractCoreAssetName(n8nAsset.Name);
await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
return;
}
var matchedIsin = preFilteredAssets.FirstOrDefault(fa =>
fa.Name.Equals(n8nAsset.Name, StringComparison.OrdinalIgnoreCase) ||
n8nAsset.Name.Contains(fa.Name, StringComparison.OrdinalIgnoreCase) ||
(n8nCoreName.Length >= 3 && ExtractCoreAssetName(fa.Name).Equals(n8nCoreName, StringComparison.OrdinalIgnoreCase)))?.Isin;
if (string.IsNullOrWhiteSpace(matchedIsin))
var matchedEntities = new List<MatchedAssetEntity>();
if (n8nResponse.MatchedAssets != null && n8nResponse.MatchedAssets.Count > 0)
{
foreach (var asset in n8nResponse.MatchedAssets)
{
matchedIsin = assetMatchers.FirstOrDefault(m =>
m.Asset.Name.Equals(n8nAsset.Name, StringComparison.OrdinalIgnoreCase) ||
n8nAsset.Name.Contains(m.Asset.Name, StringComparison.OrdinalIgnoreCase) ||
(n8nCoreName.Length >= 3 && m.CoreName.Equals(n8nCoreName, StringComparison.OrdinalIgnoreCase)))?.Asset.Isin;
}
var preMatch = preFilteredAssets.FirstOrDefault(p => p.Name.Equals(asset.Name, StringComparison.OrdinalIgnoreCase));
var isin = preMatch?.Isin ?? asset.Ticker ?? "";
if (string.IsNullOrWhiteSpace(isin)) continue;
if (!string.IsNullOrWhiteSpace(matchedIsin))
{
matchedEntities.Add(new MatchedAssetEntity
{
Id = Guid.NewGuid(),
NewsArticleId = article.Id,
Name = n8nAsset.Name,
Isin = matchedIsin
Isin = isin.Trim().ToUpperInvariant(),
Name = !string.IsNullOrWhiteSpace(asset.Name) ? asset.Name.Trim() : isin.Trim().ToUpperInvariant()
});
}
}
var completedArticle = await dbService.SaveArticleClassificationAsync(article.Id, n8nResponse, matchedEntities);
if (completedArticle != null)
if (matchedEntities.Count == 0)
{
foreach (var preMatch in preFilteredAssets)
{
matchedEntities.Add(new MatchedAssetEntity
{
Id = Guid.NewGuid(),
NewsArticleId = article.Id,
Isin = preMatch.Isin,
Name = preMatch.Name
});
}
}
var updatedArticle = await dbService.SaveArticleClassificationAsync(article.Id, n8nResponse, matchedEntities);
if (updatedArticle != null)
{
// 7. MQTT Broadcast (Sends completed article to downstream services)
var dto = new NewsArticleDto
{
Id = completedArticle.Id,
Title = completedArticle.Title,
Author = completedArticle.Author,
Summary = completedArticle.Summary,
ContentRaw = completedArticle.ContentRaw,
Language = completedArticle.Language,
SourceUrl = completedArticle.SourceUrl,
ScrapedAt = completedArticle.ScrapedAt,
PublishedAt = completedArticle.PublishedAt,
MatchedAssets = completedArticle.MatchedAssets.Select(m => new MatchedAssetDto
Id = updatedArticle.Id,
Title = updatedArticle.Title,
Author = updatedArticle.Author,
Summary = updatedArticle.Summary,
ContentRaw = updatedArticle.ContentRaw,
Language = updatedArticle.Language,
SourceUrl = updatedArticle.SourceUrl,
ScrapedAt = updatedArticle.ScrapedAt,
PublishedAt = updatedArticle.PublishedAt,
Status = updatedArticle.Status,
MatchedAssets = updatedArticle.MatchedAssets.Select(m => new MatchedAssetDto
{
Name = m.Name,
Isin = m.Isin
}).ToList(),
Status = completedArticle.Status
}).ToList()
};
await _mqttClient.BroadcastArticleAsync(dto);
@@ -336,103 +308,120 @@ public class NewsScraperBackgroundService : BackgroundService
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to complete processing pipeline for article: {Url}. Transitioning to 'Scraping' for next cycle retry.", "NewsChannel", article.SourceUrl);
try
{
await dbService.UpdateArticleStatusAsync(article.Id, "Scraping");
}
catch { /* Suppress database secondary errors */ }
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to complete processing pipeline for article: {Url}. Transitioning to 'Scraping' for next cycle retry.", article.SourceUrl);
await dbService.UpdateArticleStatusAsync(article.Id, "Scraping");
}
}
/// <summary>
/// Returns cached compiled asset matchers or parses the index file from disk if stale/missing.
/// </summary>
private List<FilteredAssetPayload> PreFilterAssets(
string content,
string? title,
List<CompiledAssetMatcher> assetMatchers,
List<string>? priorityIsins = null)
{
if (assetMatchers.Count == 0) return new List<FilteredAssetPayload>();
var fullText = (title != null ? title + " " + content : content);
var matched = new Dictionary<string, FilteredAssetPayload>(StringComparer.OrdinalIgnoreCase);
if (priorityIsins != null && priorityIsins.Count > 0)
{
foreach (var isin in priorityIsins)
{
var match = assetMatchers.FirstOrDefault(m => string.Equals(m.Asset.Isin, isin, StringComparison.OrdinalIgnoreCase));
if (match != null && !matched.ContainsKey(match.Asset.Isin))
{
matched[match.Asset.Isin] = new FilteredAssetPayload(match.Asset.Name, match.Asset.Isin);
}
}
}
foreach (var m in assetMatchers)
{
if (matched.ContainsKey(m.Asset.Isin)) continue;
if (fullText.Contains(m.Asset.Isin, StringComparison.OrdinalIgnoreCase))
{
matched[m.Asset.Isin] = new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin);
continue;
}
if (m.WordRegex != null && m.WordRegex.IsMatch(fullText))
{
matched[m.Asset.Isin] = new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin);
continue;
}
if (m.CoreWordRegex != null && m.CoreWordRegex.IsMatch(fullText))
{
matched[m.Asset.Isin] = new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin);
}
}
return matched.Values.ToList();
}
private async Task<List<CompiledAssetMatcher>> GetOrLoadAssetMatchersAsync()
{
if (_cachedAssetMatchers != null && (DateTime.UtcNow - _lastIndexLoadTime).TotalMinutes < 30)
if (_cachedAssetMatchers != null && (DateTime.UtcNow - _lastIndexLoadTime).TotalMinutes < 60)
{
return _cachedAssetMatchers;
}
if (!File.Exists(_indexPath))
{
_logger.LogWarning("[{Channel}] Asset index file not found at: {Path}. Pre-filtering will match 0 assets.", "NewsChannel", _indexPath);
return [];
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Asset index file not found at: {Path}. Pre-filtering will match 0 assets.", _indexPath);
return new List<CompiledAssetMatcher>();
}
try
{
await using var stream = File.OpenRead(_indexPath);
// Standard Deserialization for AssetIndex list
var rawList = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream);
using var stream = File.OpenRead(_indexPath);
var indexList = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream);
if (rawList != null)
if (indexList == null || indexList.Count == 0)
{
_cachedAssetMatchers = rawList.Select(asset =>
{
var coreName = ExtractCoreAssetName(asset.Name);
return new CompiledAssetMatcher(
Asset: asset,
CoreName: coreName,
WordRegex: BuildWordRegex(asset.Name),
CoreWordRegex: BuildWordRegex(coreName)
);
}).ToList();
_lastIndexLoadTime = DateTime.UtcNow;
return _cachedAssetMatchers;
return new List<CompiledAssetMatcher>();
}
var compiled = new List<CompiledAssetMatcher>(indexList.Count);
foreach (var asset in indexList)
{
if (string.IsNullOrWhiteSpace(asset.Name) || string.IsNullOrWhiteSpace(asset.Isin))
continue;
var rawName = asset.Name.Trim();
var coreName = ExtractCoreName(rawName);
Regex? wordRegex = null;
if (rawName.Length >= 4)
{
wordRegex = new Regex($@"\b{Regex.Escape(rawName)}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
}
Regex? coreWordRegex = null;
if (!string.IsNullOrWhiteSpace(coreName) && coreName.Length >= 4 && !coreName.Equals(rawName, StringComparison.OrdinalIgnoreCase))
{
coreWordRegex = new Regex($@"\b{Regex.Escape(coreName)}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
}
compiled.Add(new CompiledAssetMatcher(asset, coreName, wordRegex, coreWordRegex));
}
_cachedAssetMatchers = compiled;
_lastIndexLoadTime = DateTime.UtcNow;
return _cachedAssetMatchers;
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to read or parse asset index file from {Path}.", "NewsChannel", _indexPath);
}
return _cachedAssetMatchers ?? [];
}
/// <summary>
/// Helper to pre-compile Word Boundary Regex for an asset name.
/// </summary>
private static Regex? BuildWordRegex(string name)
{
if (string.IsNullOrWhiteSpace(name)) return null;
try
{
return new Regex($@"\b{Regex.Escape(name)}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
}
catch
{
return null;
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to read or parse asset index file from {Path}.", _indexPath);
return _cachedAssetMatchers ?? new List<CompiledAssetMatcher>();
}
}
/// <summary>
/// Extracts the core name of an asset by removing parenthetical metadata and corporate suffixes.
/// </summary>
private static string ExtractCoreAssetName(string name)
private static string ExtractCoreName(string rawName)
{
if (string.IsNullOrWhiteSpace(name)) return string.Empty;
int parenIndex = name.IndexOf('(');
if (parenIndex >= 0)
{
name = name[..parenIndex];
}
name = name.Trim();
var suffixes = new[] { "Inc.", "Inc", "AG", "SE", "Co.", "Co", "Corp.", "Corp", "Ltd.", "Ltd", "plc", "GmbH", "SA", "NV", "Group" };
foreach (var suffix in suffixes)
{
if (name.EndsWith(" " + suffix, StringComparison.OrdinalIgnoreCase))
{
name = name[..^suffix.Length].Trim();
}
}
return name;
var cleaned = Regex.Replace(rawName, @"\b(AG|SE|SA|NV|PLC|INC|CORP|LLC|GMBH|CO|KG|HOLDING|GROUP|CLASS\s+[A-Z])\b", "", RegexOptions.IgnoreCase);
return cleaned.Trim(' ', '.', ',', '-');
}
}
@@ -3,8 +3,9 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services;
using FinlyticNews.Adapters.Scraping;
using Microsoft.Extensions.Logging;
using FinlyticNews.Util;
using Microsoft.Playwright;
namespace FinlyticNews.Services;
@@ -25,7 +26,7 @@ public interface IPlaywrightScraperService
/// <inheritdoc />
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
{
private readonly ILogger<PlaywrightScraperService> _logger;
private readonly IFinlyticLogger<PlaywrightScraperService> _finlyticLogger;
private readonly IEnumerable<ArticleScraperAdapter> _scraperAdapters;
private IPlaywright? _playwright;
@@ -36,21 +37,20 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
/// Initializes a new instance of the <see cref="PlaywrightScraperService"/> class.
/// </summary>
public PlaywrightScraperService(
ILogger<PlaywrightScraperService> logger,
IFinlyticLogger<PlaywrightScraperService> finlyticLogger,
IEnumerable<ArticleScraperAdapter> scraperAdapters)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_scraperAdapters = scraperAdapters;
}
/// <inheritdoc />
public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
{
_logger.LogInformation("[{Channel}] Launching browser context to scrape article: {Url}", "NewsChannel", url);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
var browser = await GetOrInitBrowserAsync();
// Fast, isolated browser context (incognito tab environment) per article
await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
{
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -61,7 +61,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
try
{
// 1. Initial Navigation with DOMContentLoaded wait
var response = await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
@@ -74,115 +73,86 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
var finalUrl = page.Url;
_logger.LogDebug("[{Channel}] Navigation completed. Initial final URL: {Url}", "NewsChannel", finalUrl);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Navigation completed. Initial final URL: {Url}", finalUrl);
// 2. Resolve Host Specific Scraper Adapter
var uri = new Uri(url);
var host = uri.Host;
var adapter = _scraperAdapters.FirstOrDefault(a => host.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
IPage targetPage = page;
var host = new Uri(finalUrl).Host;
var adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
if (adapter != null)
{
_logger.LogInformation("[{Channel}] Executing adapter redirect check for host: {Host}", "NewsChannel", adapter.Hostname);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Executing adapter redirect check for host: {Host}", adapter.Hostname);
try
{
var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
// Loop Protection: Navigate only if redirect target is a new URL
if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) &&
!string.Equals(page.Url, resolvedRedirectUrl, StringComparison.OrdinalIgnoreCase))
!resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation("[{Channel}] Redirect resolved to target URL: {Url}", "NewsChannel", resolvedRedirectUrl);
var refererUrl = page.Url;
finalUrl = resolvedRedirectUrl;
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30000,
Referer = refererUrl
Timeout = 30000
});
if (redirectResponse == null)
{
_logger.LogWarning("[{Channel}] Failed to load response for redirect URL: {Url}", "NewsChannel", resolvedRedirectUrl);
}
else
{
finalUrl = page.Url;
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Failed to load response for redirect URL: {Url}", resolvedRedirectUrl);
}
// Check if redirect opened a new tab/popup
var matchedPage = page.Context.Pages.FirstOrDefault(p => p.Url == resolvedRedirectUrl);
if (matchedPage != null)
{
targetPage = matchedPage;
}
finalUrl = page.Url;
host = new Uri(finalUrl).Host;
adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", "NewsChannel", adapter.Hostname);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter.Hostname);
}
}
// 3. Re-resolve detail page adapter for final target page
var targetUri = new Uri(targetPage.Url);
var targetHost = targetUri.Host;
var targetAdapter = _scraperAdapters.FirstOrDefault(a => targetHost.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
// Wait a brief moment for dynamic scripts / DOM settling
await targetPage.WaitForTimeoutAsync(1000);
// 4. Extract Content via Mozilla Readability (or Adapter Fallback)
string extractedText = string.Empty;
if (targetAdapter != null)
string content;
if (adapter != null)
{
var readabilityResult = await targetAdapter.ExtractArticleContentAsync(targetPage);
if (readabilityResult != null && !string.IsNullOrWhiteSpace(readabilityResult.TextContent))
{
extractedText = readabilityResult.TextContent;
}
var result = await adapter.ExtractArticleContentAsync(page);
content = result?.TextContent ?? await FallbackExtractContentAsync(page);
}
// Standard Fallback: Body Text / Selector Extraction
if (string.IsNullOrWhiteSpace(extractedText))
else
{
var bodySelector = targetAdapter?.ArticleBodySelector ?? "body";
var locator = targetPage.Locator(bodySelector);
if (await locator.CountAsync() > 0)
{
extractedText = await locator.First.InnerTextAsync();
}
if (string.IsNullOrWhiteSpace(extractedText))
{
extractedText = await targetPage.EvaluateAsync<string>(
$"() => document.querySelector('{bodySelector}')?.innerText ?? ''");
}
content = await FallbackExtractContentAsync(page);
}
return (finalUrl, extractedText?.Trim() ?? string.Empty);
return (finalUrl, content);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to scrape page content from URL: {Url}", "NewsChannel", url);
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
throw;
}
finally
{
await context.CloseAsync();
await page.CloseAsync();
}
}
/// <summary>
/// Thread-safe singleton initialization of the Chromium browser instance.
/// </summary>
private async Task<string> FallbackExtractContentAsync(IPage page)
{
var innerText = await page.EvaluateAsync<string>(@"() => {
const scripts = document.querySelectorAll('script, style, noscript, nav, header, footer, iframe, svg');
scripts.forEach(s => s.remove());
const main = document.querySelector('article, main, .article-content, #content, .story-body') || document.body;
return main ? main.innerText : document.body.innerText;
}");
return innerText?.Trim() ?? string.Empty;
}
private async Task<IBrowser> GetOrInitBrowserAsync()
{
if (_browser != null && _browser.IsConnected)
@@ -202,10 +172,16 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = true,
Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
Args = new[]
{
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu"
}
});
_logger.LogInformation("[{Channel}] Initialized shared Chromium browser instance.", "NewsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Initialized shared Chromium browser instance.");
return _browser;
}
finally
@@ -214,9 +190,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
}
/// <summary>
/// Disposes the Playwright and Browser instances cleanly during service shutdown.
/// </summary>
public async ValueTask DisposeAsync()
{
if (_browser != null)
+122 -90
View File
@@ -1,10 +1,18 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Models;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticNews.Database;
using FinlyticNews.Entities;
using FinlyticNews.Services;
using Microsoft.Extensions.Configuration;
@@ -40,12 +48,10 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
{
Host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost",
Port = Convert.ToInt32(_configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883"),
ClientId =
$"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticNews")}_{Guid.NewGuid()}"
ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticNews")}_{Guid.NewGuid()}"
};
_logger.LogInformation("Starting News MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host,
config.ClientId);
_logger.LogInformation("Starting News MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId);
await ConnectAsync(config);
}
@@ -61,13 +67,22 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
{
_logger.LogInformation("News MQTT client connected. Subscribing to RPC topics...");
// ZUSAMMENGELEGT: Unified News Fetching (news_Get deckt news_GetDaily mit ab)
await SubscribeAsync("services/request/news_Get/#");
await SubscribeAsync("services/request/news_GetById/#");
await SubscribeAsync("services/request/news_GetPending/#");
await SubscribeAsync("services/request/news_UpdateStatus/#");
await SubscribeAsync("services/request/news_settings_GetAll/#");
await SubscribeAsync("services/request/news_settings_Update/#");
await SubscribeAsync("services/request/health_Ping/#");
await SubscribeAsync("services/config/updated/#");
FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
{
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticNews", StringComparison.OrdinalIgnoreCase))
{
await PublishAsync("finlytic/logs/FinlyticNews", logDto);
}
};
}
/// <summary>
@@ -75,12 +90,10 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
/// </summary>
public async Task BroadcastArticleAsync(NewsArticleDto article)
{
// 1. Primärer System-Broadcast
const string topic = "services/news/completed";
_logger.LogInformation("Broadcasting completed article to MQTT topic: {Topic}. ID: {Id}", topic, article.Id);
await PublishAsync(topic, article);
// 2. Zielgerichteter ISIN-Stream für Echtzeit-Frontend-Feeds
var firstIsin = article.MatchedAssets.FirstOrDefault()?.Isin;
if (!string.IsNullOrWhiteSpace(firstIsin))
{
@@ -94,14 +107,12 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
{
if (string.IsNullOrWhiteSpace(topic)) return;
// 1. System Config Updates
if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase))
{
if (topic.EndsWith("FinlyticNews", StringComparison.OrdinalIgnoreCase))
{
await OnConfigUpdatedAsync(payload);
}
return;
}
@@ -111,7 +122,6 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
var channel = segments[2];
var correlationId = segments[^1];
// 2. Unified RPC Dispatching via Switch
switch (channel)
{
case "news_Get":
@@ -130,6 +140,14 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
await OnUpdateNewsStatusAsync(payload, correlationId);
break;
case "news_settings_GetAll":
await OnSettingsGetAllAsync(correlationId);
break;
case "news_settings_Update":
await OnSettingsUpdateAsync(payload, correlationId);
break;
case "health_Ping":
await OnHealthPingAsync(segments, correlationId);
break;
@@ -142,7 +160,10 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
private async Task OnGetNewsAsync(string payload, string correlationId)
{
_logger.LogInformation("Received RPC news_Get request. Correlation: {CorrelationId}", correlationId);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<NewsMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "Received RPC news_Get request. Correlation: {CorrelationId}", correlationId);
int limit = 20;
int offset = 0;
@@ -155,9 +176,7 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
{
try
{
// Direktes Deserialisieren über das DailyNewsRequest-DTO
var req = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.DailyNewsRequest);
if (req != null)
{
limit = req.Limit > 0 ? req.Limit : 20;
@@ -167,7 +186,6 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
searchQuery = req.Query;
date = req.Date;
// Status anpassen, falls HasSentiment gesetzt ist
if (req.HasSentiment == true && string.IsNullOrEmpty(status))
{
status = "Analyzed";
@@ -176,44 +194,35 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to parse DailyNewsRequest payload on news_Get");
await finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "Failed to parse DailyNewsRequest payload on news_Get");
}
}
try
{
using var scope = _scopeFactory.CreateScope();
var dbService = scope.ServiceProvider.GetRequiredService<INewsDbService>();
var articles = await dbService.GetFilteredNewsAsync(limit, offset, isin, date, status, searchQuery);
var dtos = (await Task.WhenAll(articles.Select(a => MapToDtoAsync(a)))).ToList();
string responseTopic = $"services/response/news_Get/{correlationId}";
_logger.LogInformation("Publishing RPC response to {ResponseTopic} with {Count} articles.", responseTopic,
dtos.Count);
await finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "Publishing RPC response to {ResponseTopic} with {Count} articles.", responseTopic, dtos.Count);
await PublishAsync(responseTopic, dtos);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to compile RPC response for news_Get");
// Antworte mit leerer Liste, um RPC-Timeouts im Gateway zu vermeiden
await finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "Failed to compile RPC response for news_Get");
try
{
string responseTopic = $"services/response/news_Get/{correlationId}";
await PublishAsync(responseTopic, new List<NewsArticleDto>());
}
catch
{
}
catch { }
}
}
private async Task OnGetNewsByIdAsync(string payload, string correlationId)
{
_logger.LogInformation("Received RPC news_GetById request. Correlation: {CorrelationId}", correlationId);
string responseTopic = $"services/response/news_GetById/{correlationId}";
if (string.IsNullOrWhiteSpace(payload))
{
await PublishAsync(responseTopic, (object?)null);
@@ -222,9 +231,7 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
try
{
var request =
JsonSerializer.Deserialize<ArticleRequest>(payload,
FinlyticJsonSerializerContext.Default.ArticleRequest);
var request = JsonSerializer.Deserialize<ArticleRequest>(payload, FinlyticJsonSerializerContext.Default.ArticleRequest);
var targetIdStr = request?.ArticleId ?? request?.Id;
if (Guid.TryParse(targetIdStr, out var articleId))
@@ -241,33 +248,25 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to execute RPC news_GetById");
}
catch { }
await PublishAsync(responseTopic, (object?)null);
}
private async Task OnGetPendingNewsAsync(string payload, string correlationId)
{
_logger.LogInformation("Received RPC news_GetPending request. Correlation: {CorrelationId}", correlationId);
int limit = 10;
if (!string.IsNullOrWhiteSpace(payload))
{
try
{
using var doc = JsonDocument.Parse(payload);
if (doc.RootElement.TryGetProperty("limit", out var limitProp) &&
limitProp.TryGetInt32(out var parsedLimit))
if (doc.RootElement.TryGetProperty("limit", out var limitProp) && limitProp.TryGetInt32(out var parsedLimit))
{
limit = Math.Min(parsedLimit, 10);
}
}
catch
{
}
catch { }
}
try
@@ -279,25 +278,17 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
var dtos = (await Task.WhenAll(pendingArticles.Take(limit).Select(a => MapToDtoAsync(a)))).ToList();
string responseTopic = $"services/response/news_GetPending/{correlationId}";
_logger.LogInformation("Publishing RPC news_GetPending response to {ResponseTopic} with {Count} articles.",
responseTopic, dtos.Count);
await PublishAsync(responseTopic, dtos);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to publish RPC pending news response");
}
catch { }
}
private async Task OnUpdateNewsStatusAsync(string payload, string correlationId)
{
_logger.LogInformation("Received RPC news_UpdateStatus request. Correlation: {CorrelationId}", correlationId);
UpdateNewsStatusResponse response;
try
{
var request = JsonSerializer.Deserialize<UpdateNewsStatusRequest>(payload,
FinlyticJsonSerializerContext.Default.UpdateNewsStatusRequest);
var request = JsonSerializer.Deserialize<UpdateNewsStatusRequest>(payload, FinlyticJsonSerializerContext.Default.UpdateNewsStatusRequest);
if (request != null && request.Id != Guid.Empty)
{
using var scope = _scopeFactory.CreateScope();
@@ -313,7 +304,6 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to execute RPC news_UpdateStatus");
response = new UpdateNewsStatusResponse(false, ex.Message);
}
@@ -321,42 +311,99 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
await PublishAsync(responseTopic, response);
}
private async Task OnSettingsGetAllAsync(string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<NewsMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticNews] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
try
{
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/news_settings_GetAll/{correlationId}";
await PublishAsync(responseTopic, settings);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticNews] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticNews] [Settings_GetAll] Failed to retrieve settings.");
}
}
private async Task OnSettingsUpdateAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<NewsMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticNews] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
try
{
Dictionary<string, object?>? updates = null;
try
{
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
}
catch
{
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
if (list != null)
{
updates = new Dictionary<string, object?>();
foreach (var item in list) updates[item.Key] = item.Value;
}
}
if (updates != null && updates.Count > 0)
{
await settingsService.UpdateSettingsAsync(updates);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticNews] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
}
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/news_settings_Update/{correlationId}";
await PublishAsync(responseTopic, currentSettings);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticNews] [Settings_Update] Failed to update settings.");
}
}
private async Task OnConfigUpdatedAsync(string payload)
{
_logger.LogInformation("Received config update event for FinlyticNews.");
try
{
using var doc = JsonDocument.Parse(payload);
if (doc.RootElement.TryGetProperty("settings", out var settingsProp))
{
var dict = JsonSerializer.Deserialize<Dictionary<string, string>>(settingsProp.GetRawText());
var dict = JsonSerializer.Deserialize<Dictionary<string, object?>>(settingsProp.GetRawText());
if (dict != null && dict.Count > 0)
{
using var scope = _scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsDb.UpdateSettingsFromDictionaryAsync(dict);
_logger.LogInformation("Successfully persisted {Count} updated settings.", dict.Count);
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await settings.UpdateSettingsAsync(dict);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing MQTT config update event");
}
catch { }
}
private async Task OnHealthPingAsync(string[] segments, string correlationId)
{
// Zerlegt den Topic-Pfad z. B. services/request/health_Ping/FinlyticNews/{correlationId}
bool isForMe = segments.Length >= 5 && segments[3].Equals("FinlyticNews", StringComparison.OrdinalIgnoreCase);
if (isForMe)
{
string respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic,
new ServiceHealthResponse("FinlyticNews", "Online", DateTime.UtcNow, "Connected"));
_logger.LogInformation("Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].",
correlationId);
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticNews", "Online", DateTime.UtcNow, "Connected"));
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<NewsMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
}
}
@@ -375,41 +422,31 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
var targetId = a.Id.ToString();
IsinAnalysisEntry? sentimentEntry = null;
// 1. Snappy Local Disk Check for Article File
var articlePath = Path.Combine(Directory.GetCurrentDirectory(), "data", "summaries", "articles",
$"{targetId}.json");
var articlePath = Path.Combine(Directory.GetCurrentDirectory(), "data", "summaries", "articles", $"{targetId}.json");
if (File.Exists(articlePath))
{
try
{
var json = await File.ReadAllTextAsync(articlePath);
sentimentEntry = JsonSerializer.Deserialize<IsinAnalysisEntry>(json,
FinlyticJsonSerializerContext.Default.IsinAnalysisEntry);
}
catch
{
sentimentEntry = JsonSerializer.Deserialize<IsinAnalysisEntry>(json, FinlyticJsonSerializerContext.Default.IsinAnalysisEntry);
}
catch { }
}
// 2. ISIN Summary File Fallback
if (sentimentEntry == null && a.MatchedAssets != null && a.MatchedAssets.Count > 0)
{
foreach (var asset in a.MatchedAssets)
{
if (string.IsNullOrWhiteSpace(asset.Isin)) continue;
var isinPath = Path.Combine(Directory.GetCurrentDirectory(), "data", "summaries", "isin",
$"{asset.Isin.Trim()}.json");
var isinPath = Path.Combine(Directory.GetCurrentDirectory(), "data", "summaries", "isin", $"{asset.Isin.Trim()}.json");
if (File.Exists(isinPath))
{
try
{
var json = await File.ReadAllTextAsync(isinPath);
var isinDoc = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json,
FinlyticJsonSerializerContext.Default.IsinSentimentSummaryDto);
var match = isinDoc?.Analyses?.FirstOrDefault(entry =>
string.Equals(entry.Article?.ArticleId?.Trim(), targetId,
StringComparison.OrdinalIgnoreCase));
var isinDoc = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, FinlyticJsonSerializerContext.Default.IsinSentimentSummaryDto);
var match = isinDoc?.Analyses?.FirstOrDefault(entry => string.Equals(entry.Article?.ArticleId?.Trim(), targetId, StringComparison.OrdinalIgnoreCase));
if (match != null)
{
@@ -417,9 +454,7 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
break;
}
}
catch
{
}
catch { }
}
}
}
@@ -432,10 +467,7 @@ public class NewsMqttClient : ManagedMqttClient, IHostedService
confidence = finbertResult.Confidence;
}
}
catch (Exception ex)
{
_logger.LogTrace(ex, "[MapToDtoAsync] Sentiment fetch skipped for article {Id}", a.Id);
}
catch { }
return new NewsArticleDto
{
+24
View File
@@ -0,0 +1,24 @@
using FinlyticCore.Models.Settings;
namespace FinlyticNews.Util;
public static class SettingKeys
{
// --- Logging-Kanäle ---
public static readonly SettingKey<bool> NewsChannel = new("Logging.Channel.News", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
// --- Scraping & Feed-Konfiguration ---
public static readonly SettingKey<int> ScrapeIntervalMinutes = new("Scraping.IntervalMinutes", 15);
public static readonly SettingKey<int> MaxArticlesPerFeed = new("Scraping.MaxArticlesPerFeed", 20);
public static readonly SettingKey<bool> EnableAutoScraping = new("Feature.EnableAutoScraping", true);
public static readonly SettingKey<int> HttpTimeoutSeconds = new("Scraping.HttpTimeoutSeconds", 20);
// --- KI & Sentiment-Konfiguration ---
public static readonly SettingKey<int> FinBertBatchSize = new("AI.FinBertBatchSize", 8);
public static readonly SettingKey<double> MinSentimentConfidence = new("AI.MinSentimentConfidence", 0.65);
// --- Daten-Retention & Cleanup ---
public static readonly SettingKey<int> ArticleRetentionDays = new("Data.ArticleRetentionDays", 90);
}
@@ -1,23 +1,33 @@
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using FinlyticSentiment.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticSentiment.Database;
/// <summary>
/// EF Core DbContext for managing FinlyticSentiment settings in PostgreSQL.
/// </summary>
public class SentimentDbContext : DbContext
public class SentimentDbContext : DbContext, ISettingsDbContext
{
public SentimentDbContext(DbContextOptions<SentimentDbContext> options) : base(options)
{
}
public DbSet<SettingEntity> DynamicSettings => Set<SettingEntity>();
public DbSet<SentimentSettingsEntity> Settings => Set<SentimentSettingsEntity>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
base.OnModelCreating(modelBuilder);
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key).IsUnique();
});
modelBuilder.Entity<SentimentSettingsEntity>(entity =>
{
entity.ToTable("sentiment_settings");
@@ -27,3 +37,13 @@ public class SentimentDbContext : DbContext
});
}
}
public class SentimentDbContextFactory : IDesignTimeDbContextFactory<SentimentDbContext>
{
public SentimentDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<SentimentDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=sentiment;Username=postgres;Password=postgres");
return new SentimentDbContext(optionsBuilder.Options);
}
}
@@ -0,0 +1,94 @@
// <auto-generated />
using System;
using FinlyticSentiment.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticSentiment.Migrations
{
[DbContext(typeof(SentimentDbContext))]
[Migration("20260815184006_AddDynamicSettings")]
partial class AddDynamicSettings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticSentiment.Entities.SentimentSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("EnglishWebhookUrl")
.IsRequired()
.HasMaxLength(500)
.HasColumnType("character varying(500)");
b.Property<string>("GermanWebhookUrl")
.IsRequired()
.HasMaxLength(500)
.HasColumnType("character varying(500)");
b.Property<int>("MaxBatchSize")
.HasColumnType("integer");
b.Property<double>("MinConfidenceScore")
.HasColumnType("double precision");
b.Property<int>("SweepIntervalMinutes")
.HasColumnType("integer");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("sentiment_settings", (string)null);
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,43 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticSentiment.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DynamicSettings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
ValueJson = table.Column<string>(type: "text", nullable: false),
ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DynamicSettings", x => x.Id);
});
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DynamicSettings");
}
}
}
@@ -22,6 +22,37 @@ namespace FinlyticSentiment.Migrations
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticSentiment.Entities.SentimentSettingsEntity", b =>
{
b.Property<Guid>("Id")
+12 -2
View File
@@ -1,13 +1,24 @@
using System;
using FinlyticCore.Database;
using FinlyticCore.Services;
using FinlyticSentiment.Database;
using FinlyticSentiment.Services;
using FinlyticSentiment.Util;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
var builder = Host.CreateApplicationBuilder(args);
// Register PostgreSQL DbContext for settings persistence
builder.Services.AddDbContext<SentimentDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddScoped<ISettingsDbContext>(sp => sp.GetRequiredService<SentimentDbContext>());
// Register Core Services
builder.Services.AddSingleton<ISettingsService, SettingsService>();
builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<>));
// Register HttpClient
builder.Services.AddHttpClient();
@@ -36,8 +47,7 @@ using (var scope = host.Services.CreateScope())
}
catch (Exception ex)
{
var logger = scope.ServiceProvider.GetRequiredService<ILogger<Program>>();
logger.LogError(ex, "An error occurred during database migration for FinlyticSentiment on startup.");
Console.WriteLine($"Critical error during database migration for FinlyticSentiment: {ex.Message}");
}
}
@@ -1,9 +1,13 @@
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.DependencyInjection;
namespace FinlyticSentiment.Services;
@@ -15,8 +19,6 @@ public interface IFinBertAnalyzerService
/// <summary>
/// Analyzes a news article using language-targeted FinBERT webhooks and returns structured metrics.
/// </summary>
/// <param name="article">The news article DTO to evaluate.</param>
/// <returns>A task returning the FinBERT sentiment analysis result.</returns>
Task<FinBertResultDto?> AnalyzeArticleAsync(NewsArticleDto article);
}
@@ -27,22 +29,16 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
{
private readonly HttpClient _httpClient;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<FinBertAnalyzerService> _logger;
private readonly IFinlyticLogger<FinBertAnalyzerService> _finlyticLogger;
/// <summary>
/// Initializes a new instance of the <see cref="FinBertAnalyzerService"/> class.
/// </summary>
/// <param name="httpClient">The HTTP client instance.</param>
/// <param name="scopeFactory">The service scope factory for DB access.</param>
/// <param name="logger">The logging channel.</param>
public FinBertAnalyzerService(
HttpClient httpClient,
IServiceScopeFactory scopeFactory,
ILogger<FinBertAnalyzerService> logger)
IFinlyticLogger<FinBertAnalyzerService> finlyticLogger)
{
_httpClient = httpClient;
_scopeFactory = scopeFactory;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -64,7 +60,7 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
minConfidence = settings.MinConfidenceScore;
}
_logger.LogInformation("[{Channel}] Analyzing article (ID: {Id}, Lang: {Lang}) via webhook: {Url} (MinConf: {Conf})", "SentimentChannel", article.Id, article.Language ?? "de", targetUrl, minConfidence);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] Analyzing article (ID: {Id}, Lang: {Lang}) via webhook: {Url} (MinConf: {Conf})", article.Id, article.Language ?? "de", targetUrl, minConfidence);
var requestBody = new
{
@@ -88,13 +84,11 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
using var doc = JsonDocument.Parse(content);
var root = doc.RootElement;
// Handle array response if n8n returns an array of items (e.g. [{ "json": { ... } }])
if (root.ValueKind == JsonValueKind.Array && root.GetArrayLength() > 0)
{
root = root[0];
}
// Unwrap n8n wrapper objects: "json", "output", "data", "result", "body"
if (root.ValueKind == JsonValueKind.Object)
{
if (root.TryGetProperty("json", out var jsonChild) && jsonChild.ValueKind == JsonValueKind.Object)
@@ -134,7 +128,6 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
neu = GetDoubleProp(probsElem, "neutral") ?? neu;
}
// Normalize German vs English labels
string label = rawLabel.Trim().ToUpperInvariant() switch
{
"POSITIV" or "POSITIVE" => "POSITIVE",
@@ -142,7 +135,6 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
_ => "NEUTRAL"
};
// If compoundScore is 0 but probabilities or label indicate sentiment, compute compoundScore
if (Math.Abs(compoundScore) < 0.001)
{
if (pos > 0 || neg > 0)
@@ -173,15 +165,14 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
SummarySnippet = snippet
};
}
}
}
_logger.LogWarning("[{Channel}] n8n Webhook returned non-success status: {StatusCode}. Falling back to rule analyzer.", "SentimentChannel", response.StatusCode);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] n8n Webhook returned non-success status: {StatusCode}.", response.StatusCode);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to call n8n sentiment webhook. Executing fallback sentiment analyzer.", "SentimentChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[FinBertAnalyzerService] Failed to call n8n sentiment webhook.");
}
return null;
@@ -1,9 +1,13 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticSentiment.Services;
@@ -17,53 +21,45 @@ public class SentimentBackgroundService : BackgroundService
private readonly IFinBertAnalyzerService _analyzer;
private readonly ISentimentStorageService _storage;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<SentimentBackgroundService> _logger;
private readonly IFinlyticLogger<SentimentBackgroundService> _finlyticLogger;
// In-Memory Mutex / Cache zur Vermeidung doppelter Verarbeitung (Race Conditions zwischen Broadcast & Sweep)
private static readonly ConcurrentDictionary<Guid, byte> ProcessingArticles = new();
/// <summary>
/// Initializes a new instance of the <see cref="SentimentBackgroundService"/> class.
/// </summary>
public SentimentBackgroundService(
SentimentMqttClient mqttClient,
IFinBertAnalyzerService analyzer,
ISentimentStorageService storage,
IServiceScopeFactory scopeFactory,
ILogger<SentimentBackgroundService> logger)
IFinlyticLogger<SentimentBackgroundService> finlyticLogger)
{
_mqttClient = mqttClient;
_analyzer = analyzer;
_storage = storage;
_scopeFactory = scopeFactory;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("[{Channel}] FinlyticSentiment Background Service started.", "SentimentChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinlyticSentiment Background Service started.");
// Realtime Broadcast Event registrieren (Echtzeit-Artikel)
_mqttClient.OnArticleReceived += async (article) =>
{
await ProcessSingleArticleAsync(article, stoppingToken);
};
// Kurze Initialisierungs-Verzögerung für die MQTT-Verbindung
await Task.Delay(3000, stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
int maxBatchSize;
int sweepIntervalMinutes;
int maxBatchSize = 10;
int sweepIntervalMinutes = 5;
using (var scope = _scopeFactory.CreateScope())
{
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
var settings = await settingsDb.GetSettingsAsync();
maxBatchSize = settings.MaxBatchSize;
sweepIntervalMinutes = settings.SweepIntervalMinutes;
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
maxBatchSize = await settings.GetSettingAsync(SettingKeys.MaxBatchSize, stoppingToken);
}
var interval = TimeSpan.FromMinutes(Math.Max(1, sweepIntervalMinutes));
@@ -74,19 +70,15 @@ public class SentimentBackgroundService : BackgroundService
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Normales Beenden beim Stoppen des Hosts
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Unhandled exception encountered during sentiment sweep cycle.",
"SentimentChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentBackgroundService] Unhandled exception encountered during sentiment sweep cycle.");
}
_logger.LogInformation("[{Channel}] Waiting {Minutes} minute(s) until next sentiment sweep...",
"SentimentChannel", interval.TotalMinutes);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Waiting {Minutes} minute(s) until next sentiment sweep...", interval.TotalMinutes);
// Verwendet PeriodicTimer oder CancellationToken-resistenten Delay
using var timer = new PeriodicTimer(interval);
try
{
@@ -98,27 +90,21 @@ public class SentimentBackgroundService : BackgroundService
}
}
_logger.LogInformation("[{Channel}] FinlyticSentiment Background Service is shutting down gracefully.",
"SentimentChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinlyticSentiment Background Service is shutting down gracefully.");
}
/// <summary>
/// Performs a single batch sweep of pending news articles.
/// </summary>
private async Task PerformSentimentSweepAsync(int maxBatchSize, CancellationToken cancellationToken)
{
_logger.LogInformation("[{Channel}] Starting sentiment sweep for pending news articles (Limit: {Limit})...",
"SentimentChannel", maxBatchSize);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Starting sentiment sweep for pending news articles (Limit: {Limit})...", maxBatchSize);
List<NewsArticleDto> pendingArticles = await _mqttClient.GetPendingArticlesAsync(limit: maxBatchSize);
if (pendingArticles.Count == 0)
{
_logger.LogInformation("[{Channel}] No pending news articles found in FinlyticNews.", "SentimentChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] No pending news articles found in FinlyticNews.");
return;
}
_logger.LogInformation("[{Channel}] Retrieved {Count} pending article(s) for FinBERT sentiment evaluation.",
"SentimentChannel", pendingArticles.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Retrieved {Count} pending article(s) for FinBERT sentiment evaluation.", pendingArticles.Count);
foreach (var article in pendingArticles)
{
@@ -127,42 +113,32 @@ public class SentimentBackgroundService : BackgroundService
}
}
/// <summary>
/// Processes FinBERT sentiment analysis for a single article, updates two-stage JSON summaries, and notifies FinlyticNews of status update.
/// </summary>
private async Task ProcessSingleArticleAsync(NewsArticleDto article, CancellationToken cancellationToken = default)
{
if (article == null || article.Id == Guid.Empty) return;
// Deduplizierung: Verhindert, dass derselbe Artikel zeitgleich im Sweep & im Broadcast verarbeitet wird
if (!ProcessingArticles.TryAdd(article.Id, 0))
{
_logger.LogDebug("[{Channel}] Article {Id} is already being processed. Skipping duplicate run.",
"SentimentChannel", article.Id);
await _finlyticLogger.LogDebugAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Article {Id} is already being processed. Skipping duplicate run.", article.Id);
return;
}
try
{
_logger.LogInformation("[{Channel}] Evaluating FinBERT sentiment for article: '{Title}' (ID: {Id})",
"SentimentChannel", article.Title, article.Id);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Evaluating FinBERT sentiment for article: '{Title}' (ID: {Id})", article.Title, article.Id);
var finbert = await _analyzer.AnalyzeArticleAsync(article);
if (finbert == null)
{
_logger.LogWarning(
"[{Channel}] FinBERT analysis returned NULL for article {Id} ('{Title}'). Aborting processing for this run.",
"SentimentChannel", article.Id, article.Title);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinBERT analysis returned NULL for article {Id} ('{Title}'). Aborting processing for this run.", article.Id, article.Title);
return;
}
if (cancellationToken.IsCancellationRequested) return;
// 1. Artikel-Level Sentiment speichern
await _storage.SaveArticleSentimentAsync(article, finbert);
// 2. ISIN- & Sektor-Summaries aktualisieren
if (article.MatchedAssets != null && article.MatchedAssets.Count > 0)
{
foreach (var asset in article.MatchedAssets)
@@ -187,29 +163,22 @@ public class SentimentBackgroundService : BackgroundService
if (cancellationToken.IsCancellationRequested) return;
// 3. FinlyticNews über Erfolg informieren
bool updated = await _mqttClient.UpdateArticleStatusAsync(article.Id, "Analyzed");
if (updated)
{
_logger.LogInformation(
"[{Channel}] Article sentiment processed and status set to 'Analyzed' in FinlyticNews: {Title} (ID: {Id}) -> {Label}",
"SentimentChannel", article.Title, article.Id, finbert.Label);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Article sentiment processed and status set to 'Analyzed' in FinlyticNews: {Title} (ID: {Id}) -> {Label}", article.Title, article.Id, finbert.Label);
}
else
{
_logger.LogWarning(
"[{Channel}] Failed to confirm status update to 'Analyzed' in FinlyticNews for article: {Id}",
"SentimentChannel", article.Id);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Failed to confirm status update to 'Analyzed' in FinlyticNews for article: {Id}", article.Id);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error processing sentiment for article: {Id} ({Title})",
"SentimentChannel", article.Id, article.Title);
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentBackgroundService] Error processing sentiment for article: {Id} ({Title})", article.Id, article.Title);
}
finally
{
// Lock nach der Verarbeitung immer freigeben
ProcessingArticles.TryRemove(article.Id, out _);
}
}
@@ -1,10 +1,17 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace FinlyticSentiment.Services;
@@ -13,40 +20,10 @@ namespace FinlyticSentiment.Services;
/// </summary>
public interface ISentimentStorageService
{
/// <summary>
/// Appends a new FinBERT analysis event to the ISIN summary file and recalculates the current aggregate metrics.
/// </summary>
/// <param name="isin">The asset ISIN code.</param>
/// <param name="companyName">The name of the asset company.</param>
/// <param name="sector">The sector associated with the asset.</param>
/// <param name="article">The analyzed article DTO.</param>
/// <param name="finbert">The FinBERT sentiment result.</param>
/// <returns>A task representing the file update operation.</returns>
Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert);
/// <summary>
/// Appends a new FinBERT analysis event to the Sector summary file and recalculates the sector aggregate metrics.
/// </summary>
/// <param name="sector">The target sector name.</param>
/// <param name="isin">The asset ISIN code triggering the sector update.</param>
/// <param name="articleId">The unique news article identifier.</param>
/// <param name="finbert">The FinBERT sentiment result.</param>
/// <returns>A task representing the file update operation.</returns>
Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert);
/// <summary>
/// Persists an article's sentiment analysis directly in data/summaries/articles/{articleId}.json.
/// </summary>
Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert);
/// <summary>
/// Retrieves the sentiment analysis entry for a specific news article.
/// </summary>
Task<IsinAnalysisEntry?> GetArticleSentimentAsync(string articleId);
/// <summary>
/// Retrieves the aggregate sentiment summary for a specific asset ISIN.
/// </summary>
Task<IsinSentimentSummaryDto?> GetIsinSummaryAsync(string isin);
}
@@ -54,13 +31,13 @@ public class SentimentStorageService : ISentimentStorageService
{
private static readonly ConcurrentDictionary<string, SemaphoreSlim> FileLocks = new();
private readonly ILogger<SentimentStorageService> _logger;
private readonly IFinlyticLogger<SentimentStorageService> _finlyticLogger;
private readonly string _basePath;
private readonly JsonSerializerOptions _jsonOptions;
public SentimentStorageService(IConfiguration configuration, ILogger<SentimentStorageService> logger)
public SentimentStorageService(IConfiguration configuration, IFinlyticLogger<SentimentStorageService> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_basePath = configuration["Storage:SummariesPath"] ?? "data/summaries";
Directory.CreateDirectory(Path.Combine(_basePath, "isin"));
@@ -88,30 +65,28 @@ public class SentimentStorageService : ISentimentStorageService
try
{
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
string analysisId = $"sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}";
var entry = new IsinAnalysisEntry
{
AnalysisId = analysisId,
AnalysisId = $"sent_{Guid.NewGuid():N}",
Timestamp = nowIso,
Article = new IsinAnalysisArticleRef
{
ArticleId = cleanId,
Title = article.Title ?? "",
Source = article.Author ?? "FinlyticNews",
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ")
Title = article.Title,
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
Source = article.Author ?? "FinlyticNews"
},
FinbertResult = finbert,
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? article.Title ?? ""
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
};
string json = JsonSerializer.Serialize(entry, _jsonOptions);
var json = JsonSerializer.Serialize(entry, _jsonOptions);
await File.WriteAllTextAsync(filePath, json);
_logger.LogInformation("[{Channel}] Successfully saved article sentiment file: {Path}", "SentimentChannel", filePath);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Successfully saved article sentiment file: {Path}", filePath);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to write article sentiment file: {Path}", "SentimentChannel", filePath);
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to write article sentiment file: {Path}", filePath);
}
finally
{
@@ -124,10 +99,9 @@ public class SentimentStorageService : ISentimentStorageService
{
if (string.IsNullOrWhiteSpace(articleId)) return null;
var cleanId = articleId.Trim();
var articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
string cleanId = articleId.Trim();
string articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
// 1. Primärer Lookup
if (File.Exists(articleFilePath))
{
var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1));
@@ -139,7 +113,7 @@ public class SentimentStorageService : ISentimentStorageService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to read article sentiment file: {Path}", "SentimentChannel", articleFilePath);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to read article sentiment file: {Path}", articleFilePath);
}
finally
{
@@ -147,22 +121,6 @@ public class SentimentStorageService : ISentimentStorageService
}
}
// 2. Fallback in ISIN-Dateien
var dirPath = Path.Combine(_basePath, "isin");
if (!Directory.Exists(dirPath)) return null;
foreach (var file in Directory.GetFiles(dirPath, "*.json"))
{
try
{
var json = await File.ReadAllTextAsync(file);
var doc = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, _jsonOptions);
var match = doc?.Analyses?.FirstOrDefault(a => string.Equals(a.Article?.ArticleId?.Trim(), cleanId, StringComparison.OrdinalIgnoreCase));
if (match != null) return match;
}
catch { }
}
return null;
}
@@ -171,8 +129,8 @@ public class SentimentStorageService : ISentimentStorageService
{
if (string.IsNullOrWhiteSpace(isin)) return null;
var cleanIsin = isin.Trim();
var filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
string cleanIsin = isin.Trim().ToUpperInvariant();
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
if (!File.Exists(filePath)) return null;
@@ -186,7 +144,7 @@ public class SentimentStorageService : ISentimentStorageService
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error reading ISIN summary file: {Path}", "SentimentChannel", filePath);
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Error reading ISIN summary file: {Path}", filePath);
return null;
}
finally
@@ -198,9 +156,9 @@ public class SentimentStorageService : ISentimentStorageService
/// <inheritdoc />
public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert)
{
if (string.IsNullOrWhiteSpace(isin) || article == null) return;
if (string.IsNullOrWhiteSpace(isin)) return;
string cleanIsin = isin.Trim();
string cleanIsin = isin.Trim().ToUpperInvariant();
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
@@ -208,87 +166,92 @@ public class SentimentStorageService : ISentimentStorageService
try
{
IsinSentimentSummaryDto isinDoc;
var analyses = new List<IsinAnalysisEntry>();
if (File.Exists(filePath))
{
try
var existingJson = await File.ReadAllTextAsync(filePath);
var existing = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(existingJson, _jsonOptions);
if (existing?.Analyses != null)
{
string json = await File.ReadAllTextAsync(filePath);
isinDoc = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, _jsonOptions) ?? new IsinSentimentSummaryDto { Isin = cleanIsin };
}
catch
{
isinDoc = new IsinSentimentSummaryDto { Isin = cleanIsin };
analyses.AddRange(existing.Analyses);
}
}
else
{
isinDoc = new IsinSentimentSummaryDto { Isin = cleanIsin };
}
string cleanArticleId = article.Id.ToString();
analyses.RemoveAll(a => string.Equals(a.Article?.ArticleId, cleanArticleId, StringComparison.OrdinalIgnoreCase));
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
string analysisId = $"sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}";
var newEntry = new IsinAnalysisEntry
{
AnalysisId = analysisId,
AnalysisId = $"sent_{Guid.NewGuid():N}",
Timestamp = nowIso,
Article = new IsinAnalysisArticleRef
{
ArticleId = article.Id.ToString(),
Title = article.Title ?? "",
Source = article.Author ?? "FinlyticNews",
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ")
ArticleId = cleanArticleId,
Title = article.Title,
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
Source = article.Author ?? "FinlyticNews"
},
FinbertResult = finbert,
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? article.Title ?? ""
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
};
// Duplikat-Bereinigung: Falls Artikel bereits existiert, alten Eintrag entfernen!
var updatedAnalyses = isinDoc.Analyses?
.Where(a => !string.Equals(a.Article?.ArticleId, article.Id.ToString(), StringComparison.OrdinalIgnoreCase))
.ToList() ?? new List<IsinAnalysisEntry>();
analyses.Add(newEntry);
// Neuen Eintrag oben einfügen
updatedAnalyses.Insert(0, newEntry);
// Capping: Maximal die letzten 100 Analysen aufheben (verhindert gigantische JSON-Dateien)
if (updatedAnalyses.Count > 100)
var cutoff = DateTime.UtcNow.AddDays(-14);
var validAnalyses = analyses.Where(a =>
{
updatedAnalyses = updatedAnalyses.Take(100).ToList();
if (DateTime.TryParse(a.Article?.PublishedAt ?? a.Timestamp, out var pubDate))
{
return pubDate >= cutoff;
}
return true;
}).ToList();
var updatedAnalyses = validAnalyses.OrderByDescending(a => a.Article?.PublishedAt ?? a.Timestamp).Take(50).ToList();
double totalCompound = 0.0;
double totalConf = 0.0;
foreach (var item in updatedAnalyses)
{
if (item.FinbertResult == null) continue;
totalCompound += item.FinbertResult.CompoundScore;
totalConf += item.FinbertResult.Confidence;
}
double avgCompound = updatedAnalyses.Average(a => a.FinbertResult.CompoundScore);
double avgConf = updatedAnalyses.Average(a => a.FinbertResult.Confidence);
string label = CalculateLabel(avgCompound);
int total = updatedAnalyses.Count;
double avgCompound = total > 0 ? totalCompound / total : 0.0;
double avgConf = total > 0 ? totalConf / total : 0.0;
string textSummary = label switch
{
"POSITIVE" => $"Die Stimmungsanalyse zeigt einen weiterhin positiven Trend ({avgCompound:F2}). Hauptursache sind positive Berichte und starke Markt-Signale.",
"NEGATIVE" => $"Die Stimmungsanalyse deutet auf einen verhaltenen bis negativen Trend hin ({avgCompound:F2}). Auf kritische Markt-Berichte sollte geachtet werden.",
_ => $"Das Gesamtsentiment ist neutral ({avgCompound:F2}). Ausgewogene Signale aus der aktuellen Berichterstattung."
};
string overallLabel = "NEUTRAL";
if (avgCompound >= 0.15) overallLabel = "POSITIVE";
else if (avgCompound <= -0.15) overallLabel = "NEGATIVE";
var updatedDoc = isinDoc with
var summary = new IsinSentimentSummaryDto
{
Isin = cleanIsin,
CompanyName = !string.IsNullOrWhiteSpace(companyName) ? companyName : isinDoc.CompanyName,
Sector = !string.IsNullOrWhiteSpace(sector) ? sector : isinDoc.Sector,
CompanyName = companyName,
Sector = sector,
LastUpdated = nowIso,
CurrentSummary = new IsinCurrentSummary
{
CompoundScore = Math.Round(avgCompound, 2),
SentimentLabel = label,
AvgConfidence = Math.Round(avgConf, 2),
TotalArticlesAnalyzed = updatedAnalyses.Count,
Text = textSummary
CompoundScore = Math.Round(avgCompound, 4),
SentimentLabel = overallLabel,
AvgConfidence = Math.Round(avgConf, 4),
TotalArticlesAnalyzed = total,
Text = $"Synthesized sentiment across {total} articles is {overallLabel}."
},
Analyses = updatedAnalyses
};
await File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(updatedDoc, _jsonOptions));
_logger.LogInformation("[{Channel}] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", "SentimentChannel", filePath, updatedAnalyses.Count, avgCompound);
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
await File.WriteAllTextAsync(filePath, outJson);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", filePath, updatedAnalyses.Count, avgCompound);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update ISIN summary for {Isin}", cleanIsin);
}
finally
{
@@ -301,95 +264,82 @@ public class SentimentStorageService : ISentimentStorageService
{
if (string.IsNullOrWhiteSpace(sector)) return;
string sanitizedSector = string.Concat(sector.Split(Path.GetInvalidFileNameChars())).Trim();
string filePath = Path.Combine(_basePath, "sectors", $"{sanitizedSector}.json");
string cleanSector = sector.Trim().ToLowerInvariant();
string filePath = Path.Combine(_basePath, "sectors", $"{cleanSector}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
SectorSentimentSummaryDto sectorDoc;
var analyses = new List<SectorAnalysisEntry>();
if (File.Exists(filePath))
{
try
var existingJson = await File.ReadAllTextAsync(filePath);
var existing = JsonSerializer.Deserialize<SectorSentimentSummaryDto>(existingJson, _jsonOptions);
if (existing?.Analyses != null)
{
string json = await File.ReadAllTextAsync(filePath);
sectorDoc = JsonSerializer.Deserialize<SectorSentimentSummaryDto>(json, _jsonOptions) ?? new SectorSentimentSummaryDto { Sector = sector };
analyses.AddRange(existing.Analyses);
}
catch
{
sectorDoc = new SectorSentimentSummaryDto { Sector = sector };
}
}
else
{
sectorDoc = new SectorSentimentSummaryDto { Sector = sector };
}
string cleanIsin = isin.Trim().ToUpperInvariant();
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
string analysisId = $"sec_sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}";
var newEntry = new SectorAnalysisEntry
analyses.RemoveAll(a => string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase) && string.Equals(a.RelatedIsin, cleanIsin, StringComparison.OrdinalIgnoreCase));
analyses.Add(new SectorAnalysisEntry
{
AnalysisId = analysisId,
AnalysisId = $"sec_{Guid.NewGuid():N}",
Timestamp = nowIso,
RelatedIsin = isin,
RelatedIsin = cleanIsin,
ArticleId = articleId,
FinbertResult = finbert
};
});
// Duplikate bereinigen (selber Artikel für denselben Sektor)
var updatedAnalyses = sectorDoc.Analyses?
.Where(a => !string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase))
.ToList() ?? new List<SectorAnalysisEntry>();
updatedAnalyses.Insert(0, newEntry);
if (updatedAnalyses.Count > 100)
var cutoff = DateTime.UtcNow.AddDays(-14);
var updatedAnalyses = analyses.Where(a =>
{
updatedAnalyses = updatedAnalyses.Take(100).ToList();
}
if (DateTime.TryParse(a.Timestamp, out var ts))
{
return ts >= cutoff;
}
return true;
}).OrderByDescending(a => a.Timestamp).Take(100).ToList();
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrWhiteSpace(i)).Distinct().ToList();
double avgCompound = updatedAnalyses.Average(a => a.FinbertResult.CompoundScore);
string label = CalculateLabel(avgCompound);
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrEmpty(i)).Distinct().ToList();
double totalSectorCompound = updatedAnalyses.Sum(s => s.FinbertResult.CompoundScore);
double avgSectorCompound = updatedAnalyses.Count > 0 ? totalSectorCompound / updatedAnalyses.Count : 0.0;
string overviewText = label switch
{
"POSITIVE" => $"Der Sektor {sector} tendiert insgesamt positiv. Starke Einzelergebnisse stützen den Trend.",
"NEGATIVE" => $"Der Sektor {sector} verzeichnet dämpfende Sentiment-Signale.",
_ => $"Der Sektor {sector} zeigt ein ausgewogenes neutrales Gesamtbild."
};
string sectorLabel = "NEUTRAL";
if (avgSectorCompound >= 0.15) sectorLabel = "POSITIVE";
else if (avgSectorCompound <= -0.15) sectorLabel = "NEGATIVE";
var updatedDoc = sectorDoc with
var summary = new SectorSentimentSummaryDto
{
Sector = sector,
LastUpdated = nowIso,
CurrentSummary = new SectorCurrentSummary
{
CompoundScore = Math.Round(avgCompound, 2),
SentimentLabel = label,
CompoundScore = Math.Round(avgSectorCompound, 4),
SentimentLabel = sectorLabel,
ActiveIsins = activeIsins,
Text = overviewText
Text = $"Sector {sector} aggregate sentiment: {sectorLabel}."
},
Analyses = updatedAnalyses
};
await File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(updatedDoc, _jsonOptions));
_logger.LogInformation("[{Channel}] Updated Sector summary file: {Path} (Active ISINs: {Count})", "SentimentChannel", filePath, activeIsins.Count);
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
await File.WriteAllTextAsync(filePath, outJson);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated Sector summary file: {Path} (Active ISINs: {Count})", filePath, activeIsins.Count);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update Sector summary for {Sector}", sector);
}
finally
{
fileLock.Release();
}
}
private static string CalculateLabel(double score) => score switch
{
>= 0.15 => "POSITIVE",
<= -0.15 => "NEGATIVE",
_ => "NEUTRAL"
};
}
+141 -146
View File
@@ -1,10 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Models;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticSentiment.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@@ -21,9 +28,6 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
private readonly IConfiguration _configuration;
private readonly IServiceScopeFactory _scopeFactory;
/// <summary>
/// Initializes a new instance of the <see cref="SentimentMqttClient"/> class.
/// </summary>
public SentimentMqttClient(
ILogger<SentimentMqttClient> logger,
IConfiguration configuration,
@@ -43,12 +47,10 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
{
Host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost",
Port = Convert.ToInt32(_configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883"),
ClientId =
$"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticSentiment")}_{Guid.NewGuid()}"
ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticSentiment")}_{Guid.NewGuid()}"
};
_logger.LogInformation("[{Channel}] Starting Sentiment MQTT client. Host: {Host}, ClientId: {ClientId}",
"SentimentChannel", config.Host, config.ClientId);
_logger.LogInformation("Starting Sentiment MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId);
await ConnectAsync(config);
}
@@ -57,7 +59,7 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
/// </summary>
public async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("[{Channel}] Stopping Sentiment MQTT client and disconnecting.", "SentimentChannel");
_logger.LogInformation("Stopping Sentiment MQTT client and disconnecting.");
await DisconnectAsync();
}
@@ -69,16 +71,24 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
/// <inheritdoc />
protected override async Task OnConnectedAsync()
{
_logger.LogInformation(
"[{Channel}] Sentiment MQTT client connected. Subscribing to RPC response and broadcast topics...",
"SentimentChannel");
_logger.LogInformation("Sentiment MQTT client connected. Subscribing to RPC response and broadcast topics...");
await SubscribeAsync("services/response/#");
await SubscribeAsync("services/news/completed");
await SubscribeAsync("services/request/sentiment_GetArticle/#");
await SubscribeAsync("services/request/sentiment_GetIsin/#");
await SubscribeAsync("services/request/sentiment_Analyze/#");
await SubscribeAsync("services/request/sentiment_settings_GetAll/#");
await SubscribeAsync("services/request/sentiment_settings_Update/#");
await SubscribeAsync("services/request/health_Ping/#");
await SubscribeAsync("services/config/updated/#");
FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
{
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticSentiment", StringComparison.OrdinalIgnoreCase))
{
await PublishAsync("finlytic/logs/FinlyticSentiment", logDto);
}
};
}
/// <inheritdoc />
@@ -86,14 +96,12 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
{
if (string.IsNullOrWhiteSpace(topic)) return;
// 1. Config update events
if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase))
{
if (topic.EndsWith("FinlyticSentiment", StringComparison.OrdinalIgnoreCase))
{
await OnConfigUpdatedAsync(payload);
}
return;
}
@@ -103,13 +111,11 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
return;
}
// Extract correlationId from topic suffix (e.g. services/request/sentiment_GetArticle/{correlationId})
var lastSlash = topic.LastIndexOf('/');
if (lastSlash < 0 || lastSlash >= topic.Length - 1) return;
var correlationId = topic.Substring(lastSlash + 1);
// 2. Dispatch to specific channel handlers
if (topic.StartsWith("services/request/sentiment_GetArticle", StringComparison.OrdinalIgnoreCase))
{
await OnSentimentGetArticleAsync(payload, correlationId);
@@ -122,178 +128,205 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
{
await OnSentimentAnalyzeAsync(payload, correlationId);
}
else if (topic.StartsWith("services/request/sentiment_settings_GetAll", StringComparison.OrdinalIgnoreCase))
{
await OnSettingsGetAllAsync(correlationId);
}
else if (topic.StartsWith("services/request/sentiment_settings_Update", StringComparison.OrdinalIgnoreCase))
{
await OnSettingsUpdateAsync(payload, correlationId);
}
else if (topic.StartsWith("services/request/health_Ping", StringComparison.OrdinalIgnoreCase))
{
await OnHealthPingAsync(topic, correlationId);
}
}
/// <summary>
/// Handles dynamic service config update events.
/// </summary>
private async Task OnSettingsGetAllAsync(string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
try
{
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/sentiment_settings_GetAll/{correlationId}";
await PublishAsync(responseTopic, settings);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticSentiment] [Settings_GetAll] Failed to retrieve settings.");
}
}
private async Task OnSettingsUpdateAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
try
{
Dictionary<string, object?>? updates = null;
try
{
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
}
catch
{
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
if (list != null)
{
updates = new Dictionary<string, object?>();
foreach (var item in list) updates[item.Key] = item.Value;
}
}
if (updates != null && updates.Count > 0)
{
await settingsService.UpdateSettingsAsync(updates);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
}
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/sentiment_settings_Update/{correlationId}";
await PublishAsync(responseTopic, currentSettings);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticSentiment] [Settings_Update] Failed to update settings.");
}
}
private async Task OnConfigUpdatedAsync(string payload)
{
_logger.LogInformation("[{Channel}] [SentimentMqttClient] Received config update event for FinlyticSentiment.",
"SentimentChannel");
try
{
using var doc = JsonDocument.Parse(payload);
if (doc.RootElement.TryGetProperty("settings", out var settingsProp))
{
var dict = JsonSerializer.Deserialize(settingsProp.GetRawText(), typeof(Dictionary<string, string>),
FinlyticJsonSerializerContext.Default) as Dictionary<string, string>;
var dict = JsonSerializer.Deserialize<Dictionary<string, object?>>(settingsProp.GetRawText());
if (dict != null && dict.Count > 0)
{
using var scope = _scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsDb.UpdateSettingsFromDictionaryAsync(dict);
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Successfully persisted {Count} updated settings for FinlyticSentiment.",
"SentimentChannel", dict.Count);
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await settings.UpdateSettingsAsync(dict);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [SentimentMqttClient] Error processing MQTT config update event.",
"SentimentChannel");
}
catch { }
}
/// <summary>
/// Handles health_Ping RPC requests.
/// </summary>
private async Task OnHealthPingAsync(string topic, string correlationId)
{
if (topic.Contains("FinlyticSentiment", StringComparison.OrdinalIgnoreCase) ||
!topic.Contains("/", StringComparison.OrdinalIgnoreCase))
{
string respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic,
new ServiceHealthResponse("FinlyticSentiment", "Online", DateTime.UtcNow, "Connected"));
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].",
"SentimentChannel", correlationId);
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticSentiment", "Online", DateTime.UtcNow, "Connected"));
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[FinlyticSentiment] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
}
}
/// <summary>
/// Handles broadcasted articles on services/news/completed.
/// </summary>
private async Task OnNewsCompletedAsync(string payload)
{
if (string.IsNullOrWhiteSpace(payload)) return;
try
{
var article =
JsonSerializer.Deserialize(payload, typeof(NewsArticleDto), FinlyticJsonSerializerContext.Default) as
NewsArticleDto;
var article = JsonSerializer.Deserialize(payload, typeof(NewsArticleDto), FinlyticJsonSerializerContext.Default) as NewsArticleDto;
if (article != null && article.Id != Guid.Empty && OnArticleReceived != null)
{
_logger.LogInformation(
"[{Channel}] Received real-time article broadcast on services/news/completed: {Title} (ID: {Id})",
"SentimentChannel", article.Title, article.Id);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "Received real-time article broadcast on services/news/completed: {Title} (ID: {Id})", article.Title, article.Id);
await OnArticleReceived.Invoke(article);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error parsing broadcasted article on services/news/completed.",
"SentimentChannel");
}
catch { }
}
/// <summary>
/// Handles sentiment_GetArticle RPC requests using source-generated DTO deserialization.
/// </summary>
private async Task OnSentimentGetArticleAsync(string payload, string correlationId)
{
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Processing RPC sentiment_GetArticle request [CorrelationId: {CorrelationId}]",
"SentimentChannel", correlationId);
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_GetArticle request [CorrelationId: {CorrelationId}]", correlationId);
try
{
var request =
JsonSerializer.Deserialize(payload, typeof(ArticleRequest), FinlyticJsonSerializerContext.Default) as
ArticleRequest;
var request = JsonSerializer.Deserialize(payload, typeof(ArticleRequest), FinlyticJsonSerializerContext.Default) as ArticleRequest;
var articleId = request?.ArticleId ?? request?.Id;
if (string.IsNullOrWhiteSpace(articleId))
{
_logger.LogWarning("[{Channel}] [SentimentMqttClient] Missing articleId in request payload.",
"SentimentChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing articleId in request payload.");
return;
}
using var scope = _scopeFactory.CreateScope();
var storageService = scope.ServiceProvider.GetRequiredService<ISentimentStorageService>();
var sentimentEntry = await storageService.GetArticleSentimentAsync(articleId);
string responseTopic = $"services/response/sentiment_GetArticle/{correlationId}";
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Publishing RPC sentiment_GetArticle response for article {ArticleId} to {ResponseTopic}",
"SentimentChannel", articleId, responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_GetArticle response for article {ArticleId} to {ResponseTopic}", articleId, responseTopic);
await PublishAsync(responseTopic, sentimentEntry);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[{Channel}] [SentimentMqttClient] Failed to process sentiment_GetArticle RPC request.",
"SentimentChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_GetArticle RPC request.");
}
}
/// <summary>
/// Handles sentiment_GetIsin RPC requests using source-generated DTO deserialization.
/// </summary>
private async Task OnSentimentGetIsinAsync(string payload, string correlationId)
{
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Processing RPC sentiment_GetIsin request [CorrelationId: {CorrelationId}]",
"SentimentChannel", correlationId);
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_GetIsin request [CorrelationId: {CorrelationId}]", correlationId);
try
{
var request =
JsonSerializer.Deserialize(payload, typeof(IsinRequest), FinlyticJsonSerializerContext.Default) as
IsinRequest;
var request = JsonSerializer.Deserialize(payload, typeof(IsinRequest), FinlyticJsonSerializerContext.Default) as IsinRequest;
var isin = request?.Isin;
if (string.IsNullOrWhiteSpace(isin))
{
_logger.LogWarning("[{Channel}] [SentimentMqttClient] Missing ISIN in request payload.",
"SentimentChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing ISIN in request payload.");
return;
}
using var scope = _scopeFactory.CreateScope();
var storageService = scope.ServiceProvider.GetRequiredService<ISentimentStorageService>();
var isinSummary = await storageService.GetIsinSummaryAsync(isin);
string responseTopic = $"services/response/sentiment_GetIsin/{correlationId}";
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Publishing RPC sentiment_GetIsin response for ISIN {Isin} to {ResponseTopic}",
"SentimentChannel", isin, responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_GetIsin response for ISIN {Isin} to {ResponseTopic}", isin, responseTopic);
await PublishAsync(responseTopic, isinSummary);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [SentimentMqttClient] Failed to process sentiment_GetIsin RPC request.",
"SentimentChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_GetIsin RPC request.");
}
}
/// <summary>
/// Handles manual/forced sentiment_Analyze RPC requests using existing analyzer and storage services.
/// </summary>
private async Task OnSentimentAnalyzeAsync(string payload, string correlationId)
{
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Processing RPC sentiment_Analyze request [CorrelationId: {CorrelationId}]",
"SentimentChannel", correlationId);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_Analyze request [CorrelationId: {CorrelationId}]", correlationId);
string responseTopic = $"services/response/sentiment_Analyze/{correlationId}";
if (string.IsNullOrWhiteSpace(payload))
@@ -304,33 +337,24 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
try
{
var request =
JsonSerializer.Deserialize(payload, typeof(AnalyzeSentimentRequest),
FinlyticJsonSerializerContext.Default) as AnalyzeSentimentRequest;
var request = JsonSerializer.Deserialize(payload, typeof(AnalyzeSentimentRequest), FinlyticJsonSerializerContext.Default) as AnalyzeSentimentRequest;
if (request == null ||
(string.IsNullOrWhiteSpace(request.ArticleId) && string.IsNullOrWhiteSpace(request.Isin)))
if (request == null || (string.IsNullOrWhiteSpace(request.ArticleId) && string.IsNullOrWhiteSpace(request.Isin)))
{
_logger.LogWarning(
"[{Channel}] [SentimentMqttClient] Missing ArticleId or Isin in sentiment_Analyze request payload.",
"SentimentChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing ArticleId or Isin in sentiment_Analyze request payload.");
await PublishAsync(responseTopic, (object?)null);
return;
}
using var scope = _scopeFactory.CreateScope();
var storageService = scope.ServiceProvider.GetRequiredService<ISentimentStorageService>();
var analyzerService = scope.ServiceProvider.GetRequiredService<IFinBertAnalyzerService>();
object? result = null;
// Fall 1: Manuelle Analyse für einen einzelnen Artikel
if (!string.IsNullOrWhiteSpace(request.ArticleId))
{
var cleanArticleId = request.ArticleId.Trim();
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Processing article analysis for ArticleId: {ArticleId} (ForceReload: {ForceReload})",
"SentimentChannel", cleanArticleId, request.ForceReload);
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing article analysis for ArticleId: {ArticleId} (ForceReload: {ForceReload})", cleanArticleId, request.ForceReload);
IsinAnalysisEntry? existingEntry = null;
@@ -345,7 +369,6 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
}
else
{
// Artikel per RPC von FinlyticNews abfragen
if (Guid.TryParse(cleanArticleId, out var articleGuid))
{
var article = await SendRpcRequestAsync<NewsArticleDto, ArticleRequest>(
@@ -357,17 +380,13 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
{
var finbertResult = await analyzerService.AnalyzeArticleAsync(article);
// 🎯 NULL-CHECK: Falls Analyse fehlschlägt/null liefert -> abbrechen
if (finbertResult == null)
{
_logger.LogWarning(
"[{Channel}] [SentimentMqttClient] FinBERT analysis returned NULL for article {ArticleId}. Aborting manual analysis.",
"SentimentChannel", cleanArticleId);
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] FinBERT analysis returned NULL for article {ArticleId}. Aborting manual analysis.", cleanArticleId);
await PublishAsync(responseTopic, (object?)null);
return;
}
// Speichern & Summaries aktualisieren
await storageService.SaveArticleSentimentAsync(article, finbertResult);
if (article.MatchedAssets != null && article.MatchedAssets.Count > 0)
@@ -391,86 +410,62 @@ public class SentimentMqttClient : ManagedMqttClient, IHostedService
}
}
// FinlyticNews über Re-Analyse informieren
await UpdateArticleStatusAsync(article.Id, "Analyzed");
result = await storageService.GetArticleSentimentAsync(cleanArticleId);
}
else
{
_logger.LogWarning(
"[{Channel}] [SentimentMqttClient] Could not retrieve article {ArticleId} from FinlyticNews for re-analysis.",
"SentimentChannel", cleanArticleId);
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Could not retrieve article {ArticleId} from FinlyticNews for re-analysis.", cleanArticleId);
}
}
}
}
// Fall 2: ISIN Gesamtsummary anfordern
else if (!string.IsNullOrWhiteSpace(request.Isin))
{
var cleanIsin = request.Isin.Trim();
_logger.LogInformation("[{Channel}] [SentimentMqttClient] Fetching sentiment summary for ISIN: {Isin}",
"SentimentChannel", cleanIsin);
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Fetching sentiment summary for ISIN: {Isin}", cleanIsin);
result = await storageService.GetIsinSummaryAsync(cleanIsin);
}
_logger.LogInformation(
"[{Channel}] [SentimentMqttClient] Publishing RPC sentiment_Analyze response to {ResponseTopic}",
"SentimentChannel", responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_Analyze response to {ResponseTopic}", responseTopic);
await PublishAsync(responseTopic, result);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] [SentimentMqttClient] Failed to process sentiment_Analyze RPC request.",
"SentimentChannel");
await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_Analyze RPC request.");
await PublishAsync(responseTopic, (object?)null);
}
}
/// <summary>
/// Requests pending news articles from FinlyticNews via MQTT RPC.
/// </summary>
/// <param name="limit">The maximum number of articles to request (capped at 10).</param>
/// <returns>A list of pending news article DTOs.</returns>
public async Task<List<NewsArticleDto>> GetPendingArticlesAsync(int limit = 10)
{
try
{
var payload = new LimitRequest(Math.Min(limit, 10));
var articles =
await SendRpcRequestAsync<List<NewsArticleDto>, LimitRequest>("news_GetPending", payload,
TimeSpan.FromSeconds(10));
var articles = await SendRpcRequestAsync<List<NewsArticleDto>, LimitRequest>("news_GetPending", payload, TimeSpan.FromSeconds(10));
return articles ?? [];
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error executing MQTT RPC for news_GetPending.", "SentimentChannel");
_logger.LogError(ex, "Error executing MQTT RPC for news_GetPending.");
}
return [];
}
/// <summary>
/// Dispatches an RPC request to update the article status in FinlyticNews (e.g. to "Analyzed").
/// </summary>
/// <param name="id">The article identifier.</param>
/// <param name="status">The target status string (default "Analyzed").</param>
/// <returns>True if the status update succeeded; otherwise, false.</returns>
public async Task<bool> UpdateArticleStatusAsync(Guid id, string status = "Analyzed")
{
try
{
var request = new UpdateNewsStatusRequest(id, status);
var response =
await SendRpcRequestAsync<UpdateNewsStatusResponse, UpdateNewsStatusRequest>("news_UpdateStatus",
request, TimeSpan.FromSeconds(8));
var response = await SendRpcRequestAsync<UpdateNewsStatusResponse, UpdateNewsStatusRequest>("news_UpdateStatus", request, TimeSpan.FromSeconds(8));
return response?.Success ?? false;
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error executing MQTT RPC for news_UpdateStatus (ID: {Id}).",
"SentimentChannel", id);
_logger.LogError(ex, "Error executing MQTT RPC for news_UpdateStatus (ID: {Id}).", id);
}
return false;
+19
View File
@@ -0,0 +1,19 @@
using FinlyticCore.Models.Settings;
namespace FinlyticSentiment.Util;
public static class SettingKeys
{
// --- Logging-Kanäle ---
public static readonly SettingKey<bool> SentimentChannel = new("Logging.Channel.Sentiment", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
// --- FinBERT & Modell-Parameter ---
public static readonly SettingKey<int> MaxBatchSize = new("FinBert.MaxBatchSize", 10);
public static readonly SettingKey<double> MinimumConfidenceThreshold = new("FinBert.MinConfidenceThreshold", 0.60);
public static readonly SettingKey<int> TimeoutSeconds = new("FinBert.TimeoutSeconds", 30);
public static readonly SettingKey<int> SentimentWindowDays = new("Sentiment.WindowDays", 14);
public static readonly SettingKey<double> DecayFactorPerDay = new("Sentiment.DecayFactorPerDay", 0.90);
public static readonly SettingKey<bool> EnableAutoSummarization = new("Feature.EnableAutoSummarization", true);
}
@@ -1,10 +1,12 @@
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using FinlyticTechnicalAnalysis.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticTechnicalAnalysis.Database;
public class TechnicalAnalysisDbContext : DbContext
public class TechnicalAnalysisDbContext : DbContext, ISettingsDbContext
{
public TechnicalAnalysisDbContext(DbContextOptions<TechnicalAnalysisDbContext> options) : base(options)
{
@@ -23,7 +25,7 @@ public class TechnicalAnalysisDbContext : DbContext
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key);
entity.HasIndex(e => e.Key).IsUnique();
});
modelBuilder.Entity<MarketCandleEntity>()
@@ -34,3 +36,13 @@ public class TechnicalAnalysisDbContext : DbContext
.HasIndex(c => c.Isin);
}
}
public class TechnicalAnalysisDbContextFactory : IDesignTimeDbContextFactory<TechnicalAnalysisDbContext>
{
public TechnicalAnalysisDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<TechnicalAnalysisDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=ta;Username=postgres;Password=postgres");
return new TechnicalAnalysisDbContext(optionsBuilder.Options);
}
}
@@ -0,0 +1,193 @@
// <auto-generated />
using System;
using FinlyticTechnicalAnalysis.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticTechnicalAnalysis.Migrations
{
[DbContext(typeof(TechnicalAnalysisDbContext))]
[Migration("20260815183955_AddDynamicSettings")]
partial class AddDynamicSettings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticTechnicalAnalysis.Entities.CachedAnalysisEntity", b =>
{
b.Property<string>("Isin")
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<string>("AnalysisJson")
.IsRequired()
.HasColumnType("jsonb");
b.Property<DateTime>("CalculatedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("Ticker")
.IsRequired()
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.HasKey("Isin");
b.HasIndex("Isin");
b.ToTable("CachedAnalyses");
});
modelBuilder.Entity("FinlyticTechnicalAnalysis.Entities.MacroDataEntity", b =>
{
b.Property<string>("Symbol")
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<DateTime>("LastUpdatedAt")
.HasColumnType("timestamp with time zone");
b.Property<decimal>("PreviousClose")
.HasColumnType("decimal(18, 6)");
b.Property<string>("TrendState")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<decimal>("Value")
.HasColumnType("decimal(18, 6)");
b.HasKey("Symbol");
b.ToTable("MacroData");
});
modelBuilder.Entity("FinlyticTechnicalAnalysis.Entities.MarketCandleEntity", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id"));
b.Property<decimal?>("Ask")
.HasColumnType("decimal(18, 6)");
b.Property<decimal?>("Bid")
.HasColumnType("decimal(18, 6)");
b.Property<decimal>("Close")
.HasColumnType("decimal(18, 6)");
b.Property<decimal>("High")
.HasColumnType("decimal(18, 6)");
b.Property<string>("Interval")
.IsRequired()
.HasMaxLength(10)
.HasColumnType("character varying(10)");
b.Property<decimal>("Low")
.HasColumnType("decimal(18, 6)");
b.Property<decimal>("Open")
.HasColumnType("decimal(18, 6)");
b.Property<string>("Symbol")
.IsRequired()
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp with time zone");
b.Property<long>("Volume")
.HasColumnType("bigint");
b.HasKey("Id");
b.HasIndex("Symbol", "Interval", "Timestamp")
.IsUnique();
b.ToTable("MarketCandles");
});
modelBuilder.Entity("FinlyticTechnicalAnalysis.Entities.TaSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("EmaShortPeriod")
.HasColumnType("integer");
b.Property<double>("RsiOverboughtLimit")
.HasColumnType("double precision");
b.Property<double>("RsiOversoldLimit")
.HasColumnType("double precision");
b.Property<int>("SmaLongPeriod")
.HasColumnType("integer");
b.Property<int>("SmaMediumPeriod")
.HasColumnType("integer");
b.Property<double>("SupertrendMultiplier")
.HasColumnType("double precision");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,43 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticTechnicalAnalysis.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DynamicSettings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
ValueJson = table.Column<string>(type: "text", nullable: false),
ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DynamicSettings", x => x.Id);
});
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DynamicSettings");
}
}
}
@@ -22,6 +22,37 @@ namespace FinlyticTechnicalAnalysis.Migrations
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticTechnicalAnalysis.Entities.CachedAnalysisEntity", b =>
{
b.Property<string>("Isin")
+8 -3
View File
@@ -1,4 +1,6 @@
using System;
using FinlyticCore.Database;
using FinlyticCore.Services;
using FinlyticCore.Services.TradeRepublic;
using FinlyticCore.Services.Yahoo;
using FinlyticTechnicalAnalysis.Database;
@@ -8,13 +10,17 @@ using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
var builder = Host.CreateApplicationBuilder(args);
// Register DB Context
builder.Services.AddDbContext<TechnicalAnalysisDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddScoped<ISettingsDbContext>(sp => sp.GetRequiredService<TechnicalAnalysisDbContext>());
// Register Core Services & Logger
builder.Services.AddSingleton<ISettingsService, SettingsService>();
builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<>));
// Register HTTP Clients
builder.Services.AddHttpClient<IYahooMarketDataScraper, YahooMarketDataScraper>()
@@ -54,8 +60,7 @@ using (var scope = host.Services.CreateScope())
}
catch (Exception ex)
{
var logger = scope.ServiceProvider.GetRequiredService<ILogger<Program>>();
logger.LogError(ex, "[{Channel}] An error occurred during database migration on startup.", "TechnicalAnalysisChannel");
Console.WriteLine($"Critical error during database migration: {ex.Message}");
}
}
@@ -6,12 +6,13 @@ using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.TechnicalAnalysis;
using FinlyticCore.Services;
using FinlyticCore.Services.TradeRepublic;
using FinlyticTechnicalAnalysis.Database;
using FinlyticTechnicalAnalysis.Entities;
using FinlyticTechnicalAnalysis.Util;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FinlyticTechnicalAnalysis.Services;
@@ -29,7 +30,7 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
private readonly IYahooMarketDataScraper _yahooScraper;
private readonly ITradeRepublicService _trService;
private readonly ITechnicalAnalysisCalculator _calculator;
private readonly ILogger<TechnicalAnalysisDbService> _logger;
private readonly IFinlyticLogger<TechnicalAnalysisDbService> _finlyticLogger;
private static readonly ConcurrentDictionary<string, (List<MarketCandleEntity> Candles, string Symbol, string Currency, DateTime FetchedAt)> _candleCache = new();
private static readonly ConcurrentDictionary<string, SemaphoreSlim> _perIsinLocks = new();
@@ -41,13 +42,13 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
IYahooMarketDataScraper yahooScraper,
ITradeRepublicService trService,
ITechnicalAnalysisCalculator calculator,
ILogger<TechnicalAnalysisDbService> logger)
IFinlyticLogger<TechnicalAnalysisDbService> finlyticLogger)
{
_scopeFactory = scopeFactory;
_yahooScraper = yahooScraper;
_trService = trService;
_calculator = calculator;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
public async Task<TechnicalAnalysisDto?> GetAnalysisAsync(string isin, bool forceRefresh = false, string? ticker = null,
@@ -56,12 +57,11 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
if (string.IsNullOrWhiteSpace(isin)) return null;
var cleanIsin = isin.Trim().ToUpperInvariant();
// 1. Layer-1: Fast-Path aus In-Memory Cache (wenn kein forceRefresh)
if (!forceRefresh && _candleCache.TryGetValue(cleanIsin, out var ramEntry) &&
DateTime.UtcNow - ramEntry.FetchedAt < CandleCacheTtl &&
(string.IsNullOrWhiteSpace(ticker) || string.Equals(ramEntry.Symbol, ticker, StringComparison.OrdinalIgnoreCase)))
{
_logger.LogDebug("[{Channel}] RAM-Cache Hit for ISIN {Isin}. Merging live price...", "TechnicalAnalysisChannel", cleanIsin);
await _finlyticLogger.LogDebugAsync(SettingKeys.TechnicalAnalysisChannel, "[TechnicalAnalysisDbService] RAM-Cache Hit for ISIN {Isin}. Merging live price...", cleanIsin);
return await BuildAnalysisWithLivePriceAsync(cleanIsin, ramEntry.Candles, ramEntry.Symbol, ramEntry.Currency, cancellationToken);
}
@@ -70,7 +70,6 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
try
{
// Re-Check nach Lock-Erhalt
if (!forceRefresh && _candleCache.TryGetValue(cleanIsin, out ramEntry) &&
DateTime.UtcNow - ramEntry.FetchedAt < CandleCacheTtl &&
(string.IsNullOrWhiteSpace(ticker) || string.Equals(ramEntry.Symbol, ticker, StringComparison.OrdinalIgnoreCase)))
@@ -78,13 +77,12 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
return await BuildAnalysisWithLivePriceAsync(cleanIsin, ramEntry.Candles, ramEntry.Symbol, ramEntry.Currency, cancellationToken);
}
// 2. Layer-2: Prüfen ob frische Daten in der Datenbank liegen
if (!forceRefresh)
{
var dbDto = await GetFromDbCacheAsync(cleanIsin, ticker, cancellationToken);
if (dbDto != null)
{
_logger.LogDebug("[{Channel}] DB-Cache Hit for ISIN {Isin}.", "TechnicalAnalysisChannel", cleanIsin);
await _finlyticLogger.LogDebugAsync(SettingKeys.TechnicalAnalysisChannel, "[TechnicalAnalysisDbService] DB-Cache Hit for ISIN {Isin}.", cleanIsin);
return dbDto;
}
}
@@ -120,7 +118,7 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
private async Task<TechnicalAnalysisDto?> FullRefreshAsync(string cleanIsin, string? requestedTicker, CancellationToken cancellationToken)
{
_logger.LogInformation("[{Channel}] Full refresh for ISIN {Isin} (RequestedTicker: {Ticker})", "TechnicalAnalysisChannel", cleanIsin, requestedTicker ?? "None");
await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[TechnicalAnalysisDbService] Full refresh for ISIN {Isin} (RequestedTicker: {Ticker})", cleanIsin, requestedTicker ?? "None");
var macroTask = FetchMacroDataAsync(cancellationToken);
@@ -133,7 +131,6 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
var querySymbol = !string.IsNullOrEmpty(ticker) ? ticker : cleanIsin;
var (vix, gspc, dxy) = await macroTask;
// Lade 2y Daten für saubere Indikator-Aufwärmphasen
var yahooResult = await _yahooScraper.FetchHistoricalCandlesWithCurrencyAsync(querySymbol, "2y", "1d", cancellationToken);
var candles = yahooResult.Candles;
var currency = yahooResult.Currency;
@@ -147,7 +144,7 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
if (candles.Count == 0)
{
_logger.LogWarning("[{Channel}] No candles retrieved for {Symbol}", "TechnicalAnalysisChannel", querySymbol);
await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, "[TechnicalAnalysisDbService] No candles retrieved for {Symbol}", querySymbol);
return null;
}
@@ -173,7 +170,7 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
await Task.WhenAll(livePriceTask, macroTask);
var (livePrice, liveBid, liveAsk, preChange) = await livePriceTask; // Task-Result direkt nutzen
var (livePrice, liveBid, liveAsk, preChange) = await livePriceTask;
var (vix, gspc, dxy) = await macroTask;
ApplyLivePriceToCandles(cleanIsin, candles, querySymbol, currency, livePrice, liveBid, liveAsk);
@@ -194,11 +191,9 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
{
if (!livePrice.HasValue || livePrice.Value <= 0m) return;
// Währungsschutz: Trade Republic liefert IMMER EUR.
// Wenn die Kerzenhistorie USD ist (z.B. AAPL), darf der EUR-Livepreis NICHT direkt injiziert werden!
if (candleCurrency.Equals("USD", StringComparison.OrdinalIgnoreCase) && !cleanIsin.StartsWith("DE") && !cleanIsin.StartsWith("AT"))
{
_logger.LogDebug("[{Channel}] Skipping direct EUR live price injection for USD asset {Isin}", "TechnicalAnalysisChannel", cleanIsin);
_ = _finlyticLogger.LogDebugAsync(SettingKeys.TechnicalAnalysisChannel, "[TechnicalAnalysisDbService] Skipping direct EUR live price injection for USD asset {Isin}", cleanIsin);
return;
}
@@ -266,7 +261,7 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Real-time price fetch skipped for ISIN {Isin}", "TechnicalAnalysisChannel", cleanIsin);
await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[TechnicalAnalysisDbService] Real-time price fetch skipped for ISIN {Isin}", cleanIsin);
}
return (livePrice, liveBid, liveAsk, preChange);
@@ -330,14 +325,14 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
!string.Equals(requestedTicker.Trim(), cleanIsin, StringComparison.OrdinalIgnoreCase) &&
!string.Equals(cached.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase))
{
return null; // Ticker mismatch, force refresh required
return null;
}
return JsonSerializer.Deserialize<TechnicalAnalysisDto>(cached.AnalysisJson);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to read DB cache for ISIN {Isin}", "TechnicalAnalysisChannel", cleanIsin);
await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[TechnicalAnalysisDbService] Failed to read DB cache for ISIN {Isin}", cleanIsin);
}
return null;
@@ -374,7 +369,7 @@ public class TechnicalAnalysisDbService : ITechnicalAnalysisDbService
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to persist TA DB cache for ISIN {Isin}", "TechnicalAnalysisChannel", cleanIsin);
await _finlyticLogger.LogErrorAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[TechnicalAnalysisDbService] Failed to persist TA DB cache for ISIN {Isin}", cleanIsin);
}
}
@@ -3,9 +3,11 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services;
using FinlyticCore.Services.Yahoo;
using FinlyticTechnicalAnalysis.Entities;
using Microsoft.Extensions.Logging;
using FinlyticTechnicalAnalysis.Util;
using Microsoft.Extensions.Configuration;
namespace FinlyticTechnicalAnalysis.Services;
@@ -40,17 +42,17 @@ public interface IYahooMarketDataScraper
public class YahooMarketDataScraper : IYahooMarketDataScraper
{
private readonly YahooFinanceClient _yahooClient;
private readonly Microsoft.Extensions.Configuration.IConfiguration _configuration;
private readonly ILogger<YahooMarketDataScraper> _logger;
private readonly IConfiguration _configuration;
private readonly IFinlyticLogger<YahooMarketDataScraper> _finlyticLogger;
public YahooMarketDataScraper(
YahooFinanceClient yahooClient,
Microsoft.Extensions.Configuration.IConfiguration configuration,
ILogger<YahooMarketDataScraper> logger)
IConfiguration configuration,
IFinlyticLogger<YahooMarketDataScraper> finlyticLogger)
{
_yahooClient = yahooClient;
_configuration = configuration;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -66,7 +68,6 @@ public class YahooMarketDataScraper : IYahooMarketDataScraper
return cleanIsin;
}
// Crypto / Trade Republic interne ISINs (beginnend mit 'X', z. B. XF000BTC0017)
if (cleanIsin.StartsWith("X", StringComparison.OrdinalIgnoreCase))
{
var (cryptoSubtitle, cryptoName) = await FinlyticCore.Utils.CryptoSubtitleResolver.ResolveCryptoInfoAsync(
@@ -82,8 +83,7 @@ public class YahooMarketDataScraper : IYahooMarketDataScraper
var res = await FetchHistoricalCandlesWithCurrencyAsync(candidate, "5d", "1d", cancellationToken);
if (res.Candles.Count > 0)
{
_logger.LogInformation("[{Channel}] Resolved Crypto ISIN {Isin} to {Symbol} using Subtitle {Sub}",
"TechnicalAnalysisChannel", cleanIsin, candidate, cryptoSubtitle);
await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Resolved Crypto ISIN {Isin} to {Symbol} using Subtitle {Sub}", cleanIsin, candidate, cryptoSubtitle);
return candidate;
}
}
@@ -118,7 +118,7 @@ public class YahooMarketDataScraper : IYahooMarketDataScraper
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to resolve Yahoo ticker for ISIN {Isin}", "TechnicalAnalysisChannel", cleanIsin);
await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[YahooMarketDataScraper] Failed to resolve Yahoo ticker for ISIN {Isin}", cleanIsin);
}
return null;
@@ -150,11 +150,10 @@ public class YahooMarketDataScraper : IYahooMarketDataScraper
if (resultObj == null)
{
_logger.LogWarning("[{Channel}] No chart data returned from Yahoo Client for symbol {Symbol}", "TechnicalAnalysisChannel", symbol);
await _finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] No chart data returned from Yahoo Client for symbol {Symbol}", symbol);
return new YahooCandlesResult(results, detectedCurrency);
}
// Extract currency metadata
if (!string.IsNullOrWhiteSpace(resultObj.Meta?.Currency))
{
detectedCurrency = resultObj.Meta.Currency.ToUpperInvariant();
@@ -184,7 +183,6 @@ public class YahooMarketDataScraper : IYahooMarketDataScraper
var close = i < closes.Count && closes[i].HasValue ? (decimal)closes[i]!.Value : open;
var vol = i < volumes.Count && volumes[i].HasValue ? (long)volumes[i]!.Value : 0L;
// Skip invalid or empty weekend/holiday records
if (close <= 0m && open <= 0m) continue;
results.Add(new MarketCandleEntity
@@ -200,12 +198,12 @@ public class YahooMarketDataScraper : IYahooMarketDataScraper
});
}
_logger.LogInformation("[{Channel}] Successfully fetched {Count} candles for {Symbol} ({Range}, {Interval}, Currency: {Currency})",
"TechnicalAnalysisChannel", results.Count, symbol, range, interval, detectedCurrency);
await _finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[YahooMarketDataScraper] Successfully fetched {Count} candles for {Symbol} ({Range}, {Interval}, Currency: {Currency})",
results.Count, symbol, range, interval, detectedCurrency);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error fetching historical candles for {Symbol}", "TechnicalAnalysisChannel", symbol);
await _finlyticLogger.LogErrorAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[YahooMarketDataScraper] Error fetching historical candles for {Symbol}", symbol);
}
return new YahooCandlesResult(results, detectedCurrency);
@@ -0,0 +1,26 @@
using FinlyticCore.Models.Settings;
namespace FinlyticTechnicalAnalysis.Util;
public static class SettingKeys
{
// --- Logging-Kanäle ---
public static readonly SettingKey<bool> TechnicalAnalysisChannel = new("Logging.Channel.TechnicalAnalysis", true);
public static readonly SettingKey<bool> MqttChannel = new("Logging.Channel.MQTT", true);
public static readonly SettingKey<bool> HealthPingChannel = new("Logging.Channel.Health", true);
// --- Indikator-Konfiguration ---
public static readonly SettingKey<int> RsiPeriod = new("Indicators.RsiPeriod", 14);
public static readonly SettingKey<int> MacdFastPeriod = new("Indicators.MacdFastPeriod", 12);
public static readonly SettingKey<int> MacdSlowPeriod = new("Indicators.MacdSlowPeriod", 26);
public static readonly SettingKey<int> MacdSignalPeriod = new("Indicators.MacdSignalPeriod", 9);
public static readonly SettingKey<int> EmaShortPeriod = new("Indicators.EmaShortPeriod", 50);
public static readonly SettingKey<int> EmaLongPeriod = new("Indicators.EmaLongPeriod", 200);
public static readonly SettingKey<int> BollingerBandsPeriod = new("Indicators.BollingerBandsPeriod", 20);
public static readonly SettingKey<double> BollingerBandsStdDev = new("Indicators.BollingerBandsStdDev", 2.0);
public static readonly SettingKey<int> AtrPeriod = new("Indicators.AtrPeriod", 14);
// --- Cache & Performance ---
public static readonly SettingKey<int> CacheDurationMinutes = new("Cache.DurationMinutes", 60);
public static readonly SettingKey<bool> EnableAutoCache = new("Feature.EnableAutoCache", true);
}
+150 -48
View File
@@ -1,10 +1,15 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos;
using FinlyticCore.Dtos.Settings;
using FinlyticCore.Models;
using FinlyticCore.Services;
using FinlyticCore.Util;
using FinlyticTechnicalAnalysis.Services;
using FinlyticTechnicalAnalysis.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@@ -12,19 +17,30 @@ using Microsoft.Extensions.Logging;
namespace FinlyticTechnicalAnalysis.Util;
public class TAMqttClient(
ILogger<TAMqttClient> logger,
IConfiguration configuration,
IServiceScopeFactory scopeFactory) : ManagedMqttClient(logger), IHostedService
public class TAMqttClient : ManagedMqttClient, IHostedService
{
private readonly IConfiguration _configuration;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<TAMqttClient> _logger;
public TAMqttClient(
ILogger<TAMqttClient> logger,
IConfiguration configuration,
IServiceScopeFactory scopeFactory) : base(logger)
{
_logger = logger;
_configuration = configuration;
_scopeFactory = scopeFactory;
}
/// <summary>
/// Starts the MQTT client.
/// </summary>
public async Task StartAsync(CancellationToken cancellationToken)
{
var host = configuration["MQTT:Host"] ?? configuration["MQTT__Host"] ?? "localhost";
var portStr = configuration["MQTT:Port"] ?? configuration["MQTT__Port"] ?? "1883";
var clientId = configuration["MQTT:ClientId"] ?? "finlytic_ta_" + Guid.NewGuid().ToString("N");
var host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost";
var portStr = _configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883";
var clientId = _configuration["MQTT:ClientId"] ?? "finlytic_ta_" + Guid.NewGuid().ToString("N");
var config = new MqttConfiguration
{
@@ -33,7 +49,7 @@ public class TAMqttClient(
ClientId = clientId
};
logger.LogInformation("[{Channel}] Starting Technical Analysis MQTT client. Host: {Host}, ClientId: {ClientId}", "TechnicalAnalysisChannel", config.Host, config.ClientId);
_logger.LogInformation("Starting Technical Analysis MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId);
await ConnectAsync(config);
}
@@ -42,17 +58,27 @@ public class TAMqttClient(
/// </summary>
public async Task StopAsync(CancellationToken cancellationToken)
{
logger.LogInformation("[{Channel}] Stopping Technical Analysis MQTT client.", "TechnicalAnalysisChannel");
_logger.LogInformation("Stopping Technical Analysis MQTT client.");
await DisconnectAsync();
}
protected override async Task OnConnectedAsync()
{
logger.LogInformation("[{Channel}] Technical Analysis MQTT client connected. Subscribing to RPC topic...", "TechnicalAnalysisChannel");
_logger.LogInformation("Technical Analysis MQTT client connected. Subscribing to RPC topics...");
await SubscribeAsync("services/request/ta_GetAnalysis/#");
await SubscribeAsync("services/request/tr_GetLivePrice/#");
await SubscribeAsync("services/request/ta_settings_GetAll/#");
await SubscribeAsync("services/request/ta_settings_Update/#");
await SubscribeAsync("services/request/health_Ping/#");
await SubscribeAsync("services/config/updated/#");
FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
{
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticTechnicalAnalysis", StringComparison.OrdinalIgnoreCase))
{
await PublishAsync("finlytic/logs/FinlyticTechnicalAnalysis", logDto);
}
};
}
protected override async Task OnMessageReceivedAsync(string topic, string payload)
@@ -69,21 +95,96 @@ public class TAMqttClient(
if (segments.Length < 4) return;
var channel = segments[2];
var correlationId = segments[segments.Length - 1];
var correlationId = segments[^1];
if (topic.Contains("health_Ping", StringComparison.OrdinalIgnoreCase))
switch (channel)
{
await HandleHealthPingAsync(topic, segments, correlationId);
return;
}
case "ta_GetAnalysis":
await HandleGetAnalysisAsync(payload, correlationId);
break;
if (channel == "ta_GetAnalysis")
{
await HandleGetAnalysisAsync(payload, correlationId);
case "tr_GetLivePrice":
await HandleGetLivePriceAsync(payload, correlationId);
break;
case "ta_settings_GetAll":
await HandleSettingsGetAllAsync(correlationId);
break;
case "ta_settings_Update":
await HandleSettingsUpdateAsync(payload, correlationId);
break;
case "health_Ping":
await HandleHealthPingAsync(topic, segments, correlationId);
break;
default:
_logger.LogDebug("Received unhandled RPC channel: {Channel}", channel);
break;
}
else if (channel == "tr_GetLivePrice")
}
private async Task HandleSettingsGetAllAsync(string correlationId)
{
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<TAMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticTechnicalAnalysis] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
try
{
await HandleGetLivePriceAsync(payload, correlationId);
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/ta_settings_GetAll/{correlationId}";
await PublishAsync(responseTopic, settings);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticTechnicalAnalysis] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticTechnicalAnalysis] [Settings_GetAll] Failed to retrieve settings.");
}
}
private async Task HandleSettingsUpdateAsync(string payload, string correlationId)
{
if (string.IsNullOrWhiteSpace(payload)) return;
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<TAMqttClient>>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticTechnicalAnalysis] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
try
{
Dictionary<string, object?>? updates = null;
try
{
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
}
catch
{
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
if (list != null)
{
updates = new Dictionary<string, object?>();
foreach (var item in list) updates[item.Key] = item.Value;
}
}
if (updates != null && updates.Count > 0)
{
await settingsService.UpdateSettingsAsync(updates);
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticTechnicalAnalysis] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
}
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
var responseTopic = $"services/response/ta_settings_Update/{correlationId}";
await PublishAsync(responseTopic, currentSettings);
}
catch (Exception ex)
{
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticTechnicalAnalysis] [Settings_Update] Failed to update settings.");
}
}
@@ -92,22 +193,21 @@ public class TAMqttClient(
if (!topic.EndsWith("FinlyticTechnicalAnalysis", StringComparison.OrdinalIgnoreCase))
return;
logger.LogInformation("[{Channel}] [TAMqttClient] Received config update event for FinlyticTechnicalAnalysis.", "TechnicalAnalysisChannel");
try
{
var updatePayload = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.ServiceConfigUpdatePayload);
if (updatePayload?.Settings != null && updatePayload.Settings.Count > 0)
using var doc = JsonDocument.Parse(payload);
if (doc.RootElement.TryGetProperty("settings", out var settingsProp))
{
using var scope = scopeFactory.CreateScope();
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
await settingsDb.UpdateSettingsFromDictionaryAsync(updatePayload.Settings);
logger.LogInformation("[{Channel}] [TAMqttClient] Persisted {Count} updated settings to FinlyticTechnicalAnalysis database.", "TechnicalAnalysisChannel", updatePayload.Settings.Count);
var dict = JsonSerializer.Deserialize<Dictionary<string, object?>>(settingsProp.GetRawText());
if (dict != null && dict.Count > 0)
{
using var scope = _scopeFactory.CreateScope();
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
await settings.UpdateSettingsAsync(dict);
}
}
}
catch (Exception ex)
{
logger.LogError(ex, "[{Channel}] [TAMqttClient] Error processing MQTT config update event.", "TechnicalAnalysisChannel");
}
catch { }
}
private async Task HandleHealthPingAsync(string topic, string[] segments, string correlationId)
@@ -119,40 +219,42 @@ public class TAMqttClient(
if (isForMe)
{
string respTopic = $"services/response/health_Ping/{correlationId}";
await PublishAsync(respTopic, new FinlyticCore.Dtos.ServiceHealthResponse("FinlyticTechnicalAnalysis", "Online", DateTime.UtcNow, "Connected"));
logger.LogInformation("[{Channel}] [TAMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", "TechnicalAnalysisChannel", correlationId);
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticTechnicalAnalysis", "Online", DateTime.UtcNow, "Connected"));
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<TAMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[FinlyticTechnicalAnalysis] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
}
}
private async Task HandleGetAnalysisAsync(string payload, string correlationId)
{
logger.LogInformation("[{Channel}] Received RPC ta_GetAnalysis request. CorrelationId: {CorrelationId}", "TechnicalAnalysisChannel", correlationId);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<TAMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[FinlyticTechnicalAnalysis] Received RPC ta_GetAnalysis request. CorrelationId: {CorrelationId}", correlationId);
var req = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.IsinRequest);
string responseTopic = $"services/response/ta_GetAnalysis/{correlationId}";
if (string.IsNullOrWhiteSpace(req?.Isin))
{
logger.LogWarning("[{Channel}] Request missing mandatory ISIN parameter.", "TechnicalAnalysisChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, "[FinlyticTechnicalAnalysis] Request missing mandatory ISIN parameter.");
await PublishAsync<object?>(responseTopic, null);
return;
}
try
{
using var scope = scopeFactory.CreateScope();
var taDbService = scope.ServiceProvider.GetRequiredService<ITechnicalAnalysisDbService>();
var analysis = await taDbService.GetAnalysisAsync(req.Isin, req.ForceRefresh, req.Ticker);
logger.LogInformation("[{Channel}] Publishing RPC response to {ResponseTopic}", "TechnicalAnalysisChannel", responseTopic);
await finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[FinlyticTechnicalAnalysis] Publishing RPC response to {ResponseTopic}", responseTopic);
await PublishAsync(responseTopic, analysis);
}
catch (Exception ex)
{
logger.LogError(ex, "[{Channel}] Failed to fetch technical analysis and publish RPC response for ISIN {Isin}", "TechnicalAnalysisChannel", req.Isin);
// Antworte mit null, damit der Aufrufer nicht im RPC-Timeout verharrt
await finlyticLogger.LogErrorAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[FinlyticTechnicalAnalysis] Failed to fetch technical analysis for ISIN {Isin}", req.Isin);
try
{
await PublishAsync<object?>(responseTopic, null);
@@ -163,32 +265,32 @@ public class TAMqttClient(
private async Task HandleGetLivePriceAsync(string payload, string correlationId)
{
logger.LogInformation("[{Channel}] Received RPC tr_GetLivePrice request. CorrelationId: {CorrelationId}", "TechnicalAnalysisChannel", correlationId);
using var scope = _scopeFactory.CreateScope();
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<TAMqttClient>>();
await finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[FinlyticTechnicalAnalysis] Received RPC tr_GetLivePrice request. CorrelationId: {CorrelationId}", correlationId);
var req = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.IsinRequest);
string responseTopic = $"services/response/tr_GetLivePrice/{correlationId}";
if (string.IsNullOrWhiteSpace(req?.Isin))
{
logger.LogWarning("[{Channel}] tr_GetLivePrice request missing mandatory ISIN parameter.", "TechnicalAnalysisChannel");
await finlyticLogger.LogWarningAsync(SettingKeys.TechnicalAnalysisChannel, "[FinlyticTechnicalAnalysis] tr_GetLivePrice request missing mandatory ISIN parameter.");
await PublishAsync<object?>(responseTopic, null);
return;
}
try
{
using var scope = scopeFactory.CreateScope();
var taDbService = scope.ServiceProvider.GetRequiredService<ITechnicalAnalysisDbService>();
var livePrice = await taDbService.GetLivePriceAsync(req.Isin);
logger.LogInformation("[{Channel}] Publishing RPC response to {ResponseTopic} for ISIN {Isin}", "TechnicalAnalysisChannel", responseTopic, req.Isin);
await finlyticLogger.LogInfoAsync(SettingKeys.TechnicalAnalysisChannel, "[FinlyticTechnicalAnalysis] Publishing RPC response to {ResponseTopic} for ISIN {Isin}", responseTopic, req.Isin);
await PublishAsync(responseTopic, livePrice);
}
catch (Exception ex)
{
logger.LogError(ex, "[{Channel}] Failed to fetch live price and publish RPC response for ISIN {Isin}", "TechnicalAnalysisChannel", req.Isin);
await finlyticLogger.LogErrorAsync(SettingKeys.TechnicalAnalysisChannel, ex, "[FinlyticTechnicalAnalysis] Failed to fetch live price for ISIN {Isin}", req.Isin);
try
{
await PublishAsync<object?>(responseTopic, null);
+17 -2
View File
@@ -1,10 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FinlyticCore.Database;
using FinlyticCore.Entities.Settings;
using FinlyticTrades.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace FinlyticTrades.Database;
public class TradesDbContext : DbContext
public class TradesDbContext : DbContext, ISettingsDbContext
{
public TradesDbContext(DbContextOptions<TradesDbContext> options) : base(options) { }
@@ -20,7 +25,7 @@ public class TradesDbContext : DbContext
modelBuilder.Entity<SettingEntity>(entity =>
{
entity.HasKey(e => e.Id);
entity.HasIndex(e => e.Key);
entity.HasIndex(e => e.Key).IsUnique();
});
var stringListConverter =
@@ -57,3 +62,13 @@ public class TradesDbContext : DbContext
});
}
}
public class TradesDbContextFactory : IDesignTimeDbContextFactory<TradesDbContext>
{
public TradesDbContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<TradesDbContext>();
optionsBuilder.UseNpgsql("Host=localhost;Database=trades;Username=postgres;Password=postgres");
return new TradesDbContext(optionsBuilder.Options);
}
}
@@ -0,0 +1,358 @@
// <auto-generated />
using System;
using FinlyticTrades.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticTrades.Migrations
{
[DbContext(typeof(TradesDbContext))]
[Migration("20260815184034_AddDynamicSettings")]
partial class AddDynamicSettings
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("LastUpdatedUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("ServiceIdentifier")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("ValueJson")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});
modelBuilder.Entity("FinlyticTrades.Entities.TradeEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<decimal?>("ActualEntryPrice")
.HasColumnType("decimal(18,4)");
b.Property<string>("AnalysisId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<string>("AssetType")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<string>("CloseReason")
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<DateTime?>("ClosedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("CompanyName")
.IsRequired()
.HasMaxLength(150)
.HasColumnType("character varying(150)");
b.Property<DateTime>("CreatedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("DerivativeIsin")
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<string>("DerivativeProductCategories")
.IsRequired()
.HasColumnType("text");
b.Property<decimal?>("EntryFee")
.HasColumnType("decimal(18,4)");
b.Property<decimal>("EntryPrice")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("EntryZoneMax")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("EntryZoneMin")
.HasColumnType("decimal(18,4)");
b.Property<string>("EventId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<DateTime?>("ExecutionTimestamp")
.HasColumnType("timestamp with time zone");
b.Property<decimal?>("ExitFee")
.HasColumnType("decimal(18,4)");
b.Property<string>("FundamentalRationale")
.IsRequired()
.HasColumnType("text");
b.Property<bool>("HasCfd")
.HasColumnType("boolean");
b.Property<string>("InstrumentType")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<bool>("IsGlobalProposal")
.HasColumnType("boolean");
b.Property<bool>("IsRecurring")
.HasColumnType("boolean");
b.Property<bool?>("IsWin")
.HasColumnType("boolean");
b.Property<string>("Isin")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<decimal?>("KnockoutThreshold")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("LeverageUsed")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("MaxLeverage")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("PnlAbsolute")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("PnlPercent")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("PositionSize")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("Quantity")
.HasColumnType("decimal(18,4)");
b.Property<string>("Reasoning")
.IsRequired()
.HasColumnType("text");
b.Property<decimal?>("RiskRewardRatio")
.HasColumnType("decimal(18,4)");
b.Property<string>("RiskTolerance")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<string>("RiskWarning")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Sector")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)");
b.Property<string>("SignalType")
.IsRequired()
.HasMaxLength(10)
.HasColumnType("character varying(10)");
b.Property<int>("Status")
.HasColumnType("integer");
b.Property<decimal>("StopLoss")
.HasColumnType("decimal(18,4)");
b.Property<string>("Symbol")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<decimal>("TakeProfit")
.HasColumnType("decimal(18,4)");
b.Property<string>("TakeProfitTargets")
.HasColumnType("text");
b.Property<string>("TechnicalRationale")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Timeframe")
.IsRequired()
.HasMaxLength(20)
.HasColumnType("character varying(20)");
b.Property<string>("TradeId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<int>("TtlMinutes")
.HasColumnType("integer");
b.Property<decimal?>("UserExitPrice")
.HasColumnType("decimal(18,4)");
b.Property<DateTime?>("UserExitTimestamp")
.HasColumnType("timestamp with time zone");
b.Property<string>("UserId")
.HasMaxLength(100)
.HasColumnType("character varying(100)");
b.Property<int>("VixRegime")
.HasColumnType("integer");
b.Property<decimal>("VixValue")
.HasColumnType("decimal(18,4)");
b.Property<double>("WinRate")
.HasColumnType("double precision");
b.HasKey("Id");
b.HasIndex("AnalysisId");
b.HasIndex("CreatedAt");
b.HasIndex("EventId");
b.HasIndex("Isin");
b.HasIndex("Sector");
b.HasIndex("Status");
b.HasIndex("TradeId")
.IsUnique();
b.ToTable("trades");
});
modelBuilder.Entity("FinlyticTrades.Entities.TradeHourlyUpdateEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<decimal>("CurrentPrice")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("FloatingPnlPercent")
.HasColumnType("decimal(18,4)");
b.Property<string>("Reasoning")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Recommendation")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)");
b.Property<decimal?>("SuggestedStopLoss")
.HasColumnType("decimal(18,4)");
b.Property<decimal?>("SuggestedTakeProfit")
.HasColumnType("decimal(18,4)");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp with time zone");
b.Property<Guid>("TradeId")
.HasColumnType("uuid");
b.Property<decimal>("VixValue")
.HasColumnType("decimal(18,4)");
b.HasKey("Id");
b.HasIndex("Timestamp");
b.HasIndex("TradeId");
b.HasIndex("TradeId", "Timestamp");
b.ToTable("trade_hourly_updates");
});
modelBuilder.Entity("FinlyticTrades.Entities.TradesSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<double>("AtrStopLossMultiplier")
.HasColumnType("double precision");
b.Property<int>("MaxOpenPositions")
.HasColumnType("integer");
b.Property<double>("RiskPerTradePercentage")
.HasColumnType("double precision");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticTrades.Entities.TradeHourlyUpdateEntity", b =>
{
b.HasOne("FinlyticTrades.Entities.TradeEntity", "Trade")
.WithMany("HourlyUpdates")
.HasForeignKey("TradeId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Trade");
});
modelBuilder.Entity("FinlyticTrades.Entities.TradeEntity", b =>
{
b.Navigation("HourlyUpdates");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticTrades.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings");
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings");
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key");
}
}
}
@@ -47,7 +47,8 @@ namespace FinlyticTrades.Migrations
b.HasKey("Id");
b.HasIndex("Key");
b.HasIndex("Key")
.IsUnique();
b.ToTable("DynamicSettings");
});

Some files were not shown because too many files have changed in this diff Show More