Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(assets): update asset entity mappings, database migrations, and MQTT RPC handlers

Browse Source
This commit is contained in:
Kleidukos
2026-08-24 21:37:21 +02:00
parent 7060f0f7b1
commit 6974b2075b
26 changed files with 645 additions and 2237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticAssets/Services/AssetScannerBackgroundService.cs
+73 -42
View File
@@ -1,9 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticAssets.Models;
using FinlyticAssets.Util;
using FinlyticCore.Dtos.TradeRepublic;
using FinlyticCore.Models.Assets;
@@ -15,14 +15,16 @@ using Microsoft.Extensions.Hosting;
namespace FinlyticAssets.Services;
/// <summary>
/// A background service that runs a continuous asset synchronization loop,
/// scanning Trade Republic to retrieve, update, and index all supported asset types.
/// A background service that runs a continuous asset synchronization loop for Stocks and ETFs,
/// retrieving, updating, and indexing assets while downloading logos inline.
/// </summary>
public class AssetScannerBackgroundService : BackgroundService
{
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly IFinlyticLogger<AssetScannerBackgroundService> _finlyticLogger;
private static readonly AssetType[] ScannedAssetTypes = [AssetType.Stock, AssetType.Fund];
private AssetsCount? _assetsCount;
private AssetsCount? _currAssetsCount;
@@ -54,24 +56,35 @@ public class AssetScannerBackgroundService : BackgroundService
{
using var scope = _serviceScopeFactory.CreateScope();
var tradeRepublicService = scope.ServiceProvider.GetRequiredService<ITradeRepublicService>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
var assetsDbService = scope.ServiceProvider.GetRequiredService<IAssetsDbService>();
var indexService = scope.ServiceProvider.GetRequiredService<IAssetsIndexService>();
var isAutoScanEnabled = await settingsService.GetSettingAsync(SettingKeys.ScannerEnableAutoScan, stoppingToken);
if (!isAutoScanEnabled)
{
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Auto scan is disabled in dynamic settings. Waiting 1 minute...");
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
continue;
}
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Requesting total asset counts from Trade Republic...");
_assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken);
_currAssetsCount = new AssetsCount();
var initSettings = await settingsService.GetSettings();
var isRecoveryMode = initSettings.CurrentScanningPage > 0;
var initCurrentTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken);
var initCurrentPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken);
var isRecoveryMode = initCurrentPage > 0;
foreach (var type in Enum.GetValues<AssetType>())
Enum.TryParse<AssetType>(initCurrentTypeStr, true, out var initCurrentType);
foreach (var type in ScannedAssetTypes)
{
if (stoppingToken.IsCancellationRequested) break;
if (isRecoveryMode)
{
if (type != initSettings.CurrentScanningType)
if (type != initCurrentType)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Recovery: {AssetType} was already processed. Skipping.", type);
continue;
@@ -80,19 +93,17 @@ public class AssetScannerBackgroundService : BackgroundService
}
else
{
var settings = await settingsService.GetSettings();
settings.CurrentScanningType = type;
settings.CurrentScanningPage = 0;
await settingsService.SaveSettings(settings);
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken);
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
}
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Processing asset type: {AssetType}...", type);
await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken);
var currentSettings = await settingsService.GetSettings();
var delaySeconds = currentSettings.FinishedInitialScan
? currentSettings.AssetUpdateTypeDelay
: currentSettings.InitAssetUpdateTypeDelay;
var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken);
var delaySeconds = finishedInitial
? await settingsService.GetSettingAsync(SettingKeys.ScannerTypeDelay, stoppingToken)
: await settingsService.GetSettingAsync(SettingKeys.ScannerInitTypeDelay, stoppingToken);
if (delaySeconds > 0)
{
@@ -102,19 +113,18 @@ public class AssetScannerBackgroundService : BackgroundService
}
}
var finalSettings = await settingsService.GetSettings();
finalSettings.CurrentScanningPage = 0;
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested)
var finishedScan = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken);
if (!finishedScan && !stoppingToken.IsCancellationRequested)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Initial scan successfully completed. Switching FinishedInitialScan to true.");
finalSettings.FinishedInitialScan = true;
await settingsService.SetSettingAsync(SettingKeys.ScannerFinishedInitialScan, true, stoppingToken);
}
await settingsService.SaveSettings(finalSettings);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting 1 minute before starting the next cycle.");
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
var cycleDelay = await settingsService.GetSettingAsync(SettingKeys.ScannerCycleDelayMinutes, stoppingToken);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting {Minutes} minutes before starting the next cycle.", cycleDelay);
await Task.Delay(TimeSpan.FromMinutes(cycleDelay), stoppingToken);
}
catch (Exception e) when (!stoppingToken.IsCancellationRequested)
{
@@ -129,7 +139,7 @@ public class AssetScannerBackgroundService : BackgroundService
private async Task HandleAssetType(
AssetType type,
ITradeRepublicService tradeRepublicService,
ISettingsDbService settingsDbService,
ISettingsService settingsService,
IAssetsDbService assetsDbService,
IAssetsIndexService indexService,
CancellationToken stoppingToken)
@@ -144,26 +154,28 @@ public class AssetScannerBackgroundService : BackgroundService
_currAssetsCount ??= new AssetsCount();
var currentItemOffset = 0;
var settings = await settingsDbService.GetSettings();
var pageSize = Math.Clamp(settings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : settings.TradeRepublicMaxRequestPageSize, 1, 100);
var configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken);
var pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100);
if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0)
var currentScanningTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken);
var currentScanningPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken);
if (string.Equals(currentScanningTypeStr, type.ToString(), StringComparison.OrdinalIgnoreCase) && currentScanningPage > 0)
{
currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize;
currentItemOffset = (currentScanningPage - 1) * pageSize;
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).",
type, settings.CurrentScanningPage, currentItemOffset);
type, currentScanningPage, currentItemOffset);
}
while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested)
{
var currentSettings = await settingsDbService.GetSettings();
pageSize = Math.Clamp(currentSettings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : currentSettings.TradeRepublicMaxRequestPageSize, 1, 100);
configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken);
pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100);
var currentPage = (currentItemOffset / pageSize) + 1;
currentSettings.CurrentScanningType = type;
currentSettings.CurrentScanningPage = currentPage;
await settingsDbService.SaveSettings(currentSettings);
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken);
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, currentPage, stoppingToken);
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}",
type, currentPage, currentItemOffset, totalCount);
@@ -173,8 +185,7 @@ public class AssetScannerBackgroundService : BackgroundService
if (assets?.Results == null || assets.Results.Count == 0)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", type, currentPage);
currentSettings.CurrentScanningPage = 0;
await settingsDbService.SaveSettings(currentSettings);
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
break;
}
@@ -186,14 +197,14 @@ public class AssetScannerBackgroundService : BackgroundService
if (assets.Results.Count < pageSize)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Reached the last page for {AssetType}.", type);
currentSettings.CurrentScanningPage = 0;
await settingsDbService.SaveSettings(currentSettings);
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
break;
}
var delaySeconds = currentSettings.FinishedInitialScan
? currentSettings.BatchAssetUpdateDelay
: currentSettings.InitBatchAssetUpdateDelay;
var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken);
var delaySeconds = finishedInitial
? await settingsService.GetSettingAsync(SettingKeys.ScannerBatchDelay, stoppingToken)
: await settingsService.GetSettingAsync(SettingKeys.ScannerInitBatchDelay, stoppingToken);
if (delaySeconds > 0)
{
@@ -218,6 +229,26 @@ public class AssetScannerBackgroundService : BackgroundService
var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets);
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows);
// Inline logo fetching for each asset in the batch if missing on disk
string directoryPath = Volumes.LogosRelativePath;
foreach (var a in assets)
{
if (stoppingToken.IsCancellationRequested) break;
if (string.IsNullOrWhiteSpace(a.Isin)) continue;
string cleanIsin = a.Isin.Trim().ToUpperInvariant();
string logoPath = Path.Combine(directoryPath, $"{cleanIsin}.svg");
if (!File.Exists(logoPath))
{
try
{
await indexService.DownloadAndSaveLogoAsync(cleanIsin, stoppingToken);
}
catch { /* Ignore non-fatal logo fetch failures */ }
}
}
if (changedRows > 0)
{