refactor(assets): update asset entity mappings, database migrations, and MQTT RPC handlers
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticAssets.Models;
|
||||
using FinlyticAssets.Util;
|
||||
using FinlyticCore.Dtos.TradeRepublic;
|
||||
using FinlyticCore.Models.Assets;
|
||||
@@ -15,14 +15,16 @@ using Microsoft.Extensions.Hosting;
|
||||
namespace FinlyticAssets.Services;
|
||||
|
||||
/// <summary>
|
||||
/// A background service that runs a continuous asset synchronization loop,
|
||||
/// scanning Trade Republic to retrieve, update, and index all supported asset types.
|
||||
/// A background service that runs a continuous asset synchronization loop for Stocks and ETFs,
|
||||
/// retrieving, updating, and indexing assets while downloading logos inline.
|
||||
/// </summary>
|
||||
public class AssetScannerBackgroundService : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _serviceScopeFactory;
|
||||
private readonly IFinlyticLogger<AssetScannerBackgroundService> _finlyticLogger;
|
||||
|
||||
private static readonly AssetType[] ScannedAssetTypes = [AssetType.Stock, AssetType.Fund];
|
||||
|
||||
private AssetsCount? _assetsCount;
|
||||
private AssetsCount? _currAssetsCount;
|
||||
|
||||
@@ -54,24 +56,35 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
{
|
||||
using var scope = _serviceScopeFactory.CreateScope();
|
||||
var tradeRepublicService = scope.ServiceProvider.GetRequiredService<ITradeRepublicService>();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
||||
var assetsDbService = scope.ServiceProvider.GetRequiredService<IAssetsDbService>();
|
||||
var indexService = scope.ServiceProvider.GetRequiredService<IAssetsIndexService>();
|
||||
|
||||
var isAutoScanEnabled = await settingsService.GetSettingAsync(SettingKeys.ScannerEnableAutoScan, stoppingToken);
|
||||
if (!isAutoScanEnabled)
|
||||
{
|
||||
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Auto scan is disabled in dynamic settings. Waiting 1 minute...");
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Requesting total asset counts from Trade Republic...");
|
||||
_assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken);
|
||||
_currAssetsCount = new AssetsCount();
|
||||
|
||||
var initSettings = await settingsService.GetSettings();
|
||||
var isRecoveryMode = initSettings.CurrentScanningPage > 0;
|
||||
var initCurrentTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken);
|
||||
var initCurrentPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken);
|
||||
var isRecoveryMode = initCurrentPage > 0;
|
||||
|
||||
foreach (var type in Enum.GetValues<AssetType>())
|
||||
Enum.TryParse<AssetType>(initCurrentTypeStr, true, out var initCurrentType);
|
||||
|
||||
foreach (var type in ScannedAssetTypes)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
|
||||
if (isRecoveryMode)
|
||||
{
|
||||
if (type != initSettings.CurrentScanningType)
|
||||
if (type != initCurrentType)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Recovery: {AssetType} was already processed. Skipping.", type);
|
||||
continue;
|
||||
@@ -80,19 +93,17 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
}
|
||||
else
|
||||
{
|
||||
var settings = await settingsService.GetSettings();
|
||||
settings.CurrentScanningType = type;
|
||||
settings.CurrentScanningPage = 0;
|
||||
await settingsService.SaveSettings(settings);
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken);
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Processing asset type: {AssetType}...", type);
|
||||
await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken);
|
||||
|
||||
var currentSettings = await settingsService.GetSettings();
|
||||
var delaySeconds = currentSettings.FinishedInitialScan
|
||||
? currentSettings.AssetUpdateTypeDelay
|
||||
: currentSettings.InitAssetUpdateTypeDelay;
|
||||
var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken);
|
||||
var delaySeconds = finishedInitial
|
||||
? await settingsService.GetSettingAsync(SettingKeys.ScannerTypeDelay, stoppingToken)
|
||||
: await settingsService.GetSettingAsync(SettingKeys.ScannerInitTypeDelay, stoppingToken);
|
||||
|
||||
if (delaySeconds > 0)
|
||||
{
|
||||
@@ -102,19 +113,18 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
var finalSettings = await settingsService.GetSettings();
|
||||
finalSettings.CurrentScanningPage = 0;
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
|
||||
|
||||
if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested)
|
||||
var finishedScan = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken);
|
||||
if (!finishedScan && !stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Initial scan successfully completed. Switching FinishedInitialScan to true.");
|
||||
finalSettings.FinishedInitialScan = true;
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerFinishedInitialScan, true, stoppingToken);
|
||||
}
|
||||
|
||||
await settingsService.SaveSettings(finalSettings);
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting 1 minute before starting the next cycle.");
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||
var cycleDelay = await settingsService.GetSettingAsync(SettingKeys.ScannerCycleDelayMinutes, stoppingToken);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting {Minutes} minutes before starting the next cycle.", cycleDelay);
|
||||
await Task.Delay(TimeSpan.FromMinutes(cycleDelay), stoppingToken);
|
||||
}
|
||||
catch (Exception e) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -129,7 +139,7 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
private async Task HandleAssetType(
|
||||
AssetType type,
|
||||
ITradeRepublicService tradeRepublicService,
|
||||
ISettingsDbService settingsDbService,
|
||||
ISettingsService settingsService,
|
||||
IAssetsDbService assetsDbService,
|
||||
IAssetsIndexService indexService,
|
||||
CancellationToken stoppingToken)
|
||||
@@ -144,26 +154,28 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
_currAssetsCount ??= new AssetsCount();
|
||||
var currentItemOffset = 0;
|
||||
|
||||
var settings = await settingsDbService.GetSettings();
|
||||
var pageSize = Math.Clamp(settings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : settings.TradeRepublicMaxRequestPageSize, 1, 100);
|
||||
var configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken);
|
||||
var pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100);
|
||||
|
||||
if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0)
|
||||
var currentScanningTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken);
|
||||
var currentScanningPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken);
|
||||
|
||||
if (string.Equals(currentScanningTypeStr, type.ToString(), StringComparison.OrdinalIgnoreCase) && currentScanningPage > 0)
|
||||
{
|
||||
currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize;
|
||||
currentItemOffset = (currentScanningPage - 1) * pageSize;
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).",
|
||||
type, settings.CurrentScanningPage, currentItemOffset);
|
||||
type, currentScanningPage, currentItemOffset);
|
||||
}
|
||||
|
||||
while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var currentSettings = await settingsDbService.GetSettings();
|
||||
pageSize = Math.Clamp(currentSettings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : currentSettings.TradeRepublicMaxRequestPageSize, 1, 100);
|
||||
configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken);
|
||||
pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100);
|
||||
|
||||
var currentPage = (currentItemOffset / pageSize) + 1;
|
||||
|
||||
currentSettings.CurrentScanningType = type;
|
||||
currentSettings.CurrentScanningPage = currentPage;
|
||||
await settingsDbService.SaveSettings(currentSettings);
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken);
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, currentPage, stoppingToken);
|
||||
|
||||
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}",
|
||||
type, currentPage, currentItemOffset, totalCount);
|
||||
@@ -173,8 +185,7 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
if (assets?.Results == null || assets.Results.Count == 0)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", type, currentPage);
|
||||
currentSettings.CurrentScanningPage = 0;
|
||||
await settingsDbService.SaveSettings(currentSettings);
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -186,14 +197,14 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
if (assets.Results.Count < pageSize)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Reached the last page for {AssetType}.", type);
|
||||
currentSettings.CurrentScanningPage = 0;
|
||||
await settingsDbService.SaveSettings(currentSettings);
|
||||
await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken);
|
||||
break;
|
||||
}
|
||||
|
||||
var delaySeconds = currentSettings.FinishedInitialScan
|
||||
? currentSettings.BatchAssetUpdateDelay
|
||||
: currentSettings.InitBatchAssetUpdateDelay;
|
||||
var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken);
|
||||
var delaySeconds = finishedInitial
|
||||
? await settingsService.GetSettingAsync(SettingKeys.ScannerBatchDelay, stoppingToken)
|
||||
: await settingsService.GetSettingAsync(SettingKeys.ScannerInitBatchDelay, stoppingToken);
|
||||
|
||||
if (delaySeconds > 0)
|
||||
{
|
||||
@@ -218,6 +229,26 @@ public class AssetScannerBackgroundService : BackgroundService
|
||||
|
||||
var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows);
|
||||
|
||||
// Inline logo fetching for each asset in the batch if missing on disk
|
||||
string directoryPath = Volumes.LogosRelativePath;
|
||||
foreach (var a in assets)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
if (string.IsNullOrWhiteSpace(a.Isin)) continue;
|
||||
|
||||
string cleanIsin = a.Isin.Trim().ToUpperInvariant();
|
||||
string logoPath = Path.Combine(directoryPath, $"{cleanIsin}.svg");
|
||||
|
||||
if (!File.Exists(logoPath))
|
||||
{
|
||||
try
|
||||
{
|
||||
await indexService.DownloadAndSaveLogoAsync(cleanIsin, stoppingToken);
|
||||
}
|
||||
catch { /* Ignore non-fatal logo fetch failures */ }
|
||||
}
|
||||
}
|
||||
|
||||
if (changedRows > 0)
|
||||
{
|
||||
|
||||
@@ -14,7 +14,7 @@ using Microsoft.EntityFrameworkCore;
|
||||
namespace FinlyticAssets.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines database operations for managing Trade Republic asset entities.
|
||||
/// Defines database operations for managing Trade Republic asset entities and on-demand derivatives.
|
||||
/// </summary>
|
||||
public interface IAssetsDbService
|
||||
{
|
||||
@@ -24,7 +24,6 @@ public interface IAssetsDbService
|
||||
public Task<List<AssetEntity>> GetAssetsByIsinAsync(string isin);
|
||||
public Task<List<AssetEntity>> GetValidAssetsByIsinAsync(string isin);
|
||||
public Task<List<AssetEntity>> FindAffectedActiveAssetsAsync(string searchQuery);
|
||||
public Task UpdateAssetImageIdAsync(string isin, string imageId);
|
||||
public Task<bool> DeleteAssetAsync(string isin);
|
||||
public Task<List<AssetEntity>> GetDiscoveryAssetsAsync(int limit = 15);
|
||||
public Task<List<DerivativeEntity>> GetDerivativesByUnderlyingAsync(string underlyingIsin, string optionType = "long", decimal? targetLeverage = null, string? after = null, int? page = null, bool forceRefresh = false, CancellationToken cancellationToken = default);
|
||||
@@ -63,7 +62,6 @@ public class AssetsDbService : IAssetsDbService
|
||||
Asset = a,
|
||||
Score = (a.Tags?.Count ?? 0) * 10
|
||||
+ (a.HasCfd ? 5 : 0)
|
||||
+ (string.IsNullOrEmpty(a.ImageId) ? 0 : 15)
|
||||
+ (a.Name.Length > 3 ? 5 : 0)
|
||||
})
|
||||
.OrderByDescending(x => x.Score)
|
||||
@@ -147,7 +145,6 @@ public class AssetsDbService : IAssetsDbService
|
||||
existingEntity.Type = dtoAsset.Type;
|
||||
existingEntity.InstrumentCategory = dtoAsset.InstrumentCategory;
|
||||
existingEntity.HasCfd = dtoAsset.HasCfd;
|
||||
existingEntity.ImageId = dtoAsset.ImageId;
|
||||
existingEntity.LastUpdatedAt = now;
|
||||
|
||||
UpdateSubtypeProperties(existingEntity, dtoAsset);
|
||||
@@ -220,14 +217,13 @@ public class AssetsDbService : IAssetsDbService
|
||||
existingEntity.Type != dto.Type ||
|
||||
existingEntity.InstrumentCategory != dto.InstrumentCategory ||
|
||||
existingEntity.HasCfd != dto.HasCfd ||
|
||||
existingEntity.ImageId != dto.ImageId ||
|
||||
!existingEntity.Tags.SequenceEqual(mappedTags))
|
||||
!existingEntity.Tags.Select(t => t.Id).Order().SequenceEqual(mappedTags.Select(t => t.Id).Order()) ||
|
||||
HasSubtypeChanges(existingEntity, dto))
|
||||
{
|
||||
existingEntity.Name = dto.Name;
|
||||
existingEntity.Type = dto.Type;
|
||||
existingEntity.InstrumentCategory = dto.InstrumentCategory;
|
||||
existingEntity.HasCfd = dto.HasCfd;
|
||||
existingEntity.ImageId = dto.ImageId;
|
||||
existingEntity.LastUpdatedAt = now;
|
||||
existingEntity.Tags = mappedTags;
|
||||
|
||||
@@ -255,18 +251,8 @@ public class AssetsDbService : IAssetsDbService
|
||||
Type = stock.Type,
|
||||
InstrumentCategory = stock.InstrumentCategory,
|
||||
HasCfd = stock.HasCfd,
|
||||
ImageId = stock.ImageId,
|
||||
DerivativeProductCategories = stock.DerivativeProductCategories?.ToList() ?? new List<string>()
|
||||
},
|
||||
TradeRepublicCrypto crypto => new CryptoEntity
|
||||
{
|
||||
Isin = crypto.Isin,
|
||||
Name = crypto.Name,
|
||||
Type = crypto.Type,
|
||||
InstrumentCategory = crypto.InstrumentCategory,
|
||||
HasCfd = crypto.HasCfd,
|
||||
ImageId = crypto.ImageId
|
||||
},
|
||||
TradeRepublicEtf etf => new EtfEntity
|
||||
{
|
||||
Isin = etf.Isin,
|
||||
@@ -274,7 +260,6 @@ public class AssetsDbService : IAssetsDbService
|
||||
Type = etf.Type,
|
||||
InstrumentCategory = etf.InstrumentCategory,
|
||||
HasCfd = etf.HasCfd,
|
||||
ImageId = etf.ImageId,
|
||||
DerivativeProductCategories = etf.DerivativeProductCategories?.ToList() ?? new List<string>()
|
||||
},
|
||||
TradeRepublicSynthetic syn => new SyntheticEntity
|
||||
@@ -284,39 +269,15 @@ public class AssetsDbService : IAssetsDbService
|
||||
Type = syn.Type,
|
||||
InstrumentCategory = syn.InstrumentCategory,
|
||||
HasCfd = syn.HasCfd,
|
||||
ImageId = syn.ImageId,
|
||||
DerivativeProductCategories = syn.DerivativeProductCategories?.ToList() ?? new List<string>()
|
||||
},
|
||||
TradeRepublicBond bond => new BondEntity
|
||||
{
|
||||
Isin = bond.Isin,
|
||||
Name = bond.Name,
|
||||
Type = bond.Type,
|
||||
InstrumentCategory = bond.InstrumentCategory,
|
||||
HasCfd = bond.HasCfd,
|
||||
ImageId = bond.ImageId,
|
||||
BondIssuerName = bond.BondIssuerName,
|
||||
SearchSubtitle = bond.SearchSubtitle
|
||||
},
|
||||
TradeRepublicDerivative deriv => new DerivativeEntity
|
||||
{
|
||||
Isin = deriv.Isin,
|
||||
Name = deriv.Name,
|
||||
Type = deriv.Type,
|
||||
InstrumentCategory = deriv.InstrumentCategory,
|
||||
HasCfd = deriv.HasCfd,
|
||||
ImageId = deriv.ImageId,
|
||||
UnderlyingIsin = deriv.UnderlyingIsin,
|
||||
DerivativeProductCategories = deriv.DerivativeProductCategories?.ToList() ?? new List<string>()
|
||||
},
|
||||
_ => new StockEntity
|
||||
{
|
||||
Isin = dto.Isin,
|
||||
Name = dto.Name,
|
||||
Type = dto.Type,
|
||||
InstrumentCategory = dto.InstrumentCategory,
|
||||
HasCfd = dto.HasCfd,
|
||||
ImageId = dto.ImageId
|
||||
HasCfd = dto.HasCfd
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -334,14 +295,21 @@ public class AssetsDbService : IAssetsDbService
|
||||
case SyntheticEntity syn when dto is TradeRepublicSynthetic synDto:
|
||||
syn.DerivativeProductCategories = synDto.DerivativeProductCategories?.ToList() ?? new List<string>();
|
||||
break;
|
||||
case BondEntity bond when dto is TradeRepublicBond b:
|
||||
bond.BondIssuerName = b.BondIssuerName;
|
||||
bond.SearchSubtitle = b.SearchSubtitle;
|
||||
break;
|
||||
case DerivativeEntity deriv when dto is TradeRepublicDerivative d:
|
||||
deriv.UnderlyingIsin = d.UnderlyingIsin;
|
||||
deriv.DerivativeProductCategories = d.DerivativeProductCategories?.ToList() ?? new List<string>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool HasSubtypeChanges(AssetEntity entity, TradeRepublicAsset dto)
|
||||
{
|
||||
switch (entity)
|
||||
{
|
||||
case StockEntity stock when dto is TradeRepublicStock s:
|
||||
return !(stock.DerivativeProductCategories?.SequenceEqual(s.DerivativeProductCategories ?? Array.Empty<string>()) ?? (s.DerivativeProductCategories == null || !s.DerivativeProductCategories.Any()));
|
||||
case EtfEntity etf when dto is TradeRepublicEtf e:
|
||||
return !(etf.DerivativeProductCategories?.SequenceEqual(e.DerivativeProductCategories ?? Array.Empty<string>()) ?? (e.DerivativeProductCategories == null || !e.DerivativeProductCategories.Any()));
|
||||
case SyntheticEntity syn when dto is TradeRepublicSynthetic synDto:
|
||||
return !(syn.DerivativeProductCategories?.SequenceEqual(synDto.DerivativeProductCategories ?? Array.Empty<string>()) ?? (synDto.DerivativeProductCategories == null || !synDto.DerivativeProductCategories.Any()));
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,24 +330,6 @@ public class AssetsDbService : IAssetsDbService
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
/// <summary>Inherits documentation from interface.</summary>
|
||||
public async Task UpdateAssetImageIdAsync(string isin, string imageId)
|
||||
{
|
||||
var existingAssets = await _context.TradeRepublicAssets
|
||||
.Where(a => a.Isin == isin)
|
||||
.ToListAsync();
|
||||
|
||||
if (existingAssets.Count > 0)
|
||||
{
|
||||
foreach (var asset in existingAssets)
|
||||
{
|
||||
asset.ImageId = imageId;
|
||||
}
|
||||
await _context.SaveChangesAsync();
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Updated ImageId for ISIN {Isin} in database to '{ImageId}'", isin, imageId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Inherits documentation from interface.</summary>
|
||||
public async Task<bool> DeleteAssetAsync(string isin)
|
||||
{
|
||||
@@ -412,7 +362,6 @@ public class AssetsDbService : IAssetsDbService
|
||||
int pageIndex = Math.Max(0, page ?? 0);
|
||||
|
||||
decimal levQuery = targetLeverage.HasValue && targetLeverage.Value > 0 ? targetLeverage.Value : 0m;
|
||||
|
||||
string trAfter = !string.IsNullOrEmpty(after) ? after : (pageIndex > 0 ? pageIndex.ToString() : "0");
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Fetching derivatives for {Isin} (OptionType: {Option}, Leverage: {Lev}, Page: {Page}, TR-After: {After})",
|
||||
@@ -438,8 +387,7 @@ public class AssetsDbService : IAssetsDbService
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var isins = fetchedItems.Select(r => r.Isin).ToList();
|
||||
var existingDerivatives = await _context.TradeRepublicAssets
|
||||
.OfType<DerivativeEntity>()
|
||||
var existingDerivatives = await _context.Derivatives
|
||||
.Where(d => isins.Contains(d.Isin))
|
||||
.ToDictionaryAsync(d => d.Isin, cancellationToken);
|
||||
|
||||
@@ -466,14 +414,13 @@ public class AssetsDbService : IAssetsDbService
|
||||
existing.NextGenProductCategoryName = item.NextGenProductCategoryName;
|
||||
existing.Issuer = item.Issuer;
|
||||
existing.IssuerDisplayName = item.IssuerDisplayName;
|
||||
existing.IssuerImageId = item.IssuerImageId;
|
||||
existing.Size = item.Size;
|
||||
existing.Factor = item.Factor;
|
||||
existing.Delta = item.Delta;
|
||||
existing.Currency = item.Currency;
|
||||
existing.LastUpdatedAt = now;
|
||||
|
||||
_context.TradeRepublicAssets.Update(existing);
|
||||
_context.Derivatives.Update(existing);
|
||||
resultEntities.Add(existing);
|
||||
}
|
||||
else
|
||||
@@ -482,8 +429,6 @@ public class AssetsDbService : IAssetsDbService
|
||||
{
|
||||
Isin = item.Isin,
|
||||
Name = !string.IsNullOrWhiteSpace(item.ProductCategoryName) ? item.ProductCategoryName : item.Isin,
|
||||
Type = "derivative",
|
||||
InstrumentCategory = "derivative",
|
||||
UnderlyingIsin = underlyingIsin,
|
||||
Strike = item.Strike ?? 0m,
|
||||
Barrier = item.Barrier ?? 0m,
|
||||
@@ -494,27 +439,33 @@ public class AssetsDbService : IAssetsDbService
|
||||
NextGenProductCategoryName = item.NextGenProductCategoryName,
|
||||
Issuer = item.Issuer,
|
||||
IssuerDisplayName = item.IssuerDisplayName,
|
||||
IssuerImageId = item.IssuerImageId,
|
||||
Size = item.Size,
|
||||
Factor = item.Factor,
|
||||
Delta = item.Delta,
|
||||
Currency = item.Currency,
|
||||
LastUpdatedAt = now
|
||||
LastUpdatedAt = now,
|
||||
CreatedAt = now
|
||||
};
|
||||
|
||||
await _context.TradeRepublicAssets.AddAsync(newDeriv, cancellationToken);
|
||||
await _context.Derivatives.AddAsync(newDeriv, cancellationToken);
|
||||
resultEntities.Add(newDeriv);
|
||||
}
|
||||
}
|
||||
|
||||
await _context.SaveChangesAsync(cancellationToken);
|
||||
return resultEntities;
|
||||
return resultEntities.Where(d => d.Barrier > 0 && d.Leverage > 0).ToList();
|
||||
}
|
||||
|
||||
return await _context.TradeRepublicAssets
|
||||
.OfType<DerivativeEntity>()
|
||||
var dbQuery = _context.Derivatives
|
||||
.AsNoTracking()
|
||||
.Where(d => d.UnderlyingIsin == underlyingIsin)
|
||||
.Where(d => d.UnderlyingIsin == underlyingIsin && d.OptionType == targetOptionType && d.Barrier > 0 && d.Leverage > 0);
|
||||
|
||||
if (targetLeverage.HasValue && targetLeverage.Value > 0)
|
||||
{
|
||||
dbQuery = dbQuery.OrderBy(d => Math.Abs(d.Leverage - targetLeverage.Value));
|
||||
}
|
||||
|
||||
return await dbQuery
|
||||
.Take(pageSize)
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticAssets.Models;
|
||||
using FinlyticCore.Models.Assets;
|
||||
using FinlyticAssets.Util;
|
||||
using FinlyticCore.Services;
|
||||
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticAssets.Util;
|
||||
using FinlyticCore.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace FinlyticAssets.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Dedicated background service that periodically scans for missing asset logos in the local storage directory
|
||||
/// and fetches them in batches from Trade Republic CDN.
|
||||
/// Swaps missing/404 logos with a clean SVG placeholder image and triggers ReCreateIndexFileAsync.
|
||||
/// </summary>
|
||||
public class LogoFetcherBackgroundService : BackgroundService
|
||||
{
|
||||
private readonly IFinlyticLogger<LogoFetcherBackgroundService> _finlyticLogger;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
private const string PlaceholderSvg = """
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" width="100" height="100">
|
||||
<rect width="100" height="100" rx="30" fill="#1E293B"/>
|
||||
<path d="M 30 65 L 45 45 L 60 55 L 75 35" fill="none" stroke="#10B981" stroke-width="6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<circle cx="75" cy="35" r="5" fill="#06B6D4"/>
|
||||
</svg>
|
||||
""";
|
||||
|
||||
public LogoFetcherBackgroundService(
|
||||
IFinlyticLogger<LogoFetcherBackgroundService> finlyticLogger,
|
||||
IServiceScopeFactory scopeFactory)
|
||||
{
|
||||
_finlyticLogger = finlyticLogger;
|
||||
_scopeFactory = scopeFactory;
|
||||
_httpClient = new HttpClient();
|
||||
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
|
||||
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "image/svg+xml,image/*,*/*");
|
||||
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://traderepublic.com/");
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] LogoFetcherBackgroundService started. Will fetch missing logos periodically.");
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessMissingLogosBatchAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Error occurred while executing logo batch fetch.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] LogoFetcherBackgroundService stopped.");
|
||||
}
|
||||
|
||||
private async Task ProcessMissingLogosBatchAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var dbService = scope.ServiceProvider.GetRequiredService<IAssetsDbService>();
|
||||
var indexService = scope.ServiceProvider.GetRequiredService<IAssetsIndexService>();
|
||||
|
||||
var validAssets = await dbService.GetAllValidAssetsAsync();
|
||||
if (validAssets == null || !validAssets.Any()) return;
|
||||
|
||||
string directoryPath = Volumes.LogosRelativePath;
|
||||
if (!Directory.Exists(directoryPath))
|
||||
{
|
||||
Directory.CreateDirectory(directoryPath);
|
||||
}
|
||||
|
||||
var missingIsins = validAssets
|
||||
.Select(a => a.Isin?.Trim().ToUpperInvariant())
|
||||
.Where(isin => !string.IsNullOrEmpty(isin))
|
||||
.Distinct()
|
||||
.Where(isin => !File.Exists(Path.Combine(directoryPath, $"{isin}.svg")))
|
||||
.ToList();
|
||||
|
||||
if (missingIsins.Count == 0)
|
||||
{
|
||||
await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] All asset logos are downloaded and up to date.");
|
||||
return;
|
||||
}
|
||||
|
||||
var batchToFetch = missingIsins.Take(60).ToList();
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Found {Count} missing logos on disk. Fetching bulk batch of {BatchSize} logos...", missingIsins.Count, batchToFetch.Count);
|
||||
|
||||
int successCount = 0;
|
||||
|
||||
foreach (var isin in batchToFetch)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
|
||||
string targetUrl = $"https://assets.traderepublic.com/img/logos/{isin}/v2/dark.min.svg";
|
||||
string filePath = Path.Combine(directoryPath, $"{isin}.svg");
|
||||
string dbImageEndpoint = $"/api/v1/logo/{isin}";
|
||||
|
||||
try
|
||||
{
|
||||
using var response = await _httpClient.GetAsync(targetUrl, stoppingToken);
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
byte[] data = await response.Content.ReadAsByteArrayAsync(stoppingToken);
|
||||
await File.WriteAllBytesAsync(filePath, data, stoppingToken);
|
||||
successCount++;
|
||||
}
|
||||
else
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Logo not found on CDN for ISIN {Isin} (HTTP {StatusCode}). Saving SVG placeholder.", isin, response.StatusCode);
|
||||
byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg);
|
||||
await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken);
|
||||
successCount++;
|
||||
}
|
||||
|
||||
await dbService.UpdateAssetImageIdAsync(isin, dbImageEndpoint);
|
||||
}
|
||||
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Exception while downloading logo for ISIN {Isin} from {Url}. Saving SVG placeholder.", isin, targetUrl);
|
||||
try
|
||||
{
|
||||
byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg);
|
||||
await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken);
|
||||
successCount++;
|
||||
|
||||
await dbService.UpdateAssetImageIdAsync(isin, dbImageEndpoint);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
await Task.Delay(50, stoppingToken);
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Batch fetch complete. Successfully processed {SuccessCount}/{BatchSize} logos. Remaining missing: {Remaining}",
|
||||
successCount, batchToFetch.Count, missingIsins.Count - batchToFetch.Count);
|
||||
|
||||
if (successCount > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await indexService.ReCreateIndexFileAsync(stoppingToken);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Successfully updated index.json after logo batch fetch.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Failed to update index.json after logo batch fetch.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
using FinlyticAssets.Database;
|
||||
using FinlyticAssets.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace FinlyticAssets.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines the business logic for managing global application settings.
|
||||
/// Supports retrieving and updating (upserting) the central single-row configuration record.
|
||||
/// </summary>
|
||||
public interface ISettingsDbService
|
||||
{
|
||||
/// <summary>
|
||||
/// Retrieves the current global settings from the database.
|
||||
/// </summary>
|
||||
/// <returns>
|
||||
/// A task that represents the asynchronous operation. The task result contains the current <see cref="Settings"/>.
|
||||
/// If no settings exist in the database yet, a new instance initialized with default values is returned.
|
||||
/// </returns>
|
||||
public Task<Settings> GetSettings();
|
||||
|
||||
/// <summary>
|
||||
/// Persists the provided settings by updating the existing record or inserting the first one if the table is empty.
|
||||
/// </summary>
|
||||
/// <param name="settings">The new configuration values to be persisted.</param>
|
||||
/// <returns>
|
||||
/// A task that represents the asynchronous operation. The task result contains the freshly saved
|
||||
/// and tracked <see cref="Settings"/> instance.
|
||||
/// </returns>
|
||||
public Task<Settings> SaveSettings(Settings settings);
|
||||
|
||||
/// <summary>
|
||||
/// Updates settings from a key-value dictionary received via Admin Panel MQTT events.
|
||||
/// </summary>
|
||||
public Task UpdateSettingsFromDictionary(Dictionary<string, string> dictionary);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Implements the <see cref="ISettingsDbService"/> utilizing Entity Framework Core.
|
||||
/// This service is designed for a single-row table architecture to maintain stateful global configurations.
|
||||
/// </summary>
|
||||
public class SettingsDbService : ISettingsDbService
|
||||
{
|
||||
private readonly AssetsDbContext _context;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SettingsDbService"/> class with the required database context.
|
||||
/// </summary>
|
||||
/// <param name="context">The EF Core context used to access the assets database.</param>
|
||||
public SettingsDbService(AssetsDbContext context)
|
||||
{
|
||||
_context = context;
|
||||
}
|
||||
|
||||
/// <summary>Inherits documentation from interface.</summary>
|
||||
public async Task<Settings> GetSettings()
|
||||
{
|
||||
var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync();
|
||||
|
||||
if (settings == null)
|
||||
{
|
||||
settings = new Settings { Id = Guid.NewGuid() };
|
||||
_context.Settings.Add(settings);
|
||||
await _context.SaveChangesAsync();
|
||||
_context.ChangeTracker.Clear();
|
||||
}
|
||||
|
||||
return settings;
|
||||
}
|
||||
|
||||
/// <summary>Inherits documentation from interface.</summary>
|
||||
public async Task<Settings> SaveSettings(Settings settings)
|
||||
{
|
||||
var existing = await _context.Settings.FirstOrDefaultAsync();
|
||||
|
||||
if (existing == null)
|
||||
{
|
||||
if (settings.Id == Guid.Empty)
|
||||
{
|
||||
settings.Id = Guid.NewGuid();
|
||||
}
|
||||
_context.Settings.Add(settings);
|
||||
await _context.SaveChangesAsync();
|
||||
return settings;
|
||||
}
|
||||
else
|
||||
{
|
||||
existing.FinishedInitialScan = settings.FinishedInitialScan;
|
||||
existing.TradeRepublicMaxRequestPageSize = settings.TradeRepublicMaxRequestPageSize;
|
||||
existing.AssetUpdateTypeDelay = settings.AssetUpdateTypeDelay;
|
||||
existing.InitAssetUpdateTypeDelay = settings.InitAssetUpdateTypeDelay;
|
||||
existing.InitBatchAssetUpdateDelay = settings.InitBatchAssetUpdateDelay;
|
||||
existing.BatchAssetUpdateDelay = settings.BatchAssetUpdateDelay;
|
||||
existing.CurrentScanningType = settings.CurrentScanningType;
|
||||
existing.CurrentScanningPage = settings.CurrentScanningPage;
|
||||
_context.Settings.Update(existing);
|
||||
await _context.SaveChangesAsync();
|
||||
return existing;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Inherits documentation from interface.</summary>
|
||||
public async Task UpdateSettingsFromDictionary(Dictionary<string, string> dictionary)
|
||||
{
|
||||
var settings = await GetSettings();
|
||||
|
||||
foreach (var (key, value) in dictionary)
|
||||
{
|
||||
if (string.Equals(key, "TradeRepublicMaxRequestPageSize", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ps))
|
||||
settings.TradeRepublicMaxRequestPageSize = ps;
|
||||
else if (string.Equals(key, "AssetUpdateTypeDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var autd))
|
||||
settings.AssetUpdateTypeDelay = autd;
|
||||
else if (string.Equals(key, "InitAssetUpdateTypeDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var iautd))
|
||||
settings.InitAssetUpdateTypeDelay = iautd;
|
||||
else if (string.Equals(key, "BatchAssetUpdateDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var baud))
|
||||
settings.BatchAssetUpdateDelay = baud;
|
||||
else if (string.Equals(key, "InitBatchAssetUpdateDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ibaud))
|
||||
settings.InitBatchAssetUpdateDelay = ibaud;
|
||||
else if (string.Equals(key, "FinishedInitialScan", StringComparison.OrdinalIgnoreCase) && bool.TryParse(value, out var fis))
|
||||
settings.FinishedInitialScan = fis;
|
||||
}
|
||||
|
||||
await SaveSettings(settings);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user