using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticAssets.Util; using FinlyticCore.Dtos.TradeRepublic; using FinlyticCore.Models.Assets; using FinlyticCore.Services; using FinlyticCore.Services.TradeRepublic; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; namespace FinlyticAssets.Services; /// /// A background service that runs a continuous asset synchronization loop for Stocks and ETFs, /// retrieving, updating, and indexing assets while downloading logos inline. /// public class AssetScannerBackgroundService : BackgroundService { private readonly IServiceScopeFactory _serviceScopeFactory; private readonly IFinlyticLogger _finlyticLogger; private static readonly AssetType[] ScannedAssetTypes = [AssetType.Stock, AssetType.Fund]; private AssetsCount? _assetsCount; private AssetsCount? _currAssetsCount; public AssetScannerBackgroundService(IServiceScopeFactory serviceScopeFactory, IFinlyticLogger finlyticLogger) { _serviceScopeFactory = serviceScopeFactory; _finlyticLogger = finlyticLogger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] AssetScannerBackgroundService has started."); try { using var scope = _serviceScopeFactory.CreateScope(); var indexService = scope.ServiceProvider.GetRequiredService(); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Building initial asset index on service startup..."); await indexService.ReCreateIndexFileAsync(stoppingToken); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetScannerBackgroundService] Failed to build initial asset index on startup. Continuing service execution."); } do { try { using var scope = _serviceScopeFactory.CreateScope(); var tradeRepublicService = scope.ServiceProvider.GetRequiredService(); var settingsService = scope.ServiceProvider.GetRequiredService(); var assetsDbService = scope.ServiceProvider.GetRequiredService(); var indexService = scope.ServiceProvider.GetRequiredService(); var isAutoScanEnabled = await settingsService.GetSettingAsync(SettingKeys.ScannerEnableAutoScan, stoppingToken); if (!isAutoScanEnabled) { await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Auto scan is disabled in dynamic settings. Waiting 1 minute..."); await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); continue; } await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Requesting total asset counts from Trade Republic..."); _assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken); _currAssetsCount = new AssetsCount(); var initCurrentTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken); var initCurrentPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken); var isRecoveryMode = initCurrentPage > 0; Enum.TryParse(initCurrentTypeStr, true, out var initCurrentType); foreach (var type in ScannedAssetTypes) { if (stoppingToken.IsCancellationRequested) break; if (isRecoveryMode) { if (type != initCurrentType) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Recovery: {AssetType} was already processed. Skipping.", type); continue; } isRecoveryMode = false; } else { await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken); await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); } await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Processing asset type: {AssetType}...", type); await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken); var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken); var delaySeconds = finishedInitial ? await settingsService.GetSettingAsync(SettingKeys.ScannerTypeDelay, stoppingToken) : await settingsService.GetSettingAsync(SettingKeys.ScannerInitTypeDelay, stoppingToken); if (delaySeconds > 0) { var jitter = Random.Shared.Next(0, Math.Min(15, delaySeconds)); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Waiting {Delay}s before next asset type ({Type}).", delaySeconds + jitter, type); await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); } } await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); var finishedScan = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken); if (!finishedScan && !stoppingToken.IsCancellationRequested) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Initial scan successfully completed. Switching FinishedInitialScan to true."); await settingsService.SetSettingAsync(SettingKeys.ScannerFinishedInitialScan, true, stoppingToken); } var cycleDelay = await settingsService.GetSettingAsync(SettingKeys.ScannerCycleDelayMinutes, stoppingToken); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting {Minutes} minutes before starting the next cycle.", cycleDelay); await Task.Delay(TimeSpan.FromMinutes(cycleDelay), stoppingToken); } catch (Exception e) when (!stoppingToken.IsCancellationRequested) { await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, e, "[AssetScannerBackgroundService] An unhandled exception occurred in AssetScannerBackgroundService. Retrying in 10 seconds."); try { await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); } catch { /* Ignore */ } } } while (!stoppingToken.IsCancellationRequested); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] AssetScannerBackgroundService is stopping."); } private async Task HandleAssetType( AssetType type, ITradeRepublicService tradeRepublicService, ISettingsService settingsService, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) { var totalCount = _assetsCount?.GetCountFromType(type) ?? 0; if (totalCount == 0) { await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] No assets found for type {AssetType}.", type); return; } _currAssetsCount ??= new AssetsCount(); var currentItemOffset = 0; var configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken); var pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100); var currentScanningTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken); var currentScanningPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken); if (string.Equals(currentScanningTypeStr, type.ToString(), StringComparison.OrdinalIgnoreCase) && currentScanningPage > 0) { currentItemOffset = (currentScanningPage - 1) * pageSize; await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).", type, currentScanningPage, currentItemOffset); } while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested) { configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken); pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100); var currentPage = (currentItemOffset / pageSize) + 1; await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken); await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, currentPage, stoppingToken); await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}", type, currentPage, currentItemOffset, totalCount); var assets = await tradeRepublicService.GetAssets(type, currentPage, pageSize, stoppingToken); if (assets?.Results == null || assets.Results.Count == 0) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", type, currentPage); await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); break; } await ProcessAssets(assets.Results, assetsDbService, indexService, stoppingToken); _currAssetsCount.SetCountOfType(type, _currAssetsCount.GetCountFromType(type) + assets.Results.Count); currentItemOffset += assets.Results.Count; if (assets.Results.Count < pageSize) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Reached the last page for {AssetType}.", type); await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); break; } var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken); var delaySeconds = finishedInitial ? await settingsService.GetSettingAsync(SettingKeys.ScannerBatchDelay, stoppingToken) : await settingsService.GetSettingAsync(SettingKeys.ScannerInitBatchDelay, stoppingToken); if (delaySeconds > 0) { var maxJitter = Math.Min(5, delaySeconds); var jitter = Random.Shared.Next(0, maxJitter + 1); await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Waiting {Delay} seconds before the next batch.", delaySeconds + jitter); await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); } } await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Finished scanning {AssetType}. Total scanned in this cycle: {Count}/{Total}", type, _currAssetsCount.GetCountFromType(type), totalCount); } private async Task ProcessAssets( IList assets, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) { if (assets == null || assets.Count == 0) return; var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows); // Inline logo fetching for each asset in the batch if missing on disk string directoryPath = Volumes.LogosRelativePath; foreach (var a in assets) { if (stoppingToken.IsCancellationRequested) break; if (string.IsNullOrWhiteSpace(a.Isin)) continue; string cleanIsin = a.Isin.Trim().ToUpperInvariant(); string logoPath = Path.Combine(directoryPath, $"{cleanIsin}.svg"); if (!File.Exists(logoPath)) { try { await indexService.DownloadAndSaveLogoAsync(cleanIsin, stoppingToken); } catch { /* Ignore non-fatal logo fetch failures */ } } } if (changedRows > 0) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Database modifications detected. Recreating the asset index file..."); await indexService.ReCreateIndexFileAsync(stoppingToken); } } }