using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.Simulation; using FinlyticCore.Dtos.TechnicalAnalysis; using FinlyticCore.Services; using FinlyticSimulation.Database; using FinlyticSimulation.Database.Entities; using FinlyticSimulation.Engine; using FinlyticSimulation.Services.Mqtt; using FinlyticSimulation.Settings; using FinlyticTechnicals.Patterns; using FinlyticTechnicals.Services; using FinlyticTechnicals.Strategies; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; namespace FinlyticSimulation.Services; public record SimGetCandlesRequest(string Isin, string Timeframe); public class QuantSimulationEngine : IQuantSimulationEngine { private readonly IServiceScopeFactory _scopeFactory; private readonly IEnumerable _strategies; private readonly IEnumerable _patternDetectors; private readonly IYahooMarketDataScraper _yahooScraper; private readonly ISimulationRpcClient _rpcClient; private readonly ISettingsService _settingsService; private readonly IFinlyticLogger _logger; public QuantSimulationEngine( IServiceScopeFactory scopeFactory, IEnumerable strategies, IEnumerable patternDetectors, IYahooMarketDataScraper yahooScraper, ISimulationRpcClient rpcClient, ISettingsService settingsService, IFinlyticLogger logger) { _scopeFactory = scopeFactory; _strategies = strategies; _patternDetectors = patternDetectors; _yahooScraper = yahooScraper; _rpcClient = rpcClient; _settingsService = settingsService; _logger = logger; } public async Task RunBacktestAsync(BacktestRequestDto request, CancellationToken cancellationToken = default) { var cleanIsin = request.Isin.Trim().ToUpperInvariant(); var strategyKey = request.StrategyKey.Trim(); var strategy = _strategies.FirstOrDefault(s => string.Equals(s.StrategyKey, strategyKey, StringComparison.OrdinalIgnoreCase)); if (strategy == null) { throw new ArgumentException($"Technical Strategy '{strategyKey}' not recognized or registered."); } await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel, "[SimulationEngine] Starting backtest for {Isin} ({Symbol}) using strategy {Strategy} on {Timeframe}...", cleanIsin, request.Symbol, strategy.StrategyName, request.Timeframe); // 1. Fetch Historical Candles (first try Yahoo, then FTA fallback) IReadOnlyList? candles = null; try { string ticker = request.Symbol; if (string.IsNullOrWhiteSpace(ticker)) { ticker = await _yahooScraper.ResolveTickerFromIsinAsync(cleanIsin, cancellationToken) ?? cleanIsin; } candles = await _yahooScraper.FetchHistoricalCandlesAsync(ticker, range: ResolveYahooRange(request.Timeframe), interval: request.Timeframe, cancellationToken); } catch (Exception ex) { await _logger.LogWarningAsync(SimulationSettingKeys.SimulationChannel, ex, "[SimulationEngine] Yahoo candle fetch failed for {Isin}. Trying FTA RPC.", cleanIsin); } if (candles == null || candles.Count < 30) { candles = await _rpcClient.SendRpcRequestAsync, SimGetCandlesRequest>( "ta_GetCandles", new SimGetCandlesRequest(cleanIsin, request.Timeframe), TimeSpan.FromSeconds(5) ); } if (candles == null || candles.Count < 30) { throw new InvalidOperationException($"Insufficient historical candle data found for {cleanIsin} to execute backtest."); } // Filter date range if specified var filteredCandles = candles .Where(c => c.Timestamp >= request.StartDateUtc && c.Timestamp <= request.EndDateUtc) .OrderBy(c => c.Timestamp) .ToList(); if (filteredCandles.Count < 30) { filteredCandles = candles.OrderBy(c => c.Timestamp).ToList(); } // 2. Run Replay var slippagePercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultSlippagePercent, cancellationToken); var orderFeeEur = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultOrderFeeEur, cancellationToken); var knockOutBufferPercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.KnockOutBarrierBufferPercent, cancellationToken); var defaultTrailingStopPercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultTrailingStopPercent, cancellationToken); var runner = new HistoricalReplayRunner(strategy, _patternDetectors); var report = runner.Run(filteredCandles, request, slippagePercent, orderFeeEur, knockOutBufferPercent, defaultTrailingStopPercent); // 3. Persist Simulation Run to DB using (var scope = _scopeFactory.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var runEntity = new SimulationRunEntity { Id = report.RunId, Isin = cleanIsin, Symbol = request.Symbol, StrategyKey = strategy.StrategyKey, Timeframe = request.Timeframe, StartDateUtc = report.StartDateUtc, EndDateUtc = report.EndDateUtc, StartingCapital = request.StartingCapital, TotalTrades = report.TotalTrades, WinningTrades = report.WinningTrades, LosingTrades = report.LosingTrades, WinRatePercent = report.WinRatePercent, ProfitFactor = report.ProfitFactor, MaxDrawdownPercent = report.MaxDrawdownPercent, TotalReturnPercent = report.TotalReturnPercent, ExpectancyEur = report.ExpectancyEur, SharpeRatio = report.SharpeRatio, ReportJson = report, CreatedAtUtc = DateTime.UtcNow }; db.SimulationRuns.Add(runEntity); // 4. Update Strategy Reliability Matrix decimal minTrades = await _settingsService.GetSettingAsync(SimulationSettingKeys.MinSampleTradesForApproval, cancellationToken); decimal highPf = await _settingsService.GetSettingAsync(SimulationSettingKeys.HighProfitFactorThreshold, cancellationToken); decimal lowPf = await _settingsService.GetSettingAsync(SimulationSettingKeys.LowProfitFactorThreshold, cancellationToken); var verdict = ReliabilityMatrixCalculator.Calculate(report, minTrades, highPf, lowPf); var matrixEntry = await db.StrategyMatrix.FirstOrDefaultAsync( m => m.Isin == cleanIsin && m.StrategyKey == strategy.StrategyKey && m.Timeframe == request.Timeframe, cancellationToken); if (matrixEntry == null) { matrixEntry = new SimulationStrategyMatrixEntity { Isin = cleanIsin, StrategyKey = strategy.StrategyKey, Timeframe = request.Timeframe, SampleTradesCount = report.TotalTrades, WinRatePercent = report.WinRatePercent, ProfitFactor = report.ProfitFactor, MaxDrawdownPercent = report.MaxDrawdownPercent, ReliabilityScore = verdict.ReliabilityScore, IsApproved = verdict.IsApproved, RecommendedAction = verdict.RecommendedAction, LastBacktestRunId = report.RunId, UpdatedAtUtc = DateTime.UtcNow }; db.StrategyMatrix.Add(matrixEntry); } else { matrixEntry.SampleTradesCount = report.TotalTrades; matrixEntry.WinRatePercent = report.WinRatePercent; matrixEntry.ProfitFactor = report.ProfitFactor; matrixEntry.MaxDrawdownPercent = report.MaxDrawdownPercent; matrixEntry.ReliabilityScore = verdict.ReliabilityScore; matrixEntry.IsApproved = verdict.IsApproved; matrixEntry.RecommendedAction = verdict.RecommendedAction; matrixEntry.LastBacktestRunId = report.RunId; matrixEntry.UpdatedAtUtc = DateTime.UtcNow; } await db.SaveChangesAsync(cancellationToken); await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel, "[SimulationEngine] Backtest finished for {Isin} ({Strategy}): Trades={Trades}, WR={WR:F1}%, PF={PF:F2}, Action={Action}", cleanIsin, strategy.StrategyKey, report.TotalTrades, report.WinRatePercent, report.ProfitFactor, verdict.RecommendedAction); } return report; } public async Task GetStrategyReliabilityAsync( string isin, string strategyKey, string timeframe = "15m", CancellationToken cancellationToken = default) { var cleanIsin = isin.Trim().ToUpperInvariant(); using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var entry = await db.StrategyMatrix.AsNoTracking().FirstOrDefaultAsync( m => m.Isin == cleanIsin && m.StrategyKey == strategyKey && m.Timeframe == timeframe, cancellationToken); if (entry == null) return null; return new StrategyAssetReliabilityDto( Isin: entry.Isin, StrategyKey: entry.StrategyKey, ReliabilityScore: entry.ReliabilityScore, WinRatePercent: entry.WinRatePercent, ProfitFactor: entry.ProfitFactor, SampleTradeCount: entry.SampleTradesCount, IsStrategyApprovedForAsset: entry.IsApproved, RecommendedAction: entry.RecommendedAction ); } public async Task> GetMatrixForAssetAsync( string isin, CancellationToken cancellationToken = default) { var cleanIsin = isin.Trim().ToUpperInvariant(); using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var entries = await db.StrategyMatrix.AsNoTracking() .Where(m => m.Isin == cleanIsin) .OrderByDescending(m => m.ReliabilityScore) .ToListAsync(cancellationToken); return entries.Select(e => new StrategyAssetReliabilityDto( Isin: e.Isin, StrategyKey: e.StrategyKey, ReliabilityScore: e.ReliabilityScore, WinRatePercent: e.WinRatePercent, ProfitFactor: e.ProfitFactor, SampleTradeCount: e.SampleTradesCount, IsStrategyApprovedForAsset: e.IsApproved, RecommendedAction: e.RecommendedAction )).ToList(); } /// public async Task> GetBacktestHistoryAsync(GetBacktestHistoryRequest request, CancellationToken cancellationToken = default) { var cleanIsin = request.Isin.Trim().ToUpperInvariant(); int limit = Math.Clamp(request.Limit <= 0 ? 20 : request.Limit, 1, 100); using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var query = db.SimulationRuns.AsNoTracking().Where(r => r.Isin == cleanIsin); if (!string.IsNullOrWhiteSpace(request.StrategyKey)) { query = query.Where(r => r.StrategyKey == request.StrategyKey); } var runs = await query .OrderByDescending(r => r.CreatedAtUtc) .Take(limit) .ToListAsync(cancellationToken); return runs.Select(r => new BacktestHistoryEntryDto( RunId: r.Id, Isin: r.Isin, Symbol: r.Symbol, StrategyKey: r.StrategyKey, Timeframe: r.Timeframe, StartDateUtc: r.StartDateUtc, EndDateUtc: r.EndDateUtc, TotalTrades: r.TotalTrades, WinRatePercent: r.WinRatePercent, ProfitFactor: r.ProfitFactor, MaxDrawdownPercent: r.MaxDrawdownPercent, TotalReturnPercent: r.TotalReturnPercent, SharpeRatio: r.SharpeRatio, CreatedAtUtc: r.CreatedAtUtc )).ToList(); } /// public async Task GetBacktestRunDetailAsync(Guid runId, CancellationToken cancellationToken = default) { using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var run = await db.SimulationRuns.AsNoTracking().FirstOrDefaultAsync(r => r.Id == runId, cancellationToken); return run?.ReportJson; } /// public async Task GetStrategyParametersAsync(string isin, string strategyKey, CancellationToken cancellationToken = default) { var cleanIsin = isin.Trim().ToUpperInvariant(); using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var entity = await db.StrategyParameters.AsNoTracking() .FirstOrDefaultAsync(p => p.Isin == cleanIsin && p.StrategyKey == strategyKey, cancellationToken); if (entity == null) return null; return new StrategyParameterProfileDto(entity.Isin, entity.StrategyKey, entity.Parameters, entity.UpdatedAtUtc); } /// public async Task SaveStrategyParametersAsync(string isin, string strategyKey, Dictionary parameters, CancellationToken cancellationToken = default) { var cleanIsin = isin.Trim().ToUpperInvariant(); using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var entity = await db.StrategyParameters .FirstOrDefaultAsync(p => p.Isin == cleanIsin && p.StrategyKey == strategyKey, cancellationToken); var now = DateTime.UtcNow; if (entity == null) { entity = new SimulationStrategyParameterEntity { Isin = cleanIsin, StrategyKey = strategyKey, Parameters = parameters, UpdatedAtUtc = now }; db.StrategyParameters.Add(entity); } else { entity.Parameters = parameters; entity.UpdatedAtUtc = now; } await db.SaveChangesAsync(cancellationToken); await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel, "[SimulationEngine] Saved parameter profile for {Isin} ({Strategy}): {Count} override(s).", cleanIsin, strategyKey, parameters.Count); return new StrategyParameterProfileDto(entity.Isin, entity.StrategyKey, entity.Parameters, entity.UpdatedAtUtc); } /// /// Picks the Yahoo Finance chart-API range query parameter to request for a given candle /// , honoring Yahoo's real, publicly documented per-interval history limits /// (the same limits every Yahoo-chart-API client, e.g. Python's yfinance, has to respect) instead of /// the previous hardcoded "2y" for every interval - which silently under-delivered for anything /// finer than 1h (Yahoo does not retain 2 years of 5m/15m/30m bars) and needlessly under-fetched for 1d/1wk /// (which Yahoo happily serves far beyond 2 years). This directly determines how much real history a /// backtest on a given timeframe can actually cover. /// private static string ResolveYahooRange(string timeframe) => timeframe.Trim().ToLowerInvariant() switch { "1m" => "7d", "5m" or "15m" or "30m" => "60d", "1h" or "60m" => "730d", "1wk" => "10y", _ => "5y" // 1d and anything else Yahoo retains for many years. }; }