feat(simulation): add quant simulation microservice with virtual backtest broker and replay engine
This commit is contained in:
@@ -0,0 +1,376 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Dtos.Simulation;
|
||||
using FinlyticCore.Dtos.TechnicalAnalysis;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticSimulation.Database;
|
||||
using FinlyticSimulation.Database.Entities;
|
||||
using FinlyticSimulation.Engine;
|
||||
using FinlyticSimulation.Services.Mqtt;
|
||||
using FinlyticSimulation.Settings;
|
||||
using FinlyticTechnicals.Patterns;
|
||||
using FinlyticTechnicals.Services;
|
||||
using FinlyticTechnicals.Strategies;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace FinlyticSimulation.Services;
|
||||
|
||||
public record SimGetCandlesRequest(string Isin, string Timeframe);
|
||||
|
||||
public class QuantSimulationEngine : IQuantSimulationEngine
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IEnumerable<ITechnicalStrategy> _strategies;
|
||||
private readonly IEnumerable<IPatternDetector> _patternDetectors;
|
||||
private readonly IYahooMarketDataScraper _yahooScraper;
|
||||
private readonly ISimulationRpcClient _rpcClient;
|
||||
private readonly ISettingsService _settingsService;
|
||||
private readonly IFinlyticLogger<QuantSimulationEngine> _logger;
|
||||
|
||||
public QuantSimulationEngine(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IEnumerable<ITechnicalStrategy> strategies,
|
||||
IEnumerable<IPatternDetector> patternDetectors,
|
||||
IYahooMarketDataScraper yahooScraper,
|
||||
ISimulationRpcClient rpcClient,
|
||||
ISettingsService settingsService,
|
||||
IFinlyticLogger<QuantSimulationEngine> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_strategies = strategies;
|
||||
_patternDetectors = patternDetectors;
|
||||
_yahooScraper = yahooScraper;
|
||||
_rpcClient = rpcClient;
|
||||
_settingsService = settingsService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<BacktestReportDto> RunBacktestAsync(BacktestRequestDto request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cleanIsin = request.Isin.Trim().ToUpperInvariant();
|
||||
var strategyKey = request.StrategyKey.Trim();
|
||||
|
||||
var strategy = _strategies.FirstOrDefault(s => string.Equals(s.StrategyKey, strategyKey, StringComparison.OrdinalIgnoreCase));
|
||||
if (strategy == null)
|
||||
{
|
||||
throw new ArgumentException($"Technical Strategy '{strategyKey}' not recognized or registered.");
|
||||
}
|
||||
|
||||
await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel,
|
||||
"[SimulationEngine] Starting backtest for {Isin} ({Symbol}) using strategy {Strategy} on {Timeframe}...",
|
||||
cleanIsin, request.Symbol, strategy.StrategyName, request.Timeframe);
|
||||
|
||||
// 1. Fetch Historical Candles (first try Yahoo, then FTA fallback)
|
||||
IReadOnlyList<CandleDto>? candles = null;
|
||||
try
|
||||
{
|
||||
string ticker = request.Symbol;
|
||||
if (string.IsNullOrWhiteSpace(ticker))
|
||||
{
|
||||
ticker = await _yahooScraper.ResolveTickerFromIsinAsync(cleanIsin, cancellationToken) ?? cleanIsin;
|
||||
}
|
||||
candles = await _yahooScraper.FetchHistoricalCandlesAsync(ticker, range: ResolveYahooRange(request.Timeframe), interval: request.Timeframe, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _logger.LogWarningAsync(SimulationSettingKeys.SimulationChannel, ex,
|
||||
"[SimulationEngine] Yahoo candle fetch failed for {Isin}. Trying FTA RPC.", cleanIsin);
|
||||
}
|
||||
|
||||
|
||||
if (candles == null || candles.Count < 30)
|
||||
{
|
||||
candles = await _rpcClient.SendRpcRequestAsync<List<CandleDto>, SimGetCandlesRequest>(
|
||||
"ta_GetCandles",
|
||||
new SimGetCandlesRequest(cleanIsin, request.Timeframe),
|
||||
TimeSpan.FromSeconds(5)
|
||||
);
|
||||
}
|
||||
|
||||
if (candles == null || candles.Count < 30)
|
||||
{
|
||||
throw new InvalidOperationException($"Insufficient historical candle data found for {cleanIsin} to execute backtest.");
|
||||
}
|
||||
|
||||
// Filter date range if specified
|
||||
var filteredCandles = candles
|
||||
.Where(c => c.Timestamp >= request.StartDateUtc && c.Timestamp <= request.EndDateUtc)
|
||||
.OrderBy(c => c.Timestamp)
|
||||
.ToList();
|
||||
|
||||
if (filteredCandles.Count < 30)
|
||||
{
|
||||
filteredCandles = candles.OrderBy(c => c.Timestamp).ToList();
|
||||
}
|
||||
|
||||
// 2. Run Replay
|
||||
var slippagePercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultSlippagePercent, cancellationToken);
|
||||
var orderFeeEur = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultOrderFeeEur, cancellationToken);
|
||||
var knockOutBufferPercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.KnockOutBarrierBufferPercent, cancellationToken);
|
||||
var defaultTrailingStopPercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultTrailingStopPercent, cancellationToken);
|
||||
|
||||
var runner = new HistoricalReplayRunner(strategy, _patternDetectors);
|
||||
var report = runner.Run(filteredCandles, request, slippagePercent, orderFeeEur, knockOutBufferPercent, defaultTrailingStopPercent);
|
||||
|
||||
// 3. Persist Simulation Run to DB
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var runEntity = new SimulationRunEntity
|
||||
{
|
||||
Id = report.RunId,
|
||||
Isin = cleanIsin,
|
||||
Symbol = request.Symbol,
|
||||
StrategyKey = strategy.StrategyKey,
|
||||
Timeframe = request.Timeframe,
|
||||
StartDateUtc = report.StartDateUtc,
|
||||
EndDateUtc = report.EndDateUtc,
|
||||
StartingCapital = request.StartingCapital,
|
||||
TotalTrades = report.TotalTrades,
|
||||
WinningTrades = report.WinningTrades,
|
||||
LosingTrades = report.LosingTrades,
|
||||
WinRatePercent = report.WinRatePercent,
|
||||
ProfitFactor = report.ProfitFactor,
|
||||
MaxDrawdownPercent = report.MaxDrawdownPercent,
|
||||
TotalReturnPercent = report.TotalReturnPercent,
|
||||
ExpectancyEur = report.ExpectancyEur,
|
||||
SharpeRatio = report.SharpeRatio,
|
||||
ReportJson = report,
|
||||
CreatedAtUtc = DateTime.UtcNow
|
||||
};
|
||||
|
||||
db.SimulationRuns.Add(runEntity);
|
||||
|
||||
// 4. Update Strategy Reliability Matrix
|
||||
decimal minTrades = await _settingsService.GetSettingAsync(SimulationSettingKeys.MinSampleTradesForApproval, cancellationToken);
|
||||
decimal highPf = await _settingsService.GetSettingAsync(SimulationSettingKeys.HighProfitFactorThreshold, cancellationToken);
|
||||
decimal lowPf = await _settingsService.GetSettingAsync(SimulationSettingKeys.LowProfitFactorThreshold, cancellationToken);
|
||||
|
||||
var verdict = ReliabilityMatrixCalculator.Calculate(report, minTrades, highPf, lowPf);
|
||||
|
||||
var matrixEntry = await db.StrategyMatrix.FirstOrDefaultAsync(
|
||||
m => m.Isin == cleanIsin && m.StrategyKey == strategy.StrategyKey && m.Timeframe == request.Timeframe,
|
||||
cancellationToken);
|
||||
|
||||
if (matrixEntry == null)
|
||||
{
|
||||
matrixEntry = new SimulationStrategyMatrixEntity
|
||||
{
|
||||
Isin = cleanIsin,
|
||||
StrategyKey = strategy.StrategyKey,
|
||||
Timeframe = request.Timeframe,
|
||||
SampleTradesCount = report.TotalTrades,
|
||||
WinRatePercent = report.WinRatePercent,
|
||||
ProfitFactor = report.ProfitFactor,
|
||||
MaxDrawdownPercent = report.MaxDrawdownPercent,
|
||||
ReliabilityScore = verdict.ReliabilityScore,
|
||||
IsApproved = verdict.IsApproved,
|
||||
RecommendedAction = verdict.RecommendedAction,
|
||||
LastBacktestRunId = report.RunId,
|
||||
UpdatedAtUtc = DateTime.UtcNow
|
||||
};
|
||||
db.StrategyMatrix.Add(matrixEntry);
|
||||
}
|
||||
else
|
||||
{
|
||||
matrixEntry.SampleTradesCount = report.TotalTrades;
|
||||
matrixEntry.WinRatePercent = report.WinRatePercent;
|
||||
matrixEntry.ProfitFactor = report.ProfitFactor;
|
||||
matrixEntry.MaxDrawdownPercent = report.MaxDrawdownPercent;
|
||||
matrixEntry.ReliabilityScore = verdict.ReliabilityScore;
|
||||
matrixEntry.IsApproved = verdict.IsApproved;
|
||||
matrixEntry.RecommendedAction = verdict.RecommendedAction;
|
||||
matrixEntry.LastBacktestRunId = report.RunId;
|
||||
matrixEntry.UpdatedAtUtc = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
|
||||
await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel,
|
||||
"[SimulationEngine] Backtest finished for {Isin} ({Strategy}): Trades={Trades}, WR={WR:F1}%, PF={PF:F2}, Action={Action}",
|
||||
cleanIsin, strategy.StrategyKey, report.TotalTrades, report.WinRatePercent, report.ProfitFactor, verdict.RecommendedAction);
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
public async Task<StrategyAssetReliabilityDto?> GetStrategyReliabilityAsync(
|
||||
string isin,
|
||||
string strategyKey,
|
||||
string timeframe = "15m",
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var entry = await db.StrategyMatrix.AsNoTracking().FirstOrDefaultAsync(
|
||||
m => m.Isin == cleanIsin && m.StrategyKey == strategyKey && m.Timeframe == timeframe,
|
||||
cancellationToken);
|
||||
|
||||
if (entry == null) return null;
|
||||
|
||||
return new StrategyAssetReliabilityDto(
|
||||
Isin: entry.Isin,
|
||||
StrategyKey: entry.StrategyKey,
|
||||
ReliabilityScore: entry.ReliabilityScore,
|
||||
WinRatePercent: entry.WinRatePercent,
|
||||
ProfitFactor: entry.ProfitFactor,
|
||||
SampleTradeCount: entry.SampleTradesCount,
|
||||
IsStrategyApprovedForAsset: entry.IsApproved,
|
||||
RecommendedAction: entry.RecommendedAction
|
||||
);
|
||||
}
|
||||
|
||||
public async Task<List<StrategyAssetReliabilityDto>> GetMatrixForAssetAsync(
|
||||
string isin,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var entries = await db.StrategyMatrix.AsNoTracking()
|
||||
.Where(m => m.Isin == cleanIsin)
|
||||
.OrderByDescending(m => m.ReliabilityScore)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return entries.Select(e => new StrategyAssetReliabilityDto(
|
||||
Isin: e.Isin,
|
||||
StrategyKey: e.StrategyKey,
|
||||
ReliabilityScore: e.ReliabilityScore,
|
||||
WinRatePercent: e.WinRatePercent,
|
||||
ProfitFactor: e.ProfitFactor,
|
||||
SampleTradeCount: e.SampleTradesCount,
|
||||
IsStrategyApprovedForAsset: e.IsApproved,
|
||||
RecommendedAction: e.RecommendedAction
|
||||
)).ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<BacktestHistoryEntryDto>> GetBacktestHistoryAsync(GetBacktestHistoryRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cleanIsin = request.Isin.Trim().ToUpperInvariant();
|
||||
int limit = Math.Clamp(request.Limit <= 0 ? 20 : request.Limit, 1, 100);
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var query = db.SimulationRuns.AsNoTracking().Where(r => r.Isin == cleanIsin);
|
||||
if (!string.IsNullOrWhiteSpace(request.StrategyKey))
|
||||
{
|
||||
query = query.Where(r => r.StrategyKey == request.StrategyKey);
|
||||
}
|
||||
|
||||
var runs = await query
|
||||
.OrderByDescending(r => r.CreatedAtUtc)
|
||||
.Take(limit)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return runs.Select(r => new BacktestHistoryEntryDto(
|
||||
RunId: r.Id,
|
||||
Isin: r.Isin,
|
||||
Symbol: r.Symbol,
|
||||
StrategyKey: r.StrategyKey,
|
||||
Timeframe: r.Timeframe,
|
||||
StartDateUtc: r.StartDateUtc,
|
||||
EndDateUtc: r.EndDateUtc,
|
||||
TotalTrades: r.TotalTrades,
|
||||
WinRatePercent: r.WinRatePercent,
|
||||
ProfitFactor: r.ProfitFactor,
|
||||
MaxDrawdownPercent: r.MaxDrawdownPercent,
|
||||
TotalReturnPercent: r.TotalReturnPercent,
|
||||
SharpeRatio: r.SharpeRatio,
|
||||
CreatedAtUtc: r.CreatedAtUtc
|
||||
)).ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<BacktestReportDto?> GetBacktestRunDetailAsync(Guid runId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var run = await db.SimulationRuns.AsNoTracking().FirstOrDefaultAsync(r => r.Id == runId, cancellationToken);
|
||||
return run?.ReportJson;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<StrategyParameterProfileDto?> GetStrategyParametersAsync(string isin, string strategyKey, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var entity = await db.StrategyParameters.AsNoTracking()
|
||||
.FirstOrDefaultAsync(p => p.Isin == cleanIsin && p.StrategyKey == strategyKey, cancellationToken);
|
||||
|
||||
if (entity == null) return null;
|
||||
|
||||
return new StrategyParameterProfileDto(entity.Isin, entity.StrategyKey, entity.Parameters, entity.UpdatedAtUtc);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<StrategyParameterProfileDto> SaveStrategyParametersAsync(string isin, string strategyKey, Dictionary<string, decimal> parameters, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
||||
|
||||
var entity = await db.StrategyParameters
|
||||
.FirstOrDefaultAsync(p => p.Isin == cleanIsin && p.StrategyKey == strategyKey, cancellationToken);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
if (entity == null)
|
||||
{
|
||||
entity = new SimulationStrategyParameterEntity
|
||||
{
|
||||
Isin = cleanIsin,
|
||||
StrategyKey = strategyKey,
|
||||
Parameters = parameters,
|
||||
UpdatedAtUtc = now
|
||||
};
|
||||
db.StrategyParameters.Add(entity);
|
||||
}
|
||||
else
|
||||
{
|
||||
entity.Parameters = parameters;
|
||||
entity.UpdatedAtUtc = now;
|
||||
}
|
||||
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
|
||||
await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel,
|
||||
"[SimulationEngine] Saved parameter profile for {Isin} ({Strategy}): {Count} override(s).",
|
||||
cleanIsin, strategyKey, parameters.Count);
|
||||
|
||||
return new StrategyParameterProfileDto(entity.Isin, entity.StrategyKey, entity.Parameters, entity.UpdatedAtUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Picks the Yahoo Finance chart-API <c>range</c> query parameter to request for a given candle
|
||||
/// <paramref name="timeframe"/>, honoring Yahoo's real, publicly documented per-interval history limits
|
||||
/// (the same limits every Yahoo-chart-API client, e.g. Python's <c>yfinance</c>, has to respect) instead of
|
||||
/// the previous hardcoded <c>"2y"</c> for every interval - which silently under-delivered for anything
|
||||
/// finer than 1h (Yahoo does not retain 2 years of 5m/15m/30m bars) and needlessly under-fetched for 1d/1wk
|
||||
/// (which Yahoo happily serves far beyond 2 years). This directly determines how much real history a
|
||||
/// backtest on a given timeframe can actually cover.
|
||||
/// </summary>
|
||||
private static string ResolveYahooRange(string timeframe) => timeframe.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"1m" => "7d",
|
||||
"5m" or "15m" or "30m" => "60d",
|
||||
"1h" or "60m" => "730d",
|
||||
"1wk" => "10y",
|
||||
_ => "5y" // 1d and anything else Yahoo retains for many years.
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user