377 lines
16 KiB
C#
377 lines
16 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FinlyticCore.Dtos.Simulation;
|
|
using FinlyticCore.Dtos.TechnicalAnalysis;
|
|
using FinlyticCore.Services;
|
|
using FinlyticSimulation.Database;
|
|
using FinlyticSimulation.Database.Entities;
|
|
using FinlyticSimulation.Engine;
|
|
using FinlyticSimulation.Services.Mqtt;
|
|
using FinlyticSimulation.Settings;
|
|
using FinlyticTechnicals.Patterns;
|
|
using FinlyticTechnicals.Services;
|
|
using FinlyticTechnicals.Strategies;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
|
|
namespace FinlyticSimulation.Services;
|
|
|
|
public record SimGetCandlesRequest(string Isin, string Timeframe);
|
|
|
|
public class QuantSimulationEngine : IQuantSimulationEngine
|
|
{
|
|
private readonly IServiceScopeFactory _scopeFactory;
|
|
private readonly IEnumerable<ITechnicalStrategy> _strategies;
|
|
private readonly IEnumerable<IPatternDetector> _patternDetectors;
|
|
private readonly IYahooMarketDataScraper _yahooScraper;
|
|
private readonly ISimulationRpcClient _rpcClient;
|
|
private readonly ISettingsService _settingsService;
|
|
private readonly IFinlyticLogger<QuantSimulationEngine> _logger;
|
|
|
|
public QuantSimulationEngine(
|
|
IServiceScopeFactory scopeFactory,
|
|
IEnumerable<ITechnicalStrategy> strategies,
|
|
IEnumerable<IPatternDetector> patternDetectors,
|
|
IYahooMarketDataScraper yahooScraper,
|
|
ISimulationRpcClient rpcClient,
|
|
ISettingsService settingsService,
|
|
IFinlyticLogger<QuantSimulationEngine> logger)
|
|
{
|
|
_scopeFactory = scopeFactory;
|
|
_strategies = strategies;
|
|
_patternDetectors = patternDetectors;
|
|
_yahooScraper = yahooScraper;
|
|
_rpcClient = rpcClient;
|
|
_settingsService = settingsService;
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<BacktestReportDto> RunBacktestAsync(BacktestRequestDto request, CancellationToken cancellationToken = default)
|
|
{
|
|
var cleanIsin = request.Isin.Trim().ToUpperInvariant();
|
|
var strategyKey = request.StrategyKey.Trim();
|
|
|
|
var strategy = _strategies.FirstOrDefault(s => string.Equals(s.StrategyKey, strategyKey, StringComparison.OrdinalIgnoreCase));
|
|
if (strategy == null)
|
|
{
|
|
throw new ArgumentException($"Technical Strategy '{strategyKey}' not recognized or registered.");
|
|
}
|
|
|
|
await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel,
|
|
"[SimulationEngine] Starting backtest for {Isin} ({Symbol}) using strategy {Strategy} on {Timeframe}...",
|
|
cleanIsin, request.Symbol, strategy.StrategyName, request.Timeframe);
|
|
|
|
// 1. Fetch Historical Candles (first try Yahoo, then FTA fallback)
|
|
IReadOnlyList<CandleDto>? candles = null;
|
|
try
|
|
{
|
|
string ticker = request.Symbol;
|
|
if (string.IsNullOrWhiteSpace(ticker))
|
|
{
|
|
ticker = await _yahooScraper.ResolveTickerFromIsinAsync(cleanIsin, cancellationToken) ?? cleanIsin;
|
|
}
|
|
candles = await _yahooScraper.FetchHistoricalCandlesAsync(ticker, range: ResolveYahooRange(request.Timeframe), interval: request.Timeframe, cancellationToken);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _logger.LogWarningAsync(SimulationSettingKeys.SimulationChannel, ex,
|
|
"[SimulationEngine] Yahoo candle fetch failed for {Isin}. Trying FTA RPC.", cleanIsin);
|
|
}
|
|
|
|
|
|
if (candles == null || candles.Count < 30)
|
|
{
|
|
candles = await _rpcClient.SendRpcRequestAsync<List<CandleDto>, SimGetCandlesRequest>(
|
|
"ta_GetCandles",
|
|
new SimGetCandlesRequest(cleanIsin, request.Timeframe),
|
|
TimeSpan.FromSeconds(5)
|
|
);
|
|
}
|
|
|
|
if (candles == null || candles.Count < 30)
|
|
{
|
|
throw new InvalidOperationException($"Insufficient historical candle data found for {cleanIsin} to execute backtest.");
|
|
}
|
|
|
|
// Filter date range if specified
|
|
var filteredCandles = candles
|
|
.Where(c => c.Timestamp >= request.StartDateUtc && c.Timestamp <= request.EndDateUtc)
|
|
.OrderBy(c => c.Timestamp)
|
|
.ToList();
|
|
|
|
if (filteredCandles.Count < 30)
|
|
{
|
|
filteredCandles = candles.OrderBy(c => c.Timestamp).ToList();
|
|
}
|
|
|
|
// 2. Run Replay
|
|
var slippagePercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultSlippagePercent, cancellationToken);
|
|
var orderFeeEur = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultOrderFeeEur, cancellationToken);
|
|
var knockOutBufferPercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.KnockOutBarrierBufferPercent, cancellationToken);
|
|
var defaultTrailingStopPercent = await _settingsService.GetSettingAsync(SimulationSettingKeys.DefaultTrailingStopPercent, cancellationToken);
|
|
|
|
var runner = new HistoricalReplayRunner(strategy, _patternDetectors);
|
|
var report = runner.Run(filteredCandles, request, slippagePercent, orderFeeEur, knockOutBufferPercent, defaultTrailingStopPercent);
|
|
|
|
// 3. Persist Simulation Run to DB
|
|
using (var scope = _scopeFactory.CreateScope())
|
|
{
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var runEntity = new SimulationRunEntity
|
|
{
|
|
Id = report.RunId,
|
|
Isin = cleanIsin,
|
|
Symbol = request.Symbol,
|
|
StrategyKey = strategy.StrategyKey,
|
|
Timeframe = request.Timeframe,
|
|
StartDateUtc = report.StartDateUtc,
|
|
EndDateUtc = report.EndDateUtc,
|
|
StartingCapital = request.StartingCapital,
|
|
TotalTrades = report.TotalTrades,
|
|
WinningTrades = report.WinningTrades,
|
|
LosingTrades = report.LosingTrades,
|
|
WinRatePercent = report.WinRatePercent,
|
|
ProfitFactor = report.ProfitFactor,
|
|
MaxDrawdownPercent = report.MaxDrawdownPercent,
|
|
TotalReturnPercent = report.TotalReturnPercent,
|
|
ExpectancyEur = report.ExpectancyEur,
|
|
SharpeRatio = report.SharpeRatio,
|
|
ReportJson = report,
|
|
CreatedAtUtc = DateTime.UtcNow
|
|
};
|
|
|
|
db.SimulationRuns.Add(runEntity);
|
|
|
|
// 4. Update Strategy Reliability Matrix
|
|
decimal minTrades = await _settingsService.GetSettingAsync(SimulationSettingKeys.MinSampleTradesForApproval, cancellationToken);
|
|
decimal highPf = await _settingsService.GetSettingAsync(SimulationSettingKeys.HighProfitFactorThreshold, cancellationToken);
|
|
decimal lowPf = await _settingsService.GetSettingAsync(SimulationSettingKeys.LowProfitFactorThreshold, cancellationToken);
|
|
|
|
var verdict = ReliabilityMatrixCalculator.Calculate(report, minTrades, highPf, lowPf);
|
|
|
|
var matrixEntry = await db.StrategyMatrix.FirstOrDefaultAsync(
|
|
m => m.Isin == cleanIsin && m.StrategyKey == strategy.StrategyKey && m.Timeframe == request.Timeframe,
|
|
cancellationToken);
|
|
|
|
if (matrixEntry == null)
|
|
{
|
|
matrixEntry = new SimulationStrategyMatrixEntity
|
|
{
|
|
Isin = cleanIsin,
|
|
StrategyKey = strategy.StrategyKey,
|
|
Timeframe = request.Timeframe,
|
|
SampleTradesCount = report.TotalTrades,
|
|
WinRatePercent = report.WinRatePercent,
|
|
ProfitFactor = report.ProfitFactor,
|
|
MaxDrawdownPercent = report.MaxDrawdownPercent,
|
|
ReliabilityScore = verdict.ReliabilityScore,
|
|
IsApproved = verdict.IsApproved,
|
|
RecommendedAction = verdict.RecommendedAction,
|
|
LastBacktestRunId = report.RunId,
|
|
UpdatedAtUtc = DateTime.UtcNow
|
|
};
|
|
db.StrategyMatrix.Add(matrixEntry);
|
|
}
|
|
else
|
|
{
|
|
matrixEntry.SampleTradesCount = report.TotalTrades;
|
|
matrixEntry.WinRatePercent = report.WinRatePercent;
|
|
matrixEntry.ProfitFactor = report.ProfitFactor;
|
|
matrixEntry.MaxDrawdownPercent = report.MaxDrawdownPercent;
|
|
matrixEntry.ReliabilityScore = verdict.ReliabilityScore;
|
|
matrixEntry.IsApproved = verdict.IsApproved;
|
|
matrixEntry.RecommendedAction = verdict.RecommendedAction;
|
|
matrixEntry.LastBacktestRunId = report.RunId;
|
|
matrixEntry.UpdatedAtUtc = DateTime.UtcNow;
|
|
}
|
|
|
|
await db.SaveChangesAsync(cancellationToken);
|
|
|
|
await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel,
|
|
"[SimulationEngine] Backtest finished for {Isin} ({Strategy}): Trades={Trades}, WR={WR:F1}%, PF={PF:F2}, Action={Action}",
|
|
cleanIsin, strategy.StrategyKey, report.TotalTrades, report.WinRatePercent, report.ProfitFactor, verdict.RecommendedAction);
|
|
}
|
|
|
|
return report;
|
|
}
|
|
|
|
public async Task<StrategyAssetReliabilityDto?> GetStrategyReliabilityAsync(
|
|
string isin,
|
|
string strategyKey,
|
|
string timeframe = "15m",
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var cleanIsin = isin.Trim().ToUpperInvariant();
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var entry = await db.StrategyMatrix.AsNoTracking().FirstOrDefaultAsync(
|
|
m => m.Isin == cleanIsin && m.StrategyKey == strategyKey && m.Timeframe == timeframe,
|
|
cancellationToken);
|
|
|
|
if (entry == null) return null;
|
|
|
|
return new StrategyAssetReliabilityDto(
|
|
Isin: entry.Isin,
|
|
StrategyKey: entry.StrategyKey,
|
|
ReliabilityScore: entry.ReliabilityScore,
|
|
WinRatePercent: entry.WinRatePercent,
|
|
ProfitFactor: entry.ProfitFactor,
|
|
SampleTradeCount: entry.SampleTradesCount,
|
|
IsStrategyApprovedForAsset: entry.IsApproved,
|
|
RecommendedAction: entry.RecommendedAction
|
|
);
|
|
}
|
|
|
|
public async Task<List<StrategyAssetReliabilityDto>> GetMatrixForAssetAsync(
|
|
string isin,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var cleanIsin = isin.Trim().ToUpperInvariant();
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var entries = await db.StrategyMatrix.AsNoTracking()
|
|
.Where(m => m.Isin == cleanIsin)
|
|
.OrderByDescending(m => m.ReliabilityScore)
|
|
.ToListAsync(cancellationToken);
|
|
|
|
return entries.Select(e => new StrategyAssetReliabilityDto(
|
|
Isin: e.Isin,
|
|
StrategyKey: e.StrategyKey,
|
|
ReliabilityScore: e.ReliabilityScore,
|
|
WinRatePercent: e.WinRatePercent,
|
|
ProfitFactor: e.ProfitFactor,
|
|
SampleTradeCount: e.SampleTradesCount,
|
|
IsStrategyApprovedForAsset: e.IsApproved,
|
|
RecommendedAction: e.RecommendedAction
|
|
)).ToList();
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<List<BacktestHistoryEntryDto>> GetBacktestHistoryAsync(GetBacktestHistoryRequest request, CancellationToken cancellationToken = default)
|
|
{
|
|
var cleanIsin = request.Isin.Trim().ToUpperInvariant();
|
|
int limit = Math.Clamp(request.Limit <= 0 ? 20 : request.Limit, 1, 100);
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var query = db.SimulationRuns.AsNoTracking().Where(r => r.Isin == cleanIsin);
|
|
if (!string.IsNullOrWhiteSpace(request.StrategyKey))
|
|
{
|
|
query = query.Where(r => r.StrategyKey == request.StrategyKey);
|
|
}
|
|
|
|
var runs = await query
|
|
.OrderByDescending(r => r.CreatedAtUtc)
|
|
.Take(limit)
|
|
.ToListAsync(cancellationToken);
|
|
|
|
return runs.Select(r => new BacktestHistoryEntryDto(
|
|
RunId: r.Id,
|
|
Isin: r.Isin,
|
|
Symbol: r.Symbol,
|
|
StrategyKey: r.StrategyKey,
|
|
Timeframe: r.Timeframe,
|
|
StartDateUtc: r.StartDateUtc,
|
|
EndDateUtc: r.EndDateUtc,
|
|
TotalTrades: r.TotalTrades,
|
|
WinRatePercent: r.WinRatePercent,
|
|
ProfitFactor: r.ProfitFactor,
|
|
MaxDrawdownPercent: r.MaxDrawdownPercent,
|
|
TotalReturnPercent: r.TotalReturnPercent,
|
|
SharpeRatio: r.SharpeRatio,
|
|
CreatedAtUtc: r.CreatedAtUtc
|
|
)).ToList();
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<BacktestReportDto?> GetBacktestRunDetailAsync(Guid runId, CancellationToken cancellationToken = default)
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var run = await db.SimulationRuns.AsNoTracking().FirstOrDefaultAsync(r => r.Id == runId, cancellationToken);
|
|
return run?.ReportJson;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<StrategyParameterProfileDto?> GetStrategyParametersAsync(string isin, string strategyKey, CancellationToken cancellationToken = default)
|
|
{
|
|
var cleanIsin = isin.Trim().ToUpperInvariant();
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var entity = await db.StrategyParameters.AsNoTracking()
|
|
.FirstOrDefaultAsync(p => p.Isin == cleanIsin && p.StrategyKey == strategyKey, cancellationToken);
|
|
|
|
if (entity == null) return null;
|
|
|
|
return new StrategyParameterProfileDto(entity.Isin, entity.StrategyKey, entity.Parameters, entity.UpdatedAtUtc);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<StrategyParameterProfileDto> SaveStrategyParametersAsync(string isin, string strategyKey, Dictionary<string, decimal> parameters, CancellationToken cancellationToken = default)
|
|
{
|
|
var cleanIsin = isin.Trim().ToUpperInvariant();
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<SimulationDbContext>();
|
|
|
|
var entity = await db.StrategyParameters
|
|
.FirstOrDefaultAsync(p => p.Isin == cleanIsin && p.StrategyKey == strategyKey, cancellationToken);
|
|
|
|
var now = DateTime.UtcNow;
|
|
if (entity == null)
|
|
{
|
|
entity = new SimulationStrategyParameterEntity
|
|
{
|
|
Isin = cleanIsin,
|
|
StrategyKey = strategyKey,
|
|
Parameters = parameters,
|
|
UpdatedAtUtc = now
|
|
};
|
|
db.StrategyParameters.Add(entity);
|
|
}
|
|
else
|
|
{
|
|
entity.Parameters = parameters;
|
|
entity.UpdatedAtUtc = now;
|
|
}
|
|
|
|
await db.SaveChangesAsync(cancellationToken);
|
|
|
|
await _logger.LogInfoAsync(SimulationSettingKeys.SimulationChannel,
|
|
"[SimulationEngine] Saved parameter profile for {Isin} ({Strategy}): {Count} override(s).",
|
|
cleanIsin, strategyKey, parameters.Count);
|
|
|
|
return new StrategyParameterProfileDto(entity.Isin, entity.StrategyKey, entity.Parameters, entity.UpdatedAtUtc);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Picks the Yahoo Finance chart-API <c>range</c> query parameter to request for a given candle
|
|
/// <paramref name="timeframe"/>, honoring Yahoo's real, publicly documented per-interval history limits
|
|
/// (the same limits every Yahoo-chart-API client, e.g. Python's <c>yfinance</c>, has to respect) instead of
|
|
/// the previous hardcoded <c>"2y"</c> for every interval - which silently under-delivered for anything
|
|
/// finer than 1h (Yahoo does not retain 2 years of 5m/15m/30m bars) and needlessly under-fetched for 1d/1wk
|
|
/// (which Yahoo happily serves far beyond 2 years). This directly determines how much real history a
|
|
/// backtest on a given timeframe can actually cover.
|
|
/// </summary>
|
|
private static string ResolveYahooRange(string timeframe) => timeframe.Trim().ToLowerInvariant() switch
|
|
{
|
|
"1m" => "7d",
|
|
"5m" or "15m" or "30m" => "60d",
|
|
"1h" or "60m" => "730d",
|
|
"1wk" => "10y",
|
|
_ => "5y" // 1d and anything else Yahoo retains for many years.
|
|
};
|
|
}
|