diff --git a/FinlyticAssets/Database/AssetsDbContext.cs b/FinlyticAssets/Database/AssetsDbContext.cs index 0603190..bb3e0d5 100644 --- a/FinlyticAssets/Database/AssetsDbContext.cs +++ b/FinlyticAssets/Database/AssetsDbContext.cs @@ -17,9 +17,9 @@ public class AssetsDbContext : DbContext, ISettingsDbContext } public DbSet DynamicSettings => Set(); - public DbSet Settings { get; set; } - public DbSet TradeRepublicAssets { get; set; } - public DbSet TradeRepublicTags { get; set; } + public DbSet TradeRepublicAssets => Set(); + public DbSet Derivatives => Set(); + public DbSet TradeRepublicTags => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -30,20 +30,29 @@ public class AssetsDbContext : DbContext, ISettingsDbContext entity.HasKey(e => e.Id); entity.HasIndex(e => e.Key).IsUnique(); }); + modelBuilder.Entity(entity => { - entity.HasKey(e => new {e.Isin, e.InstrumentCategory}); + entity.HasKey(e => new { e.Isin, e.InstrumentCategory }); entity.HasIndex(e => e.LastUpdatedAt); entity.HasDiscriminator("AssetType") .HasValue("Stock") - .HasValue("Crypto") .HasValue("Etf") - .HasValue("Synthetic") - .HasValue("Bond") - .HasValue("Derivative"); + .HasValue("Synthetic"); }); + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Isin); + entity.HasIndex(e => e.UnderlyingIsin); + entity.HasIndex(e => e.OptionType); + entity.HasIndex(e => e.Leverage); + entity.HasIndex(e => e.LastUpdatedAt); + entity.HasIndex(e => new { e.UnderlyingIsin, e.OptionType, e.Leverage, e.Barrier }); + }); + + var stringListConverter = new Microsoft.EntityFrameworkCore.Storage.ValueConversion.ValueConverter, string>( v => System.Text.Json.JsonSerializer.Serialize(v, (System.Text.Json.JsonSerializerOptions?)null), @@ -75,15 +84,6 @@ public class AssetsDbContext : DbContext, ISettingsDbContext modelBuilder.Entity(entity => { entity.HasKey(e => e.Id); }); - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.Id); - - entity.Property(e => e.CurrentScanningType) - .HasConversion() - .HasMaxLength(50); - }); - modelBuilder.Entity() .HasMany(a => a.Tags) .WithMany(t => t.Assets); diff --git a/FinlyticAssets/Dockerfile b/FinlyticAssets/Dockerfile index dc732a7..81f313c 100644 --- a/FinlyticAssets/Dockerfile +++ b/FinlyticAssets/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build ARG BUILD_CONFIGURATION=Release WORKDIR /src +COPY ["FinlyticCore/FinlyticCore.csproj", "FinlyticCore/"] COPY ["FinlyticAssets/FinlyticAssets.csproj", "FinlyticAssets/"] RUN dotnet restore "FinlyticAssets/FinlyticAssets.csproj" COPY . . diff --git a/FinlyticAssets/Entities/AssetEntity.cs b/FinlyticAssets/Entities/AssetEntity.cs index ce07f96..71e26ff 100644 --- a/FinlyticAssets/Entities/AssetEntity.cs +++ b/FinlyticAssets/Entities/AssetEntity.cs @@ -7,8 +7,6 @@ public abstract class AssetEntity public string Type { get; set; } = string.Empty; public string InstrumentCategory { get; set; } = string.Empty; public bool HasCfd { get; set; } - public string? ImageId { get; set; } - public DateTime LastUpdatedAt { get; set; } = DateTime.UtcNow; public List Tags { get; set; } = new(); diff --git a/FinlyticAssets/Entities/DerivativeEntity.cs b/FinlyticAssets/Entities/DerivativeEntity.cs index 035883a..2cd9128 100644 --- a/FinlyticAssets/Entities/DerivativeEntity.cs +++ b/FinlyticAssets/Entities/DerivativeEntity.cs @@ -11,11 +11,17 @@ public enum OptionType Short } -public class DerivativeEntity : AssetEntity +public class DerivativeEntity { + [Key] + [StringLength(12, MinimumLength = 12)] + public string Isin { get; set; } = string.Empty; + // --- Verknüpfung zum Basiswert (Underlying) --- [StringLength(12, MinimumLength = 12)] - public string? UnderlyingIsin { get; set; } + public string UnderlyingIsin { get; set; } = string.Empty; + + public string Name { get; set; } = string.Empty; // --- Derivat-Spezifikationen --- public OptionType OptionType { get; set; } // Long / Short @@ -54,7 +60,8 @@ public class DerivativeEntity : AssetEntity [MaxLength(150)] public string IssuerDisplayName { get; set; } = string.Empty; - public string? IssuerImageId { get; set; } + public DateTime LastUpdatedAt { get; set; } = DateTime.UtcNow; + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; // PostgreSQL Mapped Array für TR-Kategorien public List DerivativeProductCategories { get; set; } = new(); diff --git a/FinlyticAssets/Entities/Settings.cs b/FinlyticAssets/Entities/Settings.cs deleted file mode 100644 index cefd688..0000000 --- a/FinlyticAssets/Entities/Settings.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.ComponentModel.DataAnnotations; -using FinlyticAssets.Models; -using FinlyticCore.Models.Assets; - -namespace FinlyticAssets.Entities; - -/// -/// Represents the global synchronization and timing configurations for the Trade Republic asset scanner. -/// -public class Settings -{ - /// - /// Gets or sets the unique identifier for the settings record. - /// - [Key] - public Guid Id { get; set; } - - /// - /// Gets or sets a value indicating whether the very first full scan of all assets has been completed. - /// Used to switch from fast initial discovery delays to stealthy incremental update delays. - /// - public bool FinishedInitialScan { get; set; } - - - - /// - /// Gets or sets the maximum number of assets requested per single API pagination call. - /// Values around 100 look like standard dynamic-scrolling payloads from a real client device. - /// - public int TradeRepublicMaxRequestPageSize { get; set; } = 100; - - /// - /// Gets or sets the idle delay in seconds between switching from one full asset category to another (e.g., from Stocks to Crypto) - /// during standard maintenance mode. (Default 12000s = ~3.3 hours). - /// - public int AssetUpdateTypeDelay { get; set; } = 12000; - - /// - /// Gets or sets the idle delay in seconds between switching asset categories during the initial setup scan. - /// Set to 0 to move immediately to the next type after finishing the current one. (Default 0s). - /// - public int InitAssetUpdateTypeDelay { get; set; } = 0; - - /// - /// Gets or sets the baseline delay in seconds between sequential page requests of the same asset type during the initial setup scan. - /// (Default 120s = 2 minutes). - /// - public int InitBatchAssetUpdateDelay { get; set; } = 120; - - /// - /// Gets or sets the standard baseline delay in seconds between sequential page requests of the same asset type during recurring incremental updates. - /// Spreads pagination widely over time to blend into regular human traffic profiles. (Default 600s = 10 minutes). - /// - public int BatchAssetUpdateDelay { get; set; } = 600; - - /// - /// Gets or sets the asset type that is currently being processed by the full scan. - /// Acts as a live pointer for recovery after a service interruption. - /// - public AssetType CurrentScanningType { get; set; } = AssetType.Stock; - - /// - /// Gets or sets the page number of the that is currently being fetched or was just processed. - /// Trade Republic uses 1-based pagination. A value of 0 means the scan is currently idle or between types. - /// - public int CurrentScanningPage { get; set; } = 0; -} diff --git a/FinlyticAssets/Migrations/20260801073314_Init.Designer.cs b/FinlyticAssets/Migrations/20260801073314_Init.Designer.cs deleted file mode 100644 index 4f1cca7..0000000 --- a/FinlyticAssets/Migrations/20260801073314_Init.Designer.cs +++ /dev/null @@ -1,282 +0,0 @@ -// -using System; -using FinlyticAssets.Database; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - [DbContext(typeof(AssetsDbContext))] - [Migration("20260801073314_Init")] - partial class Init - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.Property("TagsId") - .HasColumnType("text"); - - b.Property("AssetsIsin") - .HasColumnType("text"); - - b.Property("AssetsInstrumentCategory") - .HasColumnType("text"); - - b.HasKey("TagsId", "AssetsIsin", "AssetsInstrumentCategory"); - - b.HasIndex("AssetsIsin", "AssetsInstrumentCategory"); - - b.ToTable("AssetEntityTagEntity"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("CurrentScanningPage") - .HasColumnType("integer"); - - b.Property("CurrentScanningType") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)"); - - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); - - b.Property("InitAssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => - { - b.Property("Isin") - .HasColumnType("text"); - - b.Property("InstrumentCategory") - .HasColumnType("text"); - - b.Property("AssetType") - .IsRequired() - .HasMaxLength(13) - .HasColumnType("character varying(13)"); - - b.Property("HasCfd") - .HasColumnType("boolean"); - - b.Property("ImageId") - .HasColumnType("text"); - - b.Property("LastUpdatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Isin", "InstrumentCategory"); - - b.HasIndex("LastUpdatedAt"); - - b.ToTable("TradeRepublicAssets"); - - b.HasDiscriminator("AssetType").HasValue("AssetEntity"); - - b.UseTphMappingStrategy(); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("TradeRepublicTags"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("UnderlyingIsin") - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Derivative"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("EtfDescription") - .IsRequired() - .HasColumnType("text"); - - b.Property("MappedEtfIndexName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - - b.HasDiscriminator().HasValue("Etf"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("StockEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Stock"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("SyntheticEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Synthetic"); - }); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) - .WithMany() - .HasForeignKey("TagsId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) - .WithMany() - .HasForeignKey("AssetsIsin", "AssetsInstrumentCategory") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/FinlyticAssets/Migrations/20260813202348_UpdateDerivativeAssets.cs b/FinlyticAssets/Migrations/20260813202348_UpdateDerivativeAssets.cs deleted file mode 100644 index 6f23f85..0000000 --- a/FinlyticAssets/Migrations/20260813202348_UpdateDerivativeAssets.cs +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - /// - public partial class UpdateDerivativeAssets : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "UnderlyingIsin", - table: "TradeRepublicAssets", - type: "character varying(12)", - maxLength: 12, - nullable: true, - oldClrType: typeof(string), - oldType: "text", - oldNullable: true); - - migrationBuilder.AddColumn( - name: "Barrier", - table: "TradeRepublicAssets", - type: "numeric(18,6)", - nullable: true); - - migrationBuilder.AddColumn( - name: "Currency", - table: "TradeRepublicAssets", - type: "character varying(10)", - maxLength: 10, - nullable: true); - - migrationBuilder.AddColumn( - name: "Delta", - table: "TradeRepublicAssets", - type: "numeric", - nullable: true); - - migrationBuilder.AddColumn( - name: "Expiry", - table: "TradeRepublicAssets", - type: "timestamp with time zone", - nullable: true); - - migrationBuilder.AddColumn( - name: "Factor", - table: "TradeRepublicAssets", - type: "numeric", - nullable: true); - - migrationBuilder.AddColumn( - name: "Issuer", - table: "TradeRepublicAssets", - type: "character varying(150)", - maxLength: 150, - nullable: true); - - migrationBuilder.AddColumn( - name: "IssuerDisplayName", - table: "TradeRepublicAssets", - type: "character varying(150)", - maxLength: 150, - nullable: true); - - migrationBuilder.AddColumn( - name: "IssuerImageId", - table: "TradeRepublicAssets", - type: "text", - nullable: true); - - migrationBuilder.AddColumn( - name: "Leverage", - table: "TradeRepublicAssets", - type: "numeric(10,4)", - nullable: true); - - migrationBuilder.AddColumn( - name: "NextGenProductCategoryName", - table: "TradeRepublicAssets", - type: "character varying(100)", - maxLength: 100, - nullable: true); - - migrationBuilder.AddColumn( - name: "OptionType", - table: "TradeRepublicAssets", - type: "integer", - nullable: true); - - migrationBuilder.AddColumn( - name: "ProductCategoryName", - table: "TradeRepublicAssets", - type: "character varying(100)", - maxLength: 100, - nullable: true); - - migrationBuilder.AddColumn( - name: "Size", - table: "TradeRepublicAssets", - type: "numeric", - nullable: true); - - migrationBuilder.AddColumn( - name: "Strike", - table: "TradeRepublicAssets", - type: "numeric(18,6)", - nullable: true); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropColumn( - name: "Barrier", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Currency", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Delta", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Expiry", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Factor", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Issuer", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "IssuerDisplayName", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "IssuerImageId", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Leverage", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "NextGenProductCategoryName", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "OptionType", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "ProductCategoryName", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Size", - table: "TradeRepublicAssets"); - - migrationBuilder.DropColumn( - name: "Strike", - table: "TradeRepublicAssets"); - - migrationBuilder.AlterColumn( - name: "UnderlyingIsin", - table: "TradeRepublicAssets", - type: "text", - nullable: true, - oldClrType: typeof(string), - oldType: "character varying(12)", - oldMaxLength: 12, - oldNullable: true); - } - } -} diff --git a/FinlyticAssets/Migrations/20260813205059_AddDynamicSettings.Designer.cs b/FinlyticAssets/Migrations/20260813205059_AddDynamicSettings.Designer.cs deleted file mode 100644 index 96e1052..0000000 --- a/FinlyticAssets/Migrations/20260813205059_AddDynamicSettings.Designer.cs +++ /dev/null @@ -1,365 +0,0 @@ -// -using System; -using FinlyticAssets.Database; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - [DbContext(typeof(AssetsDbContext))] - [Migration("20260813205059_AddDynamicSettings")] - partial class AddDynamicSettings - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.Property("TagsId") - .HasColumnType("text"); - - b.Property("AssetsIsin") - .HasColumnType("text"); - - b.Property("AssetsInstrumentCategory") - .HasColumnType("text"); - - b.HasKey("TagsId", "AssetsIsin", "AssetsInstrumentCategory"); - - b.HasIndex("AssetsIsin", "AssetsInstrumentCategory"); - - b.ToTable("AssetEntityTagEntity"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.AssetEntity", b => - { - b.Property("Isin") - .HasColumnType("text"); - - b.Property("InstrumentCategory") - .HasColumnType("text"); - - b.Property("AssetType") - .IsRequired() - .HasMaxLength(13) - .HasColumnType("character varying(13)"); - - b.Property("HasCfd") - .HasColumnType("boolean"); - - b.Property("ImageId") - .HasColumnType("text"); - - b.Property("LastUpdatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Isin", "InstrumentCategory"); - - b.HasIndex("LastUpdatedAt"); - - b.ToTable("TradeRepublicAssets"); - - b.HasDiscriminator("AssetType").HasValue("AssetEntity"); - - b.UseTphMappingStrategy(); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("CurrentScanningPage") - .HasColumnType("integer"); - - b.Property("CurrentScanningType") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)"); - - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); - - b.Property("InitAssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.TagEntity", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("TradeRepublicTags"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("Key") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("LastUpdatedUtc") - .HasColumnType("timestamp with time zone"); - - b.Property("ServiceIdentifier") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("ValueJson") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.HasIndex("Key"); - - b.ToTable("DynamicSettings"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.BondEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.CryptoEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("Barrier") - .HasColumnType("numeric(18,6)"); - - b.Property("Currency") - .IsRequired() - .HasMaxLength(10) - .HasColumnType("character varying(10)"); - - b.Property("Delta") - .HasColumnType("numeric"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("Expiry") - .HasColumnType("timestamp with time zone"); - - b.Property("Factor") - .HasColumnType("numeric"); - - b.Property("Issuer") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("IssuerDisplayName") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("IssuerImageId") - .HasColumnType("text"); - - b.Property("Leverage") - .HasColumnType("numeric(10,4)"); - - b.Property("NextGenProductCategoryName") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("OptionType") - .HasColumnType("integer"); - - b.Property("ProductCategoryName") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Size") - .HasColumnType("numeric"); - - b.Property("Strike") - .HasColumnType("numeric(18,6)"); - - b.Property("UnderlyingIsin") - .HasMaxLength(12) - .HasColumnType("character varying(12)"); - - b.HasDiscriminator().HasValue("Derivative"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.EtfEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("EtfDescription") - .IsRequired() - .HasColumnType("text"); - - b.Property("MappedEtfIndexName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - - b.HasDiscriminator().HasValue("Etf"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.StockEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("StockEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Stock"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.SyntheticEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("SyntheticEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Synthetic"); - }); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.HasOne("FinlyticAssets.Entities.TagEntity", null) - .WithMany() - .HasForeignKey("TagsId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("FinlyticAssets.Entities.AssetEntity", null) - .WithMany() - .HasForeignKey("AssetsIsin", "AssetsInstrumentCategory") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/FinlyticAssets/Migrations/20260813205059_AddDynamicSettings.cs b/FinlyticAssets/Migrations/20260813205059_AddDynamicSettings.cs deleted file mode 100644 index c66bf6d..0000000 --- a/FinlyticAssets/Migrations/20260813205059_AddDynamicSettings.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - /// - public partial class AddDynamicSettings : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateTable( - name: "DynamicSettings", - columns: table => new - { - Id = table.Column(type: "uuid", nullable: false), - Key = table.Column(type: "character varying(150)", maxLength: 150, nullable: false), - ValueJson = table.Column(type: "text", nullable: false), - ServiceIdentifier = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), - LastUpdatedUtc = table.Column(type: "timestamp with time zone", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("PK_DynamicSettings", x => x.Id); - }); - - migrationBuilder.CreateIndex( - name: "IX_DynamicSettings_Key", - table: "DynamicSettings", - column: "Key"); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable( - name: "DynamicSettings"); - } - } -} diff --git a/FinlyticAssets/Migrations/20260815184053_UpdateDynamicSettingsUniqueIndex.cs b/FinlyticAssets/Migrations/20260815184053_UpdateDynamicSettingsUniqueIndex.cs deleted file mode 100644 index e21f173..0000000 --- a/FinlyticAssets/Migrations/20260815184053_UpdateDynamicSettingsUniqueIndex.cs +++ /dev/null @@ -1,42 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - /// - public partial class UpdateDynamicSettingsUniqueIndex : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.Sql(@" - DELETE FROM ""DynamicSettings"" a USING ""DynamicSettings"" b - WHERE a.""Key"" = b.""Key"" AND a.""Id"" < b.""Id""; - "); - - migrationBuilder.DropIndex( - name: "IX_DynamicSettings_Key", - table: "DynamicSettings"); - - migrationBuilder.CreateIndex( - name: "IX_DynamicSettings_Key", - table: "DynamicSettings", - column: "Key", - unique: true); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropIndex( - name: "IX_DynamicSettings_Key", - table: "DynamicSettings"); - - migrationBuilder.CreateIndex( - name: "IX_DynamicSettings_Key", - table: "DynamicSettings", - column: "Key"); - } - } -} diff --git a/FinlyticAssets/Migrations/20260815184053_UpdateDynamicSettingsUniqueIndex.Designer.cs b/FinlyticAssets/Migrations/20260818185753_Init.Designer.cs similarity index 73% rename from FinlyticAssets/Migrations/20260815184053_UpdateDynamicSettingsUniqueIndex.Designer.cs rename to FinlyticAssets/Migrations/20260818185753_Init.Designer.cs index e7c5aa2..e383e34 100644 --- a/FinlyticAssets/Migrations/20260815184053_UpdateDynamicSettingsUniqueIndex.Designer.cs +++ b/FinlyticAssets/Migrations/20260818185753_Init.Designer.cs @@ -12,8 +12,8 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace FinlyticAssets.Migrations { [DbContext(typeof(AssetsDbContext))] - [Migration("20260815184053_UpdateDynamicSettingsUniqueIndex")] - partial class UpdateDynamicSettingsUniqueIndex + [Migration("20260818185753_Init")] + partial class Init { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -59,9 +59,6 @@ namespace FinlyticAssets.Migrations b.Property("HasCfd") .HasColumnType("boolean"); - b.Property("ImageId") - .HasColumnType("text"); - b.Property("LastUpdatedAt") .HasColumnType("timestamp with time zone"); @@ -84,41 +81,91 @@ namespace FinlyticAssets.Migrations b.UseTphMappingStrategy(); }); - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => + modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b => { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); + b.Property("Isin") + .HasMaxLength(12) + .HasColumnType("character varying(12)"); - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); + b.Property("Barrier") + .HasColumnType("numeric(18,6)"); - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); - b.Property("CurrentScanningPage") - .HasColumnType("integer"); - - b.Property("CurrentScanningType") + b.Property("Currency") .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)"); + .HasMaxLength(10) + .HasColumnType("character varying(10)"); - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); + b.Property("Delta") + .HasColumnType("numeric"); - b.Property("InitAssetUpdateTypeDelay") + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("Expiry") + .HasColumnType("timestamp with time zone"); + + b.Property("Factor") + .HasColumnType("numeric"); + + b.Property("Issuer") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("IssuerDisplayName") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Leverage") + .HasColumnType("numeric(10,4)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextGenProductCategoryName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("OptionType") .HasColumnType("integer"); - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); + b.Property("ProductCategoryName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); + b.Property("Size") + .HasColumnType("numeric"); - b.HasKey("Id"); + b.Property("Strike") + .HasColumnType("numeric(18,6)"); - b.ToTable("Settings"); + b.Property("UnderlyingIsin") + .IsRequired() + .HasMaxLength(12) + .HasColumnType("character varying(12)"); + + b.HasKey("Isin"); + + b.HasIndex("LastUpdatedAt"); + + b.HasIndex("Leverage"); + + b.HasIndex("OptionType"); + + b.HasIndex("UnderlyingIsin"); + + b.ToTable("Derivatives"); }); modelBuilder.Entity("FinlyticAssets.Entities.TagEntity", b => @@ -170,109 +217,6 @@ namespace FinlyticAssets.Migrations b.ToTable("DynamicSettings"); }); - modelBuilder.Entity("FinlyticAssets.Entities.BondEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.CryptoEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("Barrier") - .HasColumnType("numeric(18,6)"); - - b.Property("Currency") - .IsRequired() - .HasMaxLength(10) - .HasColumnType("character varying(10)"); - - b.Property("Delta") - .HasColumnType("numeric"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("Expiry") - .HasColumnType("timestamp with time zone"); - - b.Property("Factor") - .HasColumnType("numeric"); - - b.Property("Issuer") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("IssuerDisplayName") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("IssuerImageId") - .HasColumnType("text"); - - b.Property("Leverage") - .HasColumnType("numeric(10,4)"); - - b.Property("NextGenProductCategoryName") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("OptionType") - .HasColumnType("integer"); - - b.Property("ProductCategoryName") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Size") - .HasColumnType("numeric"); - - b.Property("Strike") - .HasColumnType("numeric(18,6)"); - - b.Property("UnderlyingIsin") - .HasMaxLength(12) - .HasColumnType("character varying(12)"); - - b.HasDiscriminator().HasValue("Derivative"); - }); - modelBuilder.Entity("FinlyticAssets.Entities.EtfEntity", b => { b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); @@ -297,18 +241,6 @@ namespace FinlyticAssets.Migrations .IsRequired() .HasColumnType("text"); - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - b.HasDiscriminator().HasValue("Etf"); }); diff --git a/FinlyticAssets/Migrations/20260801073314_Init.cs b/FinlyticAssets/Migrations/20260818185753_Init.cs similarity index 57% rename from FinlyticAssets/Migrations/20260801073314_Init.cs rename to FinlyticAssets/Migrations/20260818185753_Init.cs index 813eb3b..a65740b 100644 --- a/FinlyticAssets/Migrations/20260801073314_Init.cs +++ b/FinlyticAssets/Migrations/20260818185753_Init.cs @@ -1,4 +1,4 @@ -using System; +using System; using Microsoft.EntityFrameworkCore.Migrations; #nullable disable @@ -12,22 +12,47 @@ namespace FinlyticAssets.Migrations protected override void Up(MigrationBuilder migrationBuilder) { migrationBuilder.CreateTable( - name: "Settings", + name: "Derivatives", columns: table => new { - Id = table.Column(type: "uuid", nullable: false), - FinishedInitialScan = table.Column(type: "boolean", nullable: false), - TradeRepublicMaxRequestPageSize = table.Column(type: "integer", nullable: false), - AssetUpdateTypeDelay = table.Column(type: "integer", nullable: false), - InitAssetUpdateTypeDelay = table.Column(type: "integer", nullable: false), - InitBatchAssetUpdateDelay = table.Column(type: "integer", nullable: false), - BatchAssetUpdateDelay = table.Column(type: "integer", nullable: false), - CurrentScanningType = table.Column(type: "character varying(50)", maxLength: 50, nullable: false), - CurrentScanningPage = table.Column(type: "integer", nullable: false) + Isin = table.Column(type: "character varying(12)", maxLength: 12, nullable: false), + UnderlyingIsin = table.Column(type: "character varying(12)", maxLength: 12, nullable: false), + Name = table.Column(type: "text", nullable: false), + OptionType = table.Column(type: "integer", nullable: false), + ProductCategoryName = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + NextGenProductCategoryName = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + Strike = table.Column(type: "numeric(18,6)", nullable: false), + Barrier = table.Column(type: "numeric(18,6)", nullable: false), + Leverage = table.Column(type: "numeric(10,4)", nullable: false), + Size = table.Column(type: "numeric", nullable: true), + Factor = table.Column(type: "numeric", nullable: true), + Delta = table.Column(type: "numeric", nullable: true), + Currency = table.Column(type: "character varying(10)", maxLength: 10, nullable: false), + Expiry = table.Column(type: "timestamp with time zone", nullable: true), + Issuer = table.Column(type: "character varying(150)", maxLength: 150, nullable: false), + IssuerDisplayName = table.Column(type: "character varying(150)", maxLength: 150, nullable: false), + LastUpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + DerivativeProductCategories = table.Column(type: "text", nullable: false) }, constraints: table => { - table.PrimaryKey("PK_Settings", x => x.Id); + table.PrimaryKey("PK_Derivatives", x => x.Isin); + }); + + migrationBuilder.CreateTable( + name: "DynamicSettings", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Key = table.Column(type: "character varying(150)", maxLength: 150, nullable: false), + ValueJson = table.Column(type: "text", nullable: false), + ServiceIdentifier = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + LastUpdatedUtc = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_DynamicSettings", x => x.Id); }); migrationBuilder.CreateTable( @@ -39,20 +64,13 @@ namespace FinlyticAssets.Migrations Name = table.Column(type: "text", nullable: false), Type = table.Column(type: "text", nullable: false), HasCfd = table.Column(type: "boolean", nullable: false), - ImageId = table.Column(type: "text", nullable: true), LastUpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), AssetType = table.Column(type: "character varying(13)", maxLength: 13, nullable: false), - BondIssuerName = table.Column(type: "text", nullable: true), - SearchSubtitle = table.Column(type: "text", nullable: true), - Subtitle = table.Column(type: "text", nullable: true), - CryptoEntity_SearchSubtitle = table.Column(type: "text", nullable: true), DerivativeProductCategories = table.Column(type: "text", nullable: true), - UnderlyingIsin = table.Column(type: "text", nullable: true), - EtfEntity_DerivativeProductCategories = table.Column(type: "text", nullable: true), EtfDescription = table.Column(type: "text", nullable: true), MappedEtfIndexName = table.Column(type: "text", nullable: true), - EtfEntity_Subtitle = table.Column(type: "text", nullable: true), - EtfEntity_SearchSubtitle = table.Column(type: "text", nullable: true), + Subtitle = table.Column(type: "text", nullable: true), + SearchSubtitle = table.Column(type: "text", nullable: true), StockEntity_DerivativeProductCategories = table.Column(type: "text", nullable: true), SyntheticEntity_DerivativeProductCategories = table.Column(type: "text", nullable: true) }, @@ -104,6 +122,32 @@ namespace FinlyticAssets.Migrations table: "AssetEntityTagEntity", columns: new[] { "AssetsIsin", "AssetsInstrumentCategory" }); + migrationBuilder.CreateIndex( + name: "IX_Derivatives_LastUpdatedAt", + table: "Derivatives", + column: "LastUpdatedAt"); + + migrationBuilder.CreateIndex( + name: "IX_Derivatives_Leverage", + table: "Derivatives", + column: "Leverage"); + + migrationBuilder.CreateIndex( + name: "IX_Derivatives_OptionType", + table: "Derivatives", + column: "OptionType"); + + migrationBuilder.CreateIndex( + name: "IX_Derivatives_UnderlyingIsin", + table: "Derivatives", + column: "UnderlyingIsin"); + + migrationBuilder.CreateIndex( + name: "IX_DynamicSettings_Key", + table: "DynamicSettings", + column: "Key", + unique: true); + migrationBuilder.CreateIndex( name: "IX_TradeRepublicAssets_LastUpdatedAt", table: "TradeRepublicAssets", @@ -117,7 +161,10 @@ namespace FinlyticAssets.Migrations name: "AssetEntityTagEntity"); migrationBuilder.DropTable( - name: "Settings"); + name: "Derivatives"); + + migrationBuilder.DropTable( + name: "DynamicSettings"); migrationBuilder.DropTable( name: "TradeRepublicAssets"); diff --git a/FinlyticAssets/Migrations/20260813202348_UpdateDerivativeAssets.Designer.cs b/FinlyticAssets/Migrations/20260821153552_SyncAssetModelDrift.Designer.cs similarity index 74% rename from FinlyticAssets/Migrations/20260813202348_UpdateDerivativeAssets.Designer.cs rename to FinlyticAssets/Migrations/20260821153552_SyncAssetModelDrift.Designer.cs index 7946af8..f7313be 100644 --- a/FinlyticAssets/Migrations/20260813202348_UpdateDerivativeAssets.Designer.cs +++ b/FinlyticAssets/Migrations/20260821153552_SyncAssetModelDrift.Designer.cs @@ -12,8 +12,8 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace FinlyticAssets.Migrations { [DbContext(typeof(AssetsDbContext))] - [Migration("20260813202348_UpdateDerivativeAssets")] - partial class UpdateDerivativeAssets + [Migration("20260821153552_SyncAssetModelDrift")] + partial class SyncAssetModelDrift { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -59,9 +59,6 @@ namespace FinlyticAssets.Migrations b.Property("HasCfd") .HasColumnType("boolean"); - b.Property("ImageId") - .HasColumnType("text"); - b.Property("LastUpdatedAt") .HasColumnType("timestamp with time zone"); @@ -84,104 +81,18 @@ namespace FinlyticAssets.Migrations b.UseTphMappingStrategy(); }); - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("CurrentScanningPage") - .HasColumnType("integer"); - - b.Property("CurrentScanningType") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)"); - - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); - - b.Property("InitAssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.TagEntity", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("TradeRepublicTags"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.BondEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.CryptoEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b => { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); + b.Property("Isin") + .HasMaxLength(12) + .HasColumnType("character varying(12)"); b.Property("Barrier") .HasColumnType("numeric(18,6)"); + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + b.Property("Currency") .IsRequired() .HasMaxLength(10) @@ -210,12 +121,16 @@ namespace FinlyticAssets.Migrations .HasMaxLength(150) .HasColumnType("character varying(150)"); - b.Property("IssuerImageId") - .HasColumnType("text"); + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); b.Property("Leverage") .HasColumnType("numeric(10,4)"); + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + b.Property("NextGenProductCategoryName") .IsRequired() .HasMaxLength(100) @@ -236,10 +151,72 @@ namespace FinlyticAssets.Migrations .HasColumnType("numeric(18,6)"); b.Property("UnderlyingIsin") + .IsRequired() .HasMaxLength(12) .HasColumnType("character varying(12)"); - b.HasDiscriminator().HasValue("Derivative"); + b.HasKey("Isin"); + + b.HasIndex("LastUpdatedAt"); + + b.HasIndex("Leverage"); + + b.HasIndex("OptionType"); + + b.HasIndex("UnderlyingIsin"); + + b.HasIndex("UnderlyingIsin", "OptionType", "Leverage", "Barrier"); + + b.ToTable("Derivatives"); + }); + + modelBuilder.Entity("FinlyticAssets.Entities.TagEntity", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("TradeRepublicTags"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("ServiceIdentifier") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ValueJson") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Key") + .IsUnique(); + + b.ToTable("DynamicSettings"); }); modelBuilder.Entity("FinlyticAssets.Entities.EtfEntity", b => @@ -266,18 +243,6 @@ namespace FinlyticAssets.Migrations .IsRequired() .HasColumnType("text"); - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - b.HasDiscriminator().HasValue("Etf"); }); diff --git a/FinlyticAssets/Migrations/20260821153552_SyncAssetModelDrift.cs b/FinlyticAssets/Migrations/20260821153552_SyncAssetModelDrift.cs new file mode 100644 index 0000000..f201f17 --- /dev/null +++ b/FinlyticAssets/Migrations/20260821153552_SyncAssetModelDrift.cs @@ -0,0 +1,27 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + /// + public partial class SyncAssetModelDrift : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_Derivatives_UnderlyingIsin_OptionType_Leverage_Barrier", + table: "Derivatives", + columns: new[] { "UnderlyingIsin", "OptionType", "Leverage", "Barrier" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Derivatives_UnderlyingIsin_OptionType_Leverage_Barrier", + table: "Derivatives"); + } + } +} diff --git a/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs b/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs index 26ad3b5..d7bed4c 100644 --- a/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs +++ b/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs @@ -56,9 +56,6 @@ namespace FinlyticAssets.Migrations b.Property("HasCfd") .HasColumnType("boolean"); - b.Property("ImageId") - .HasColumnType("text"); - b.Property("LastUpdatedAt") .HasColumnType("timestamp with time zone"); @@ -81,41 +78,93 @@ namespace FinlyticAssets.Migrations b.UseTphMappingStrategy(); }); - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => + modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b => { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); + b.Property("Isin") + .HasMaxLength(12) + .HasColumnType("character varying(12)"); - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); + b.Property("Barrier") + .HasColumnType("numeric(18,6)"); - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); - b.Property("CurrentScanningPage") - .HasColumnType("integer"); - - b.Property("CurrentScanningType") + b.Property("Currency") .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)"); + .HasMaxLength(10) + .HasColumnType("character varying(10)"); - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); + b.Property("Delta") + .HasColumnType("numeric"); - b.Property("InitAssetUpdateTypeDelay") + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("Expiry") + .HasColumnType("timestamp with time zone"); + + b.Property("Factor") + .HasColumnType("numeric"); + + b.Property("Issuer") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("IssuerDisplayName") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Leverage") + .HasColumnType("numeric(10,4)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextGenProductCategoryName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("OptionType") .HasColumnType("integer"); - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); + b.Property("ProductCategoryName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); + b.Property("Size") + .HasColumnType("numeric"); - b.HasKey("Id"); + b.Property("Strike") + .HasColumnType("numeric(18,6)"); - b.ToTable("Settings"); + b.Property("UnderlyingIsin") + .IsRequired() + .HasMaxLength(12) + .HasColumnType("character varying(12)"); + + b.HasKey("Isin"); + + b.HasIndex("LastUpdatedAt"); + + b.HasIndex("Leverage"); + + b.HasIndex("OptionType"); + + b.HasIndex("UnderlyingIsin"); + + b.HasIndex("UnderlyingIsin", "OptionType", "Leverage", "Barrier"); + + b.ToTable("Derivatives"); }); modelBuilder.Entity("FinlyticAssets.Entities.TagEntity", b => @@ -167,109 +216,6 @@ namespace FinlyticAssets.Migrations b.ToTable("DynamicSettings"); }); - modelBuilder.Entity("FinlyticAssets.Entities.BondEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.CryptoEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.DerivativeEntity", b => - { - b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); - - b.Property("Barrier") - .HasColumnType("numeric(18,6)"); - - b.Property("Currency") - .IsRequired() - .HasMaxLength(10) - .HasColumnType("character varying(10)"); - - b.Property("Delta") - .HasColumnType("numeric"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("Expiry") - .HasColumnType("timestamp with time zone"); - - b.Property("Factor") - .HasColumnType("numeric"); - - b.Property("Issuer") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("IssuerDisplayName") - .IsRequired() - .HasMaxLength(150) - .HasColumnType("character varying(150)"); - - b.Property("IssuerImageId") - .HasColumnType("text"); - - b.Property("Leverage") - .HasColumnType("numeric(10,4)"); - - b.Property("NextGenProductCategoryName") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("OptionType") - .HasColumnType("integer"); - - b.Property("ProductCategoryName") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Size") - .HasColumnType("numeric"); - - b.Property("Strike") - .HasColumnType("numeric(18,6)"); - - b.Property("UnderlyingIsin") - .HasMaxLength(12) - .HasColumnType("character varying(12)"); - - b.HasDiscriminator().HasValue("Derivative"); - }); - modelBuilder.Entity("FinlyticAssets.Entities.EtfEntity", b => { b.HasBaseType("FinlyticAssets.Entities.AssetEntity"); @@ -294,18 +240,6 @@ namespace FinlyticAssets.Migrations .IsRequired() .HasColumnType("text"); - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - b.HasDiscriminator().HasValue("Etf"); }); diff --git a/FinlyticAssets/Program.cs b/FinlyticAssets/Program.cs index 2b98df0..c251338 100644 --- a/FinlyticAssets/Program.cs +++ b/FinlyticAssets/Program.cs @@ -1,4 +1,3 @@ -using System; using FinlyticAssets.Database; using FinlyticAssets.Services; using FinlyticAssets.Util; @@ -24,14 +23,12 @@ builder.Services.AddSingleton(typeof(IFinlyticLogger<>), typeof(FinlyticLogger<> builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); -builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); -builder.Services.AddHostedService(); var host = builder.Build(); @@ -40,21 +37,14 @@ using (var scope = host.Services.CreateScope()) try { var context = scope.ServiceProvider.GetRequiredService(); - await context.Database.MigrateAsync(); - - var settingsService = scope.ServiceProvider.GetRequiredService(); - var settings = await settingsService.GetSettings(); - if (settings.InitAssetUpdateTypeDelay == 3600) - { - settings.InitAssetUpdateTypeDelay = 0; - await settingsService.SaveSettings(settings); - Console.WriteLine("[Startup] InitAssetUpdateTypeDelay reset from 3600 to 0."); - } + var connStr = builder.Configuration.GetConnectionString("DefaultConnection") ?? ""; + await context.MigrateWithBootstrapAsync(connStr); } catch (Exception ex) { Console.WriteLine($"Critical error during database migration for FinlyticAssets: {ex.Message}"); } + } host.Run(); diff --git a/FinlyticAssets/Project.md b/FinlyticAssets/Project.md deleted file mode 100644 index d9af5f6..0000000 --- a/FinlyticAssets/Project.md +++ /dev/null @@ -1,67 +0,0 @@ -# Finlytic Assets Service - -Finlytic Assets is a scalable C# microservice designed for asset discovery, metadata ingestion, and quick querying. It interfaces directly with the Trade Republic API via WebSockets and exposes a high-performance RPC interface over MQTT to other services in the Finlytic ecosystem. - ---- - -## Architecture Overview - -```mermaid -graph TD - TR[Trade Republic WebSocket API] <-->|WS Protocol| TRS[TradeRepublicService] - TRS <-->|Ingest| ADS[AssetsFullScanService] - ADS <-->|Save / Update| DB[PostgreSQL Database] - DB -->|Trigger Index Update| AIS[AssetsIndexService] - AIS -->|Write Cache| Index[assets/index/index.json] - - MS[Other Finlytic Services] <-->|MQTT Request-Reply| MqttClient[AssetsMqttClient] - MqttClient <-->|Query Cache| DB -``` - ---- - -## Core Features & Workflows - -### 1. Automated WebSocket scraping -- **Continuous Scan**: The background worker (`AssetsFullScanService`) iterates through all supported asset types: - - **Stocks** (`StockEntity`) - - **Crypto** (`CryptoEntity`) - - **Derivatives** (`DerivativeEntity`) - - **Bonds** (`BondEntity`) - - **Funds/ETFs** (`EtfEntity`) -- **Resilient Reconnection**: Uses a robust custom socket wrapper (`TradeRepublicClient`) that closes automatically after 5 minutes of inactivity to mimic human interaction profiles and reconnects dynamically when a request is made. - -### 2. Scanner State Persistence & Crash Recovery -- **Resilient State**: The scan loop continuously persists progress to the database (`Settings` table), recording `CurrentScanningType` and `CurrentScanningPage`. -- **Exit Recovery**: Resumes exactly from the last saved page and asset type upon restart. - -### 3. Stealth Scheduling with Randomized Jitter -- **Dynamic Delays**: Scanner wait times are read dynamically from database configuration (`Settings`). -- **Jitter Offset**: Randomized offsets between batches and asset types to prevent rate limiting. - -### 4. Active vs. Dead Asset Tracking -- **Recency Filter**: Assets updated within 14 days are active. Inactive assets are flagged. - -### 5. MQTT RPC Interface -- **Get Asset by ISIN**: Subscribes to `services/request/assets_Get/#`. -- **Omnibox Search**: Subscribes to `services/request/assets_Search/#`. -- **JIT Fallback**: Performs JIT-lookup directly against the Trade Republic API for unknown ISINs. - -### 6. Local Indexing File (`index.json`) -- **Automatic Regeneration**: Serializes active assets list (`ISIN`, `Name`) to shared volume `assets/index/index.json`. - ---- - -## Feature Status - -### Implemented Features -- [x] Trade Republic WebSocket scanner for Stocks, Crypto, Derivatives, Bonds, ETFs. -- [x] State persistence & crash recovery in PostgreSQL. -- [x] JIT Trade Republic lookup fallback for new ISINs. -- [x] Shared local index file generation (`index.json`). -- [x] Zero-Allocation MQTT RPC handlers. -- [x] Pure Worker Service architecture (No Kestrel HTTP server). - -### Planned Features -- [ ] Real-time WebSocket price tick streaming over MQTT topic `finlytic/assets/ticks/{isin}`. -- [ ] Derivative option-chain Greeks calculator integration. diff --git a/FinlyticAssets/Services/AssetScannerBackgroundService.cs b/FinlyticAssets/Services/AssetScannerBackgroundService.cs index ed57968..5f91627 100644 --- a/FinlyticAssets/Services/AssetScannerBackgroundService.cs +++ b/FinlyticAssets/Services/AssetScannerBackgroundService.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; -using FinlyticAssets.Models; using FinlyticAssets.Util; using FinlyticCore.Dtos.TradeRepublic; using FinlyticCore.Models.Assets; @@ -15,14 +15,16 @@ using Microsoft.Extensions.Hosting; namespace FinlyticAssets.Services; /// -/// A background service that runs a continuous asset synchronization loop, -/// scanning Trade Republic to retrieve, update, and index all supported asset types. +/// A background service that runs a continuous asset synchronization loop for Stocks and ETFs, +/// retrieving, updating, and indexing assets while downloading logos inline. /// public class AssetScannerBackgroundService : BackgroundService { private readonly IServiceScopeFactory _serviceScopeFactory; private readonly IFinlyticLogger _finlyticLogger; + private static readonly AssetType[] ScannedAssetTypes = [AssetType.Stock, AssetType.Fund]; + private AssetsCount? _assetsCount; private AssetsCount? _currAssetsCount; @@ -54,24 +56,35 @@ public class AssetScannerBackgroundService : BackgroundService { using var scope = _serviceScopeFactory.CreateScope(); var tradeRepublicService = scope.ServiceProvider.GetRequiredService(); - var settingsService = scope.ServiceProvider.GetRequiredService(); + var settingsService = scope.ServiceProvider.GetRequiredService(); var assetsDbService = scope.ServiceProvider.GetRequiredService(); var indexService = scope.ServiceProvider.GetRequiredService(); + + var isAutoScanEnabled = await settingsService.GetSettingAsync(SettingKeys.ScannerEnableAutoScan, stoppingToken); + if (!isAutoScanEnabled) + { + await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Auto scan is disabled in dynamic settings. Waiting 1 minute..."); + await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + continue; + } await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Requesting total asset counts from Trade Republic..."); _assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken); _currAssetsCount = new AssetsCount(); - var initSettings = await settingsService.GetSettings(); - var isRecoveryMode = initSettings.CurrentScanningPage > 0; + var initCurrentTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken); + var initCurrentPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken); + var isRecoveryMode = initCurrentPage > 0; - foreach (var type in Enum.GetValues()) + Enum.TryParse(initCurrentTypeStr, true, out var initCurrentType); + + foreach (var type in ScannedAssetTypes) { if (stoppingToken.IsCancellationRequested) break; if (isRecoveryMode) { - if (type != initSettings.CurrentScanningType) + if (type != initCurrentType) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Recovery: {AssetType} was already processed. Skipping.", type); continue; @@ -80,19 +93,17 @@ public class AssetScannerBackgroundService : BackgroundService } else { - var settings = await settingsService.GetSettings(); - settings.CurrentScanningType = type; - settings.CurrentScanningPage = 0; - await settingsService.SaveSettings(settings); + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken); + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); } await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Processing asset type: {AssetType}...", type); await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken); - var currentSettings = await settingsService.GetSettings(); - var delaySeconds = currentSettings.FinishedInitialScan - ? currentSettings.AssetUpdateTypeDelay - : currentSettings.InitAssetUpdateTypeDelay; + var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken); + var delaySeconds = finishedInitial + ? await settingsService.GetSettingAsync(SettingKeys.ScannerTypeDelay, stoppingToken) + : await settingsService.GetSettingAsync(SettingKeys.ScannerInitTypeDelay, stoppingToken); if (delaySeconds > 0) { @@ -102,19 +113,18 @@ public class AssetScannerBackgroundService : BackgroundService } } - var finalSettings = await settingsService.GetSettings(); - finalSettings.CurrentScanningPage = 0; + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); - if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested) + var finishedScan = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken); + if (!finishedScan && !stoppingToken.IsCancellationRequested) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Initial scan successfully completed. Switching FinishedInitialScan to true."); - finalSettings.FinishedInitialScan = true; + await settingsService.SetSettingAsync(SettingKeys.ScannerFinishedInitialScan, true, stoppingToken); } - await settingsService.SaveSettings(finalSettings); - - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting 1 minute before starting the next cycle."); - await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + var cycleDelay = await settingsService.GetSettingAsync(SettingKeys.ScannerCycleDelayMinutes, stoppingToken); + await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Full scan cycle completed. Waiting {Minutes} minutes before starting the next cycle.", cycleDelay); + await Task.Delay(TimeSpan.FromMinutes(cycleDelay), stoppingToken); } catch (Exception e) when (!stoppingToken.IsCancellationRequested) { @@ -129,7 +139,7 @@ public class AssetScannerBackgroundService : BackgroundService private async Task HandleAssetType( AssetType type, ITradeRepublicService tradeRepublicService, - ISettingsDbService settingsDbService, + ISettingsService settingsService, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) @@ -144,26 +154,28 @@ public class AssetScannerBackgroundService : BackgroundService _currAssetsCount ??= new AssetsCount(); var currentItemOffset = 0; - var settings = await settingsDbService.GetSettings(); - var pageSize = Math.Clamp(settings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : settings.TradeRepublicMaxRequestPageSize, 1, 100); + var configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken); + var pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100); - if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0) + var currentScanningTypeStr = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningType, stoppingToken); + var currentScanningPage = await settingsService.GetSettingAsync(SettingKeys.ScannerCurrentScanningPage, stoppingToken); + + if (string.Equals(currentScanningTypeStr, type.ToString(), StringComparison.OrdinalIgnoreCase) && currentScanningPage > 0) { - currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize; + currentItemOffset = (currentScanningPage - 1) * pageSize; await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).", - type, settings.CurrentScanningPage, currentItemOffset); + type, currentScanningPage, currentItemOffset); } while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested) { - var currentSettings = await settingsDbService.GetSettings(); - pageSize = Math.Clamp(currentSettings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : currentSettings.TradeRepublicMaxRequestPageSize, 1, 100); + configuredPageSize = await settingsService.GetSettingAsync(SettingKeys.ScannerMaxPageSize, stoppingToken); + pageSize = Math.Clamp(configuredPageSize <= 0 ? 50 : configuredPageSize, 1, 100); var currentPage = (currentItemOffset / pageSize) + 1; - currentSettings.CurrentScanningType = type; - currentSettings.CurrentScanningPage = currentPage; - await settingsDbService.SaveSettings(currentSettings); + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningType, type.ToString(), stoppingToken); + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, currentPage, stoppingToken); await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}", type, currentPage, currentItemOffset, totalCount); @@ -173,8 +185,7 @@ public class AssetScannerBackgroundService : BackgroundService if (assets?.Results == null || assets.Results.Count == 0) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", type, currentPage); - currentSettings.CurrentScanningPage = 0; - await settingsDbService.SaveSettings(currentSettings); + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); break; } @@ -186,14 +197,14 @@ public class AssetScannerBackgroundService : BackgroundService if (assets.Results.Count < pageSize) { await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] Reached the last page for {AssetType}.", type); - currentSettings.CurrentScanningPage = 0; - await settingsDbService.SaveSettings(currentSettings); + await settingsService.SetSettingAsync(SettingKeys.ScannerCurrentScanningPage, 0, stoppingToken); break; } - var delaySeconds = currentSettings.FinishedInitialScan - ? currentSettings.BatchAssetUpdateDelay - : currentSettings.InitBatchAssetUpdateDelay; + var finishedInitial = await settingsService.GetSettingAsync(SettingKeys.ScannerFinishedInitialScan, stoppingToken); + var delaySeconds = finishedInitial + ? await settingsService.GetSettingAsync(SettingKeys.ScannerBatchDelay, stoppingToken) + : await settingsService.GetSettingAsync(SettingKeys.ScannerInitBatchDelay, stoppingToken); if (delaySeconds > 0) { @@ -218,6 +229,26 @@ public class AssetScannerBackgroundService : BackgroundService var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetScannerBackgroundService] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows); + + // Inline logo fetching for each asset in the batch if missing on disk + string directoryPath = Volumes.LogosRelativePath; + foreach (var a in assets) + { + if (stoppingToken.IsCancellationRequested) break; + if (string.IsNullOrWhiteSpace(a.Isin)) continue; + + string cleanIsin = a.Isin.Trim().ToUpperInvariant(); + string logoPath = Path.Combine(directoryPath, $"{cleanIsin}.svg"); + + if (!File.Exists(logoPath)) + { + try + { + await indexService.DownloadAndSaveLogoAsync(cleanIsin, stoppingToken); + } + catch { /* Ignore non-fatal logo fetch failures */ } + } + } if (changedRows > 0) { diff --git a/FinlyticAssets/Services/AssetsDbService.cs b/FinlyticAssets/Services/AssetsDbService.cs index 7c92892..eeb11de 100644 --- a/FinlyticAssets/Services/AssetsDbService.cs +++ b/FinlyticAssets/Services/AssetsDbService.cs @@ -14,7 +14,7 @@ using Microsoft.EntityFrameworkCore; namespace FinlyticAssets.Services; /// -/// Defines database operations for managing Trade Republic asset entities. +/// Defines database operations for managing Trade Republic asset entities and on-demand derivatives. /// public interface IAssetsDbService { @@ -24,7 +24,6 @@ public interface IAssetsDbService public Task> GetAssetsByIsinAsync(string isin); public Task> GetValidAssetsByIsinAsync(string isin); public Task> FindAffectedActiveAssetsAsync(string searchQuery); - public Task UpdateAssetImageIdAsync(string isin, string imageId); public Task DeleteAssetAsync(string isin); public Task> GetDiscoveryAssetsAsync(int limit = 15); public Task> GetDerivativesByUnderlyingAsync(string underlyingIsin, string optionType = "long", decimal? targetLeverage = null, string? after = null, int? page = null, bool forceRefresh = false, CancellationToken cancellationToken = default); @@ -63,7 +62,6 @@ public class AssetsDbService : IAssetsDbService Asset = a, Score = (a.Tags?.Count ?? 0) * 10 + (a.HasCfd ? 5 : 0) - + (string.IsNullOrEmpty(a.ImageId) ? 0 : 15) + (a.Name.Length > 3 ? 5 : 0) }) .OrderByDescending(x => x.Score) @@ -147,7 +145,6 @@ public class AssetsDbService : IAssetsDbService existingEntity.Type = dtoAsset.Type; existingEntity.InstrumentCategory = dtoAsset.InstrumentCategory; existingEntity.HasCfd = dtoAsset.HasCfd; - existingEntity.ImageId = dtoAsset.ImageId; existingEntity.LastUpdatedAt = now; UpdateSubtypeProperties(existingEntity, dtoAsset); @@ -220,14 +217,13 @@ public class AssetsDbService : IAssetsDbService existingEntity.Type != dto.Type || existingEntity.InstrumentCategory != dto.InstrumentCategory || existingEntity.HasCfd != dto.HasCfd || - existingEntity.ImageId != dto.ImageId || - !existingEntity.Tags.SequenceEqual(mappedTags)) + !existingEntity.Tags.Select(t => t.Id).Order().SequenceEqual(mappedTags.Select(t => t.Id).Order()) || + HasSubtypeChanges(existingEntity, dto)) { existingEntity.Name = dto.Name; existingEntity.Type = dto.Type; existingEntity.InstrumentCategory = dto.InstrumentCategory; existingEntity.HasCfd = dto.HasCfd; - existingEntity.ImageId = dto.ImageId; existingEntity.LastUpdatedAt = now; existingEntity.Tags = mappedTags; @@ -255,18 +251,8 @@ public class AssetsDbService : IAssetsDbService Type = stock.Type, InstrumentCategory = stock.InstrumentCategory, HasCfd = stock.HasCfd, - ImageId = stock.ImageId, DerivativeProductCategories = stock.DerivativeProductCategories?.ToList() ?? new List() }, - TradeRepublicCrypto crypto => new CryptoEntity - { - Isin = crypto.Isin, - Name = crypto.Name, - Type = crypto.Type, - InstrumentCategory = crypto.InstrumentCategory, - HasCfd = crypto.HasCfd, - ImageId = crypto.ImageId - }, TradeRepublicEtf etf => new EtfEntity { Isin = etf.Isin, @@ -274,7 +260,6 @@ public class AssetsDbService : IAssetsDbService Type = etf.Type, InstrumentCategory = etf.InstrumentCategory, HasCfd = etf.HasCfd, - ImageId = etf.ImageId, DerivativeProductCategories = etf.DerivativeProductCategories?.ToList() ?? new List() }, TradeRepublicSynthetic syn => new SyntheticEntity @@ -284,39 +269,15 @@ public class AssetsDbService : IAssetsDbService Type = syn.Type, InstrumentCategory = syn.InstrumentCategory, HasCfd = syn.HasCfd, - ImageId = syn.ImageId, DerivativeProductCategories = syn.DerivativeProductCategories?.ToList() ?? new List() }, - TradeRepublicBond bond => new BondEntity - { - Isin = bond.Isin, - Name = bond.Name, - Type = bond.Type, - InstrumentCategory = bond.InstrumentCategory, - HasCfd = bond.HasCfd, - ImageId = bond.ImageId, - BondIssuerName = bond.BondIssuerName, - SearchSubtitle = bond.SearchSubtitle - }, - TradeRepublicDerivative deriv => new DerivativeEntity - { - Isin = deriv.Isin, - Name = deriv.Name, - Type = deriv.Type, - InstrumentCategory = deriv.InstrumentCategory, - HasCfd = deriv.HasCfd, - ImageId = deriv.ImageId, - UnderlyingIsin = deriv.UnderlyingIsin, - DerivativeProductCategories = deriv.DerivativeProductCategories?.ToList() ?? new List() - }, _ => new StockEntity { Isin = dto.Isin, Name = dto.Name, Type = dto.Type, InstrumentCategory = dto.InstrumentCategory, - HasCfd = dto.HasCfd, - ImageId = dto.ImageId + HasCfd = dto.HasCfd } }; } @@ -334,14 +295,21 @@ public class AssetsDbService : IAssetsDbService case SyntheticEntity syn when dto is TradeRepublicSynthetic synDto: syn.DerivativeProductCategories = synDto.DerivativeProductCategories?.ToList() ?? new List(); break; - case BondEntity bond when dto is TradeRepublicBond b: - bond.BondIssuerName = b.BondIssuerName; - bond.SearchSubtitle = b.SearchSubtitle; - break; - case DerivativeEntity deriv when dto is TradeRepublicDerivative d: - deriv.UnderlyingIsin = d.UnderlyingIsin; - deriv.DerivativeProductCategories = d.DerivativeProductCategories?.ToList() ?? new List(); - break; + } + } + + private static bool HasSubtypeChanges(AssetEntity entity, TradeRepublicAsset dto) + { + switch (entity) + { + case StockEntity stock when dto is TradeRepublicStock s: + return !(stock.DerivativeProductCategories?.SequenceEqual(s.DerivativeProductCategories ?? Array.Empty()) ?? (s.DerivativeProductCategories == null || !s.DerivativeProductCategories.Any())); + case EtfEntity etf when dto is TradeRepublicEtf e: + return !(etf.DerivativeProductCategories?.SequenceEqual(e.DerivativeProductCategories ?? Array.Empty()) ?? (e.DerivativeProductCategories == null || !e.DerivativeProductCategories.Any())); + case SyntheticEntity syn when dto is TradeRepublicSynthetic synDto: + return !(syn.DerivativeProductCategories?.SequenceEqual(synDto.DerivativeProductCategories ?? Array.Empty()) ?? (synDto.DerivativeProductCategories == null || !synDto.DerivativeProductCategories.Any())); + default: + return false; } } @@ -362,24 +330,6 @@ public class AssetsDbService : IAssetsDbService .ToListAsync(); } - /// Inherits documentation from interface. - public async Task UpdateAssetImageIdAsync(string isin, string imageId) - { - var existingAssets = await _context.TradeRepublicAssets - .Where(a => a.Isin == isin) - .ToListAsync(); - - if (existingAssets.Count > 0) - { - foreach (var asset in existingAssets) - { - asset.ImageId = imageId; - } - await _context.SaveChangesAsync(); - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Updated ImageId for ISIN {Isin} in database to '{ImageId}'", isin, imageId); - } - } - /// Inherits documentation from interface. public async Task DeleteAssetAsync(string isin) { @@ -412,7 +362,6 @@ public class AssetsDbService : IAssetsDbService int pageIndex = Math.Max(0, page ?? 0); decimal levQuery = targetLeverage.HasValue && targetLeverage.Value > 0 ? targetLeverage.Value : 0m; - string trAfter = !string.IsNullOrEmpty(after) ? after : (pageIndex > 0 ? pageIndex.ToString() : "0"); await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[AssetsDbService] Fetching derivatives for {Isin} (OptionType: {Option}, Leverage: {Lev}, Page: {Page}, TR-After: {After})", @@ -438,8 +387,7 @@ public class AssetsDbService : IAssetsDbService { var now = DateTime.UtcNow; var isins = fetchedItems.Select(r => r.Isin).ToList(); - var existingDerivatives = await _context.TradeRepublicAssets - .OfType() + var existingDerivatives = await _context.Derivatives .Where(d => isins.Contains(d.Isin)) .ToDictionaryAsync(d => d.Isin, cancellationToken); @@ -466,14 +414,13 @@ public class AssetsDbService : IAssetsDbService existing.NextGenProductCategoryName = item.NextGenProductCategoryName; existing.Issuer = item.Issuer; existing.IssuerDisplayName = item.IssuerDisplayName; - existing.IssuerImageId = item.IssuerImageId; existing.Size = item.Size; existing.Factor = item.Factor; existing.Delta = item.Delta; existing.Currency = item.Currency; existing.LastUpdatedAt = now; - _context.TradeRepublicAssets.Update(existing); + _context.Derivatives.Update(existing); resultEntities.Add(existing); } else @@ -482,8 +429,6 @@ public class AssetsDbService : IAssetsDbService { Isin = item.Isin, Name = !string.IsNullOrWhiteSpace(item.ProductCategoryName) ? item.ProductCategoryName : item.Isin, - Type = "derivative", - InstrumentCategory = "derivative", UnderlyingIsin = underlyingIsin, Strike = item.Strike ?? 0m, Barrier = item.Barrier ?? 0m, @@ -494,27 +439,33 @@ public class AssetsDbService : IAssetsDbService NextGenProductCategoryName = item.NextGenProductCategoryName, Issuer = item.Issuer, IssuerDisplayName = item.IssuerDisplayName, - IssuerImageId = item.IssuerImageId, Size = item.Size, Factor = item.Factor, Delta = item.Delta, Currency = item.Currency, - LastUpdatedAt = now + LastUpdatedAt = now, + CreatedAt = now }; - await _context.TradeRepublicAssets.AddAsync(newDeriv, cancellationToken); + await _context.Derivatives.AddAsync(newDeriv, cancellationToken); resultEntities.Add(newDeriv); } } await _context.SaveChangesAsync(cancellationToken); - return resultEntities; + return resultEntities.Where(d => d.Barrier > 0 && d.Leverage > 0).ToList(); } - return await _context.TradeRepublicAssets - .OfType() + var dbQuery = _context.Derivatives .AsNoTracking() - .Where(d => d.UnderlyingIsin == underlyingIsin) + .Where(d => d.UnderlyingIsin == underlyingIsin && d.OptionType == targetOptionType && d.Barrier > 0 && d.Leverage > 0); + + if (targetLeverage.HasValue && targetLeverage.Value > 0) + { + dbQuery = dbQuery.OrderBy(d => Math.Abs(d.Leverage - targetLeverage.Value)); + } + + return await dbQuery .Take(pageSize) .ToListAsync(cancellationToken); } diff --git a/FinlyticAssets/Services/AssetsIndexService.cs b/FinlyticAssets/Services/AssetsIndexService.cs index cefff25..ecc905f 100644 --- a/FinlyticAssets/Services/AssetsIndexService.cs +++ b/FinlyticAssets/Services/AssetsIndexService.cs @@ -5,7 +5,7 @@ using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using FinlyticAssets.Models; +using FinlyticCore.Models.Assets; using FinlyticAssets.Util; using FinlyticCore.Services; diff --git a/FinlyticAssets/Services/LogoFetcherBackgroundService.cs b/FinlyticAssets/Services/LogoFetcherBackgroundService.cs deleted file mode 100644 index 2530a2d..0000000 --- a/FinlyticAssets/Services/LogoFetcherBackgroundService.cs +++ /dev/null @@ -1,169 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FinlyticAssets.Util; -using FinlyticCore.Services; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; - -namespace FinlyticAssets.Services; - -/// -/// Dedicated background service that periodically scans for missing asset logos in the local storage directory -/// and fetches them in batches from Trade Republic CDN. -/// Swaps missing/404 logos with a clean SVG placeholder image and triggers ReCreateIndexFileAsync. -/// -public class LogoFetcherBackgroundService : BackgroundService -{ - private readonly IFinlyticLogger _finlyticLogger; - private readonly IServiceScopeFactory _scopeFactory; - private readonly HttpClient _httpClient; - - private const string PlaceholderSvg = """ - - - - - - """; - - public LogoFetcherBackgroundService( - IFinlyticLogger finlyticLogger, - IServiceScopeFactory scopeFactory) - { - _finlyticLogger = finlyticLogger; - _scopeFactory = scopeFactory; - _httpClient = new HttpClient(); - _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); - _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "image/svg+xml,image/*,*/*"); - _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://traderepublic.com/"); - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] LogoFetcherBackgroundService started. Will fetch missing logos periodically."); - - await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); - - while (!stoppingToken.IsCancellationRequested) - { - try - { - await ProcessMissingLogosBatchAsync(stoppingToken); - } - catch (Exception ex) when (!stoppingToken.IsCancellationRequested) - { - await _finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Error occurred while executing logo batch fetch."); - } - - try - { - await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); - } - catch (OperationCanceledException) - { - break; - } - } - - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] LogoFetcherBackgroundService stopped."); - } - - private async Task ProcessMissingLogosBatchAsync(CancellationToken stoppingToken) - { - using var scope = _scopeFactory.CreateScope(); - var dbService = scope.ServiceProvider.GetRequiredService(); - var indexService = scope.ServiceProvider.GetRequiredService(); - - var validAssets = await dbService.GetAllValidAssetsAsync(); - if (validAssets == null || !validAssets.Any()) return; - - string directoryPath = Volumes.LogosRelativePath; - if (!Directory.Exists(directoryPath)) - { - Directory.CreateDirectory(directoryPath); - } - - var missingIsins = validAssets - .Select(a => a.Isin?.Trim().ToUpperInvariant()) - .Where(isin => !string.IsNullOrEmpty(isin)) - .Distinct() - .Where(isin => !File.Exists(Path.Combine(directoryPath, $"{isin}.svg"))) - .ToList(); - - if (missingIsins.Count == 0) - { - await _finlyticLogger.LogDebugAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] All asset logos are downloaded and up to date."); - return; - } - - var batchToFetch = missingIsins.Take(60).ToList(); - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Found {Count} missing logos on disk. Fetching bulk batch of {BatchSize} logos...", missingIsins.Count, batchToFetch.Count); - - int successCount = 0; - - foreach (var isin in batchToFetch) - { - if (stoppingToken.IsCancellationRequested) break; - - string targetUrl = $"https://assets.traderepublic.com/img/logos/{isin}/v2/dark.min.svg"; - string filePath = Path.Combine(directoryPath, $"{isin}.svg"); - string dbImageEndpoint = $"/api/v1/logo/{isin}"; - - try - { - using var response = await _httpClient.GetAsync(targetUrl, stoppingToken); - if (response.IsSuccessStatusCode) - { - byte[] data = await response.Content.ReadAsByteArrayAsync(stoppingToken); - await File.WriteAllBytesAsync(filePath, data, stoppingToken); - successCount++; - } - else - { - await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Logo not found on CDN for ISIN {Isin} (HTTP {StatusCode}). Saving SVG placeholder.", isin, response.StatusCode); - byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg); - await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken); - successCount++; - } - - await dbService.UpdateAssetImageIdAsync(isin, dbImageEndpoint); - } - catch (Exception ex) when (!stoppingToken.IsCancellationRequested) - { - await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Exception while downloading logo for ISIN {Isin} from {Url}. Saving SVG placeholder.", isin, targetUrl); - try - { - byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg); - await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken); - successCount++; - - await dbService.UpdateAssetImageIdAsync(isin, dbImageEndpoint); - } - catch { } - } - - await Task.Delay(50, stoppingToken); - } - - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Batch fetch complete. Successfully processed {SuccessCount}/{BatchSize} logos. Remaining missing: {Remaining}", - successCount, batchToFetch.Count, missingIsins.Count - batchToFetch.Count); - - if (successCount > 0) - { - try - { - await indexService.ReCreateIndexFileAsync(stoppingToken); - await _finlyticLogger.LogInfoAsync(SettingKeys.AssetsChannel, "[LogoFetcherBackgroundService] Successfully updated index.json after logo batch fetch."); - } - catch (Exception ex) - { - await _finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, ex, "[LogoFetcherBackgroundService] Failed to update index.json after logo batch fetch."); - } - } - } -} \ No newline at end of file diff --git a/FinlyticAssets/Services/SettingsDbService.cs b/FinlyticAssets/Services/SettingsDbService.cs deleted file mode 100644 index 0210523..0000000 --- a/FinlyticAssets/Services/SettingsDbService.cs +++ /dev/null @@ -1,125 +0,0 @@ -using FinlyticAssets.Database; -using FinlyticAssets.Entities; -using Microsoft.EntityFrameworkCore; - -namespace FinlyticAssets.Services; - -/// -/// Defines the business logic for managing global application settings. -/// Supports retrieving and updating (upserting) the central single-row configuration record. -/// -public interface ISettingsDbService -{ - /// - /// Retrieves the current global settings from the database. - /// - /// - /// A task that represents the asynchronous operation. The task result contains the current . - /// If no settings exist in the database yet, a new instance initialized with default values is returned. - /// - public Task GetSettings(); - - /// - /// Persists the provided settings by updating the existing record or inserting the first one if the table is empty. - /// - /// The new configuration values to be persisted. - /// - /// A task that represents the asynchronous operation. The task result contains the freshly saved - /// and tracked instance. - /// - public Task SaveSettings(Settings settings); - - /// - /// Updates settings from a key-value dictionary received via Admin Panel MQTT events. - /// - public Task UpdateSettingsFromDictionary(Dictionary dictionary); -} - -/// -/// Implements the utilizing Entity Framework Core. -/// This service is designed for a single-row table architecture to maintain stateful global configurations. -/// -public class SettingsDbService : ISettingsDbService -{ - private readonly AssetsDbContext _context; - - /// - /// Initializes a new instance of the class with the required database context. - /// - /// The EF Core context used to access the assets database. - public SettingsDbService(AssetsDbContext context) - { - _context = context; - } - - /// Inherits documentation from interface. - public async Task GetSettings() - { - var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync(); - - if (settings == null) - { - settings = new Settings { Id = Guid.NewGuid() }; - _context.Settings.Add(settings); - await _context.SaveChangesAsync(); - _context.ChangeTracker.Clear(); - } - - return settings; - } - - /// Inherits documentation from interface. - public async Task SaveSettings(Settings settings) - { - var existing = await _context.Settings.FirstOrDefaultAsync(); - - if (existing == null) - { - if (settings.Id == Guid.Empty) - { - settings.Id = Guid.NewGuid(); - } - _context.Settings.Add(settings); - await _context.SaveChangesAsync(); - return settings; - } - else - { - existing.FinishedInitialScan = settings.FinishedInitialScan; - existing.TradeRepublicMaxRequestPageSize = settings.TradeRepublicMaxRequestPageSize; - existing.AssetUpdateTypeDelay = settings.AssetUpdateTypeDelay; - existing.InitAssetUpdateTypeDelay = settings.InitAssetUpdateTypeDelay; - existing.InitBatchAssetUpdateDelay = settings.InitBatchAssetUpdateDelay; - existing.BatchAssetUpdateDelay = settings.BatchAssetUpdateDelay; - existing.CurrentScanningType = settings.CurrentScanningType; - existing.CurrentScanningPage = settings.CurrentScanningPage; - _context.Settings.Update(existing); - await _context.SaveChangesAsync(); - return existing; - } - } - - /// Inherits documentation from interface. - public async Task UpdateSettingsFromDictionary(Dictionary dictionary) - { - var settings = await GetSettings(); - - foreach (var (key, value) in dictionary) - { - if (string.Equals(key, "TradeRepublicMaxRequestPageSize", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ps)) - settings.TradeRepublicMaxRequestPageSize = ps; - else if (string.Equals(key, "AssetUpdateTypeDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var autd)) - settings.AssetUpdateTypeDelay = autd; - else if (string.Equals(key, "InitAssetUpdateTypeDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var iautd)) - settings.InitAssetUpdateTypeDelay = iautd; - else if (string.Equals(key, "BatchAssetUpdateDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var baud)) - settings.BatchAssetUpdateDelay = baud; - else if (string.Equals(key, "InitBatchAssetUpdateDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ibaud)) - settings.InitBatchAssetUpdateDelay = ibaud; - else if (string.Equals(key, "FinishedInitialScan", StringComparison.OrdinalIgnoreCase) && bool.TryParse(value, out var fis)) - settings.FinishedInitialScan = fis; - } - - await SaveSettings(settings); - } -} diff --git a/FinlyticAssets/Util/AssetMapper.cs b/FinlyticAssets/Util/AssetMapper.cs index 7b65f45..246a385 100644 --- a/FinlyticAssets/Util/AssetMapper.cs +++ b/FinlyticAssets/Util/AssetMapper.cs @@ -1,3 +1,6 @@ +using System; +using System.Collections.Generic; +using System.Linq; using FinlyticAssets.Entities; using FinlyticCore.Dtos.Assets; @@ -14,6 +17,8 @@ public static class AssetMapper Type = t.Type }).ToList(); + var dynamicLogoUrl = $"/api/v1/logo/{entity.Isin}"; + return entity switch { StockEntity stock => new StockDto @@ -23,7 +28,7 @@ public static class AssetMapper Type = stock.Type, InstrumentCategory = stock.InstrumentCategory, HasCfd = stock.HasCfd, - ImageId = stock.ImageId, + ImageId = dynamicLogoUrl, LastUpdatedAt = stock.LastUpdatedAt, Tags = dtoTags, DerivativeProductCategories = stock.DerivativeProductCategories @@ -35,7 +40,7 @@ public static class AssetMapper Type = etf.Type, InstrumentCategory = etf.InstrumentCategory, HasCfd = etf.HasCfd, - ImageId = etf.ImageId, + ImageId = dynamicLogoUrl, LastUpdatedAt = etf.LastUpdatedAt, Tags = dtoTags, DerivativeProductCategories = etf.DerivativeProductCategories, @@ -44,59 +49,6 @@ public static class AssetMapper Subtitle = etf.Subtitle, SearchSubtitle = etf.SearchSubtitle }, - CryptoEntity crypto => new CryptoDto - { - Isin = crypto.Isin, - Name = crypto.Name, - Type = crypto.Type, - InstrumentCategory = crypto.InstrumentCategory, - HasCfd = crypto.HasCfd, - ImageId = crypto.ImageId, - LastUpdatedAt = crypto.LastUpdatedAt, - Tags = dtoTags, - Subtitle = crypto.Subtitle, - SearchSubtitle = crypto.SearchSubtitle - }, - BondEntity bond => new BondDto - { - Isin = bond.Isin, - Name = bond.Name, - Type = bond.Type, - InstrumentCategory = bond.InstrumentCategory, - HasCfd = bond.HasCfd, - ImageId = bond.ImageId, - LastUpdatedAt = bond.LastUpdatedAt, - Tags = dtoTags, - BondIssuerName = bond.BondIssuerName, - SearchSubtitle = bond.SearchSubtitle - }, - DerivativeEntity deriv => new DerivativeDto - { - Isin = deriv.Isin, - Name = deriv.Name, - Type = deriv.Type, - InstrumentCategory = deriv.InstrumentCategory, - HasCfd = deriv.HasCfd, - ImageId = deriv.ImageId, - LastUpdatedAt = deriv.LastUpdatedAt, - Tags = dtoTags, - DerivativeProductCategories = deriv.DerivativeProductCategories, - UnderlyingIsin = deriv.UnderlyingIsin, - OptionType = deriv.OptionType.ToString(), - ProductCategoryName = deriv.ProductCategoryName, - NextGenProductCategoryName = deriv.NextGenProductCategoryName, - Strike = deriv.Strike, - Barrier = deriv.Barrier, - Leverage = deriv.Leverage, - Size = deriv.Size, - Factor = deriv.Factor, - Delta = deriv.Delta, - Currency = deriv.Currency, - Expiry = deriv.Expiry, - Issuer = deriv.Issuer, - IssuerDisplayName = deriv.IssuerDisplayName, - IssuerImageId = deriv.IssuerImageId - }, SyntheticEntity synth => new SyntheticDto { Isin = synth.Isin, @@ -104,12 +56,53 @@ public static class AssetMapper Type = synth.Type, InstrumentCategory = synth.InstrumentCategory, HasCfd = synth.HasCfd, - ImageId = synth.ImageId, + ImageId = dynamicLogoUrl, LastUpdatedAt = synth.LastUpdatedAt, Tags = dtoTags, DerivativeProductCategories = synth.DerivativeProductCategories }, - _ => throw new NotSupportedException($"Mapping for type {entity.GetType().Name} is not supported.") + _ => new StockDto + { + Isin = entity.Isin, + Name = entity.Name, + Type = entity.Type, + InstrumentCategory = entity.InstrumentCategory, + HasCfd = entity.HasCfd, + ImageId = dynamicLogoUrl, + LastUpdatedAt = entity.LastUpdatedAt, + Tags = dtoTags + } + }; + } + + public static DerivativeDto ToDto(this DerivativeEntity deriv) + { + return new DerivativeDto + { + Isin = deriv.Isin, + Name = deriv.Name, + Type = "derivative", + InstrumentCategory = "derivative", + HasCfd = false, + ImageId = $"/api/v1/logo/{deriv.UnderlyingIsin}", + LastUpdatedAt = deriv.LastUpdatedAt, + Tags = new List(), + DerivativeProductCategories = deriv.DerivativeProductCategories, + UnderlyingIsin = deriv.UnderlyingIsin, + OptionType = deriv.OptionType.ToString(), + ProductCategoryName = deriv.ProductCategoryName, + NextGenProductCategoryName = deriv.NextGenProductCategoryName, + Strike = deriv.Strike, + Barrier = deriv.Barrier, + Leverage = deriv.Leverage, + Size = deriv.Size, + Factor = deriv.Factor, + Delta = deriv.Delta, + Currency = deriv.Currency, + Expiry = deriv.Expiry, + Issuer = deriv.Issuer, + IssuerDisplayName = deriv.IssuerDisplayName, + IssuerImageId = null }; } @@ -117,4 +110,9 @@ public static class AssetMapper { return entities.Select(e => e.ToDto()).ToList(); } + + public static List ToDtoList(this IEnumerable derivatives) + { + return derivatives.Select(d => d.ToDto()).ToList(); + } } \ No newline at end of file diff --git a/FinlyticAssets/Util/AssetsMqttClient.cs b/FinlyticAssets/Util/AssetsMqttClient.cs index 6dc303c..cb0ed42 100644 --- a/FinlyticAssets/Util/AssetsMqttClient.cs +++ b/FinlyticAssets/Util/AssetsMqttClient.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text.Json; @@ -6,9 +7,14 @@ using System.Threading; using System.Threading.Tasks; using FinlyticAssets.Entities; using FinlyticAssets.Services; +using FinlyticCore.Dtos; +using FinlyticCore.Dtos.Assets; using FinlyticCore.Dtos.Settings; +using FinlyticCore.Dtos.TechnicalAnalysis; using FinlyticCore.Models; +using FinlyticCore.Models.Assets; using FinlyticCore.Services; +using FinlyticCore.Services.TradeRepublic; using FinlyticCore.Util; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -18,13 +24,14 @@ using Microsoft.Extensions.Logging; namespace FinlyticAssets.Util; /// -/// Represents a managed MQTT client acting as a server-side RPC provider within the asset microservice. +/// Managed MQTT client acting as an RPC provider for asset lookups, discovery, and on-demand derivatives. /// public class AssetsMqttClient : ManagedMqttClient, IHostedService { private readonly ILogger _logger; private readonly IServiceScopeFactory _scopeFactory; private readonly IConfiguration _configuration; + private readonly ConcurrentDictionary _priceCache = new(StringComparer.OrdinalIgnoreCase); public AssetsMqttClient( ILogger logger, @@ -41,12 +48,7 @@ public class AssetsMqttClient : ManagedMqttClient, IHostedService /// public async Task StartAsync(CancellationToken cancellationToken) { - var config = new MqttConfiguration() - { - Host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost", - Port = Convert.ToInt32(_configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883"), - ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticAssets")}_{Guid.NewGuid()}" - }; + var config = MqttConfiguration.FromConfiguration(_configuration, "FinlyticAssets"); _logger.LogInformation("Starting Assets MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId); await ConnectAsync(config); @@ -66,278 +68,158 @@ public class AssetsMqttClient : ManagedMqttClient, IHostedService /// protected override async Task OnConnectedAsync() { - _logger.LogInformation("Assets MQTT Client connected. Subscribing to topics..."); - await SubscribeAsync("services/request/assets_Get/#"); - await SubscribeAsync("services/request/assets_Search/#"); - await SubscribeAsync("services/request/assets_GetDiscovery/#"); - await SubscribeAsync("services/request/assets_GetDerivatives/#"); - await SubscribeAsync("services/request/assets_FetchLogo/#"); - await SubscribeAsync("services/request/assets_settings_GetAll/#"); - await SubscribeAsync("services/request/assets_settings_Update/#"); - await SubscribeAsync("services/request/health_Ping/#"); - await SubscribeAsync("services/config/updated/#"); + _logger.LogInformation("Assets MQTT Client connected. Registering topic subscriptions..."); - FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) => + await SubscribeAsync(MqttTopics.ResponseWildcard); + await SubscribeRpcAsync>(MqttTopics.RequestFilter(MqttTopics.Channels.AssetsGet), HandleAssetsGetRpcAsync); + await SubscribeRpcAsync>(MqttTopics.RequestFilter(MqttTopics.Channels.AssetsGetDiscovery), HandleAssetsGetDiscoveryRpcAsync); + await SubscribeRpcAsync>(MqttTopics.RequestFilter(MqttTopics.Channels.AssetsGetDerivatives), HandleAssetsGetDerivativesRpcAsync); + await SubscribeRpcAsync(MqttTopics.RequestFilter(MqttTopics.Channels.TrGetLivePrice), HandleGetLivePriceRpcAsync); + await SubscribeRpcAsync>(MqttTopics.RequestFilter(MqttTopics.Channels.AssetsSettingsGetAll), HandleSettingsGetAllRpcAsync); + await SubscribeRpcAsync, List>(MqttTopics.RequestFilter(MqttTopics.Channels.AssetsSettingsUpdate), HandleSettingsUpdateRpcAsync); + await SubscribeAsync(MqttTopics.RequestFilter(MqttTopics.Channels.HealthPing), HandleHealthPingRpcAsync); + + FinlyticLogBroadcaster.OnLogPublished = async (logDto) => { if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticAssets", StringComparison.OrdinalIgnoreCase)) { - await PublishAsync("finlytic/logs/FinlyticAssets", logDto); + await PublishAsync(MqttTopics.Logs("FinlyticAssets"), logDto); } }; } - /// - /// Processes incoming messages on the subscribed topics. - /// - protected override async Task OnMessageReceivedAsync(string topic, string payload) + private async Task> HandleAssetsGetRpcAsync(GetValidAssetRequest? req, string correlationId) { - if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase)) + if (req == null || string.IsNullOrWhiteSpace(req.Isin)) return []; + + using var scope = _scopeFactory.CreateScope(); + var dbService = scope.ServiceProvider.GetRequiredService(); + var assets = await dbService.GetValidAssetsByIsinAsync(req.Isin.Trim().ToUpperInvariant()); + return assets.ToDtoList(); + } + + private async Task> HandleAssetsGetDiscoveryRpcAsync(GetDiscoveryAssetsRequest? req, string correlationId) + { + using var scope = _scopeFactory.CreateScope(); + var dbService = scope.ServiceProvider.GetRequiredService(); + int limit = req?.Limit > 0 ? req.Limit : 15; + var discoveryAssets = await dbService.GetDiscoveryAssetsAsync(limit); + return discoveryAssets.ToDtoList(); + } + + private async Task> HandleAssetsGetDerivativesRpcAsync(GetDerivativesRequest? req, string correlationId) + { + if (req == null || string.IsNullOrWhiteSpace(req.UnderlyingIsin)) return []; + + using var scope = _scopeFactory.CreateScope(); + var dbService = scope.ServiceProvider.GetRequiredService(); + var derivatives = await dbService.GetDerivativesByUnderlyingAsync( + req.UnderlyingIsin.Trim().ToUpperInvariant(), + req.OptionType, + req.TargetLeverage, + req.After, + req.Page, + req.ShouldForceRefresh); + return derivatives.ToDtoList(); + } + + private async Task HandleGetLivePriceRpcAsync(IsinRequest? req, string correlationId) + { + if (req == null || string.IsNullOrWhiteSpace(req.Isin)) return null; + + var cleanIsin = req.Isin.Trim().ToUpperInvariant(); + + // Return fresh price from cache if less than 3 seconds old + if (_priceCache.TryGetValue(cleanIsin, out var cached) && (DateTime.UtcNow - cached.CachedAt).TotalSeconds < 3) { - await HandleConfigUpdatedAsync(topic, payload); - return; + return cached.Price; } - var segments = topic.Split('/'); - if (segments.Length < 4) return; + using var scope = _scopeFactory.CreateScope(); + var trService = scope.ServiceProvider.GetRequiredService(); + var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); - var channel = segments[2]; - var correlationId = segments[segments.Length - 1]; - - if (topic.Contains("health_Ping", StringComparison.OrdinalIgnoreCase)) - { - await HandleHealthPingAsync(topic, segments, correlationId); - return; - } - - if (topic.StartsWith("services/request/assets_settings_GetAll", StringComparison.OrdinalIgnoreCase)) - { - await HandleSettingsGetAllAsync(correlationId); - return; - } - - if (topic.StartsWith("services/request/assets_settings_Update", StringComparison.OrdinalIgnoreCase)) - { - await HandleSettingsUpdateAsync(payload, correlationId); - return; - } + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int? subId = null; try { - using var scope = _scopeFactory.CreateScope(); - var dbService = scope.ServiceProvider.GetRequiredService(); - var indexService = scope.ServiceProvider.GetRequiredService(); - - if (channel == "assets_FetchLogo") + subId = await trService.SubscribeRealtimeTickerAsync(cleanIsin, tick => { - await HandleFetchLogoAsync(payload, correlationId, indexService); - return; - } + decimal currentPrice = tick.Last?.PriceValue > 0 ? tick.Last.PriceValue : + (tick.Bid?.PriceValue > 0 && tick.Ask?.PriceValue > 0 ? (tick.Bid.PriceValue + tick.Ask.PriceValue) / 2m : + (tick.Ask?.PriceValue ?? tick.Bid?.PriceValue ?? 0m)); - List responseData = []; + decimal preClose = tick.Pre?.PriceValue > 0 ? tick.Pre.PriceValue : (tick.Open?.PriceValue ?? 0m); + decimal dailyChange = preClose > 0m ? ((currentPrice - preClose) / preClose) * 100m : 0m; - switch (channel) - { - case "assets_Get": - responseData = await HandleAssetsGetAsync(payload, dbService); - break; - case "assets_Search": - responseData = await HandleAssetsSearchAsync(payload, dbService); - break; - case "assets_GetDiscovery": - responseData = await HandleAssetsGetDiscoveryAsync(payload, dbService); - break; - case "assets_GetDerivatives": - responseData = (await HandleAssetsGetDerivativesAsync(payload, dbService)).Cast().ToList(); - break; - } + var livePrice = new LivePriceDto( + Isin: cleanIsin, + CurrentPrice: currentPrice, + DailyChangePercent: dailyChange, + Bid: tick.Bid?.PriceValue, + Ask: tick.Ask?.PriceValue + ); - string defaultResponseTopic = $"services/response/{channel}/{correlationId}"; - await PublishAsync(defaultResponseTopic, responseData.ToDtoList()); + _priceCache[cleanIsin] = (livePrice, DateTime.UtcNow); + tcs.TrySetResult(livePrice); + }); + + var result = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(3)); + return result; } catch (Exception ex) { - OnError(ex); + await finlyticLogger.LogWarningAsync(SettingKeys.AssetsChannel, "[AssetsMqttClient] Failed or timed out fetching live price for {Isin}: {Message}", cleanIsin, ex.Message); + if (_priceCache.TryGetValue(cleanIsin, out var stale)) + { + return stale.Price; + } + return null; + } + finally + { + if (subId.HasValue) + { + _ = trService.UnsubscribeRealtimeTickerAsync(subId.Value); + } } } - private async Task HandleSettingsGetAllAsync(string correlationId) + private async Task> HandleSettingsGetAllRpcAsync(object? _, string correlationId) { using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); var settingsService = scope.ServiceProvider.GetRequiredService(); - await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId); - try - { - var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) }); - var responseTopic = $"services/response/assets_settings_GetAll/{correlationId}"; - - await PublishAsync(responseTopic, settings); - await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic); - } - catch (Exception ex) - { - await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticAssets] [Settings_GetAll] Failed to retrieve settings."); - } + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_GetAll] Retrieving service dynamic settings [CorrelationId: {CorrelationId}]", correlationId); + return await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) }); } - private async Task HandleSettingsUpdateAsync(string payload, string correlationId) + private async Task> HandleSettingsUpdateRpcAsync(Dictionary? updates, string correlationId) { - if (string.IsNullOrWhiteSpace(payload)) return; - using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); var settingsService = scope.ServiceProvider.GetRequiredService(); await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId); - try + if (updates != null && updates.Count > 0) { - Dictionary? updates = null; - try - { - updates = JsonSerializer.Deserialize>(payload); - } - catch - { - var list = JsonSerializer.Deserialize>(payload); - if (list != null) - { - updates = new Dictionary(); - foreach (var item in list) updates[item.Key] = item.Value; - } - } - - if (updates != null && updates.Count > 0) - { - await settingsService.UpdateSettingsAsync(updates); - await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count); - } - - var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) }); - var responseTopic = $"services/response/assets_settings_Update/{correlationId}"; - await PublishAsync(responseTopic, currentSettings); - } - catch (Exception ex) - { - await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticAssets] [Settings_Update] Failed to update settings."); + await settingsService.UpdateSettingsAsync(updates); + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticAssets] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count); } + return await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) }); } - private async Task HandleConfigUpdatedAsync(string topic, string payload) + private async Task HandleHealthPingRpcAsync(object? _, string topic, string correlationId) { - if (!topic.EndsWith("FinlyticAssets", StringComparison.OrdinalIgnoreCase)) - return; - - try + if (topic.Contains("FinlyticAssets", StringComparison.OrdinalIgnoreCase)) { - var updatePayload = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.ServiceConfigUpdatePayload); - if (updatePayload?.Settings != null && updatePayload.Settings.Count > 0) - { - using var scope = _scopeFactory.CreateScope(); - var settings = scope.ServiceProvider.GetRequiredService(); - var dict = updatePayload.Settings.ToDictionary(k => k.Key, v => (object?)v.Value); - await settings.UpdateSettingsAsync(dict); - } - } - catch (Exception ex) - { - using var scope = _scopeFactory.CreateScope(); - var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); - await finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsMqttClient] Error processing MQTT config update event."); - } - } - - private async Task HandleHealthPingAsync(string topic, string[] segments, string correlationId) - { - bool isForMe = segments.Length >= 5 - ? segments[3].Equals("FinlyticAssets", StringComparison.OrdinalIgnoreCase) - : topic.Contains("FinlyticAssets", StringComparison.OrdinalIgnoreCase); - - if (isForMe) - { - string respTopic = $"services/response/health_Ping/{correlationId}"; + string respTopic = MqttTopics.ResponseTopic(MqttTopics.Channels.HealthPing, correlationId); await PublishAsync(respTopic, new FinlyticCore.Dtos.ServiceHealthResponse("FinlyticAssets", "Online", DateTime.UtcNow, "Connected")); using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); - await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[AssetsMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId); + await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[AssetsMqttClient] Responded to health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId); } } - - private async Task HandleFetchLogoAsync(string payload, string correlationId, IAssetsIndexService indexService) - { - var req = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.IsinRequest); - string? isin = req?.Isin; - string? savedPath = null; - - if (!string.IsNullOrEmpty(isin)) - { - savedPath = await indexService.DownloadAndSaveLogoAsync(isin); - } - - string responseTopic = $"services/response/assets_FetchLogo/{correlationId}"; - await PublishAsync(responseTopic, new FinlyticCore.Dtos.FetchLogoResponse(isin, savedPath, savedPath != null)); - } - - private async Task> HandleAssetsGetAsync(string payload, IAssetsDbService dbService) - { - var validReq = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.GetValidAssetRequest); - if (validReq != null) - { - return await dbService.GetValidAssetsByIsinAsync(validReq.Isin); - } - return []; - } - - private async Task> HandleAssetsSearchAsync(string payload, IAssetsDbService dbService) - { - var searchReq = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.SearchAssetsRequest); - if (searchReq != null) - { - return await dbService.FindAffectedActiveAssetsAsync(searchReq.SearchQuery); - } - return []; - } - - private async Task> HandleAssetsGetDiscoveryAsync(string payload, IAssetsDbService dbService) - { - int limit = 15; - if (!string.IsNullOrWhiteSpace(payload)) - { - try - { - var discReq = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.GetDiscoveryAssetsRequest); - if (discReq != null && discReq.Limit > 0) limit = discReq.Limit; - } - catch { } - } - return await dbService.GetDiscoveryAssetsAsync(limit); - } - - private async Task> HandleAssetsGetDerivativesAsync(string payload, IAssetsDbService dbService) - { - if (string.IsNullOrWhiteSpace(payload)) return []; - - try - { - var req = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.GetDerivativesRequest); - if (req != null && !string.IsNullOrEmpty(req.UnderlyingIsin)) - { - return await dbService.GetDerivativesByUnderlyingAsync( - req.UnderlyingIsin, - req.OptionType, - req.TargetLeverage, - req.After, - req.Page, - req.ShouldForceRefresh); - } - } - catch (Exception ex) - { - using var scope = _scopeFactory.CreateScope(); - var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); - await finlyticLogger.LogErrorAsync(SettingKeys.AssetsChannel, ex, "[AssetsMqttClient] Error parsing GetDerivativesRequest payload."); - } - - return []; - } } diff --git a/FinlyticAssets/Util/SettingKeys.cs b/FinlyticAssets/Util/SettingKeys.cs index e0794be..d3cdda3 100644 --- a/FinlyticAssets/Util/SettingKeys.cs +++ b/FinlyticAssets/Util/SettingKeys.cs @@ -8,15 +8,21 @@ public static class SettingKeys public static readonly SettingKey AssetsChannel = new("Logging.Channel.Assets", true); public static readonly SettingKey MqttChannel = new("Logging.Channel.MQTT", true); public static readonly SettingKey HealthPingChannel = new("Logging.Channel.Health", true); + public static readonly SettingKey TradeRepublicChannel = new("Logging.Channel.TradeRepublic", true); - // --- Asset Scanning --- - public static readonly SettingKey EnableAutoScan = new("Scanner.EnableAutoScan", true); - public static readonly SettingKey ScanIntervalHours = new("Scanner.ScanIntervalHours", 12); - public static readonly SettingKey MaxConcurrentScans = new("Scanner.MaxConcurrentScans", 5); - public static readonly SettingKey EnableDerivativeScanning = new("Scanner.EnableDerivativeScanning", true); + // --- Trade Republic Connection --- + public static readonly SettingKey TradeRepublicWsReconnectInterval = new("TradeRepublic.WsReconnectIntervalSeconds", 5); + public static readonly SettingKey TradeRepublicWsTimeout = new("TradeRepublic.WsTimeoutSeconds", 15); - // --- Logos & Media --- - public static readonly SettingKey AutoFetchLogos = new("Media.AutoFetchLogos", true); - public static readonly SettingKey LogoFetchBatchSize = new("Media.LogoFetchBatchSize", 25); - public static readonly SettingKey LogoStorageDirectory = new("Media.LogoStorageDirectory", "data/logos"); + // --- Asset Scanning Config --- + public static readonly SettingKey ScannerEnableAutoScan = new("Scanner.EnableAutoScan", true); + public static readonly SettingKey ScannerCurrentScanningType = new("Scanner.CurrentScanningType", "Stock"); + public static readonly SettingKey ScannerCurrentScanningPage = new("Scanner.CurrentScanningPage", 0); + public static readonly SettingKey ScannerFinishedInitialScan = new("Scanner.FinishedInitialScan", false); + public static readonly SettingKey ScannerBatchDelay = new("Scanner.BatchAssetUpdateDelay", 0); + public static readonly SettingKey ScannerTypeDelay = new("Scanner.AssetUpdateTypeDelay", 0); + public static readonly SettingKey ScannerInitBatchDelay = new("Scanner.InitBatchAssetUpdateDelay", 0); + public static readonly SettingKey ScannerInitTypeDelay = new("Scanner.InitAssetUpdateTypeDelay", 0); + public static readonly SettingKey ScannerMaxPageSize = new("Scanner.TradeRepublicMaxRequestPageSize", 50); + public static readonly SettingKey ScannerCycleDelayMinutes = new("Scanner.CycleDelayMinutes", 1440); } diff --git a/FinlyticAssets/Util/StringCodeGenerator.cs b/FinlyticAssets/Util/StringCodeGenerator.cs deleted file mode 100644 index 818ccd5..0000000 --- a/FinlyticAssets/Util/StringCodeGenerator.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace FinlyticAssets.Util; - -public class StringCodeGenerator -{ - - /// - /// Generates a W3C traceparent string for telemetry tracking. - /// - public static string GenerateTraceparent() - { - var traceId = Guid.NewGuid().ToString("N"); - var spanId = Guid.NewGuid().ToString("N").Substring(0, 16); - - return $"00-{traceId}-{spanId}-01"; - } - -}