diff --git a/FinlyticAssets/Database/AssetsDbContext.cs b/FinlyticAssets/Database/AssetsDbContext.cs index d1e59cf..d755bce 100644 --- a/FinlyticAssets/Database/AssetsDbContext.cs +++ b/FinlyticAssets/Database/AssetsDbContext.cs @@ -22,6 +22,7 @@ public class AssetsDbContext : DbContext modelBuilder.Entity(entity => { entity.HasKey(e => new {e.Isin, e.InstrumentCategory}); + entity.HasIndex(e => e.LastUpdatedAt); entity.HasDiscriminator("AssetType") .HasValue("Stock") @@ -76,4 +77,4 @@ public class AssetsDbContext : DbContext .HasMany(a => a.Tags) .WithMany(t => t.Assets); } -} \ No newline at end of file +} diff --git a/FinlyticAssets/Dockerfile b/FinlyticAssets/Dockerfile index 7f498f5..dc732a7 100644 --- a/FinlyticAssets/Dockerfile +++ b/FinlyticAssets/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/runtime:10.0 AS base +FROM mcr.microsoft.com/dotnet/runtime:10.0 AS base USER $APP_UID WORKDIR /app diff --git a/FinlyticAssets/Entities/Settings.cs b/FinlyticAssets/Entities/Settings.cs index 3a1f609..cefd688 100644 --- a/FinlyticAssets/Entities/Settings.cs +++ b/FinlyticAssets/Entities/Settings.cs @@ -22,27 +22,7 @@ public class Settings /// public bool FinishedInitialScan { get; set; } - /// - /// Gets or sets the minimum number of days to wait before an asset becomes eligible for re-validation. - /// Combined with to achieve a flat 2-to-3-month rotation cycle. - /// - public int MinRandomUpdateDay { get; set; } = 60; - /// - /// Gets or sets the maximum number of days to wait before an asset must be re-validated (roughly 3 months). - /// - public int MaxRandomUpdateDay { get; set; } = 90; - - /// - /// Gets or sets the earliest hour (0-23) of the day when background synchronization is allowed to execute. - /// Prevents unusual nocturnal API traffic. - /// - public int UpdateDayTimeStart { get; set; } = 8; - - /// - /// Gets or sets the latest hour (0-23) of the day when background synchronization is allowed to execute. - /// - public int UpdateDayTimeStop { get; set; } = 21; /// /// Gets or sets the maximum number of assets requested per single API pagination call. @@ -58,9 +38,9 @@ public class Settings /// /// Gets or sets the idle delay in seconds between switching asset categories during the initial setup scan. - /// Faster than standard mode but kept high enough to prevent early rate limiting. (Default 3600s = 1 hour). + /// Set to 0 to move immediately to the next type after finishing the current one. (Default 0s). /// - public int InitAssetUpdateTypeDelay { get; set; } = 3600; + public int InitAssetUpdateTypeDelay { get; set; } = 0; /// /// Gets or sets the baseline delay in seconds between sequential page requests of the same asset type during the initial setup scan. @@ -85,4 +65,4 @@ public class Settings /// Trade Republic uses 1-based pagination. A value of 0 means the scan is currently idle or between types. /// public int CurrentScanningPage { get; set; } = 0; -} \ No newline at end of file +} diff --git a/FinlyticAssets/FinlyticAssets.csproj b/FinlyticAssets/FinlyticAssets.csproj index e89e094..58304c0 100644 --- a/FinlyticAssets/FinlyticAssets.csproj +++ b/FinlyticAssets/FinlyticAssets.csproj @@ -34,4 +34,8 @@ + + + <_ContentIncludedByDefault Remove="assets\index\index.json" /> + diff --git a/FinlyticAssets/Migrations/20260628184437_Init.Designer.cs b/FinlyticAssets/Migrations/20260628184437_Init.Designer.cs deleted file mode 100644 index a89b6a3..0000000 --- a/FinlyticAssets/Migrations/20260628184437_Init.Designer.cs +++ /dev/null @@ -1,285 +0,0 @@ -// -using System; -using FinlyticAssets.Database; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - [DbContext(typeof(AssetsDbContext))] - [Migration("20260628184437_Init")] - partial class Init - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.Property("AssetsIsin") - .HasColumnType("text"); - - b.Property("TagsId") - .HasColumnType("text"); - - b.HasKey("AssetsIsin", "TagsId"); - - b.HasIndex("TagsId"); - - b.ToTable("AssetEntityTagEntity"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); - - b.Property("InitAssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("MaxRandomUpdateDay") - .HasColumnType("integer"); - - b.Property("MinRandomUpdateDay") - .HasColumnType("integer"); - - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); - - b.Property("UpdateDayTimeStart") - .HasColumnType("integer"); - - b.Property("UpdateDayTimeStop") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => - { - b.Property("Isin") - .HasColumnType("text"); - - b.Property("AssetType") - .IsRequired() - .HasMaxLength(13) - .HasColumnType("character varying(13)"); - - b.Property("HasCfd") - .HasColumnType("boolean"); - - b.Property("ImageId") - .HasColumnType("text"); - - b.Property("InstrumentCategory") - .IsRequired() - .HasColumnType("text"); - - b.Property("LastUpdatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.Property("UpdateAt") - .HasColumnType("timestamp with time zone"); - - b.HasKey("Isin"); - - b.ToTable("TradeRepublicAssets"); - - b.HasDiscriminator("AssetType").HasValue("AssetEntity"); - - b.UseTphMappingStrategy(); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("TradeRepublicTags"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("UnderlyingIsin") - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Derivative"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("EtfDescription") - .IsRequired() - .HasColumnType("text"); - - b.Property("MappedEtfIndexName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - - b.HasDiscriminator().HasValue("Etf"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("StockEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Stock"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("SyntheticEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Synthetic"); - }); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) - .WithMany() - .HasForeignKey("AssetsIsin") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) - .WithMany() - .HasForeignKey("TagsId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs b/FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs deleted file mode 100644 index f9d85d2..0000000 --- a/FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs +++ /dev/null @@ -1,293 +0,0 @@ -// -using System; -using FinlyticAssets.Database; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - [DbContext(typeof(AssetsDbContext))] - [Migration("20260628195842_AddedScannerState")] - partial class AddedScannerState - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.9") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.Property("AssetsIsin") - .HasColumnType("text"); - - b.Property("TagsId") - .HasColumnType("text"); - - b.HasKey("AssetsIsin", "TagsId"); - - b.HasIndex("TagsId"); - - b.ToTable("AssetEntityTagEntity"); - }); - - modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("AssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("BatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("CurrentScanningPage") - .HasColumnType("integer"); - - b.Property("CurrentScanningType") - .IsRequired() - .HasMaxLength(50) - .HasColumnType("character varying(50)"); - - b.Property("FinishedInitialScan") - .HasColumnType("boolean"); - - b.Property("InitAssetUpdateTypeDelay") - .HasColumnType("integer"); - - b.Property("InitBatchAssetUpdateDelay") - .HasColumnType("integer"); - - b.Property("MaxRandomUpdateDay") - .HasColumnType("integer"); - - b.Property("MinRandomUpdateDay") - .HasColumnType("integer"); - - b.Property("TradeRepublicMaxRequestPageSize") - .HasColumnType("integer"); - - b.Property("UpdateDayTimeStart") - .HasColumnType("integer"); - - b.Property("UpdateDayTimeStop") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => - { - b.Property("Isin") - .HasColumnType("text"); - - b.Property("AssetType") - .IsRequired() - .HasMaxLength(13) - .HasColumnType("character varying(13)"); - - b.Property("HasCfd") - .HasColumnType("boolean"); - - b.Property("ImageId") - .HasColumnType("text"); - - b.Property("InstrumentCategory") - .IsRequired() - .HasColumnType("text"); - - b.Property("LastUpdatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.Property("UpdateAt") - .HasColumnType("timestamp with time zone"); - - b.HasKey("Isin"); - - b.ToTable("TradeRepublicAssets"); - - b.HasDiscriminator("AssetType").HasValue("AssetEntity"); - - b.UseTphMappingStrategy(); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("TradeRepublicTags"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("BondIssuerName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Bond"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("SearchSubtitle") - .HasColumnName("CryptoEntity_SearchSubtitle"); - }); - - b.HasDiscriminator().HasValue("Crypto"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("UnderlyingIsin") - .HasColumnType("text"); - - b.HasDiscriminator().HasValue("Derivative"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.Property("EtfDescription") - .IsRequired() - .HasColumnType("text"); - - b.Property("MappedEtfIndexName") - .IsRequired() - .HasColumnType("text"); - - b.Property("SearchSubtitle") - .IsRequired() - .HasColumnType("text"); - - b.Property("Subtitle") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("EtfEntity_DerivativeProductCategories"); - - t.Property("SearchSubtitle") - .HasColumnName("EtfEntity_SearchSubtitle"); - - t.Property("Subtitle") - .HasColumnName("EtfEntity_Subtitle"); - }); - - b.HasDiscriminator().HasValue("Etf"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("StockEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Stock"); - }); - - modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => - { - b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); - - b.Property("DerivativeProductCategories") - .IsRequired() - .HasColumnType("text"); - - b.ToTable("TradeRepublicAssets", t => - { - t.Property("DerivativeProductCategories") - .HasColumnName("SyntheticEntity_DerivativeProductCategories"); - }); - - b.HasDiscriminator().HasValue("Synthetic"); - }); - - modelBuilder.Entity("AssetEntityTagEntity", b => - { - b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) - .WithMany() - .HasForeignKey("AssetsIsin") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) - .WithMany() - .HasForeignKey("TagsId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs b/FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs deleted file mode 100644 index fec561b..0000000 --- a/FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - /// - public partial class AddedScannerState : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AddColumn( - name: "CurrentScanningPage", - table: "Settings", - type: "integer", - nullable: false, - defaultValue: 0); - - migrationBuilder.AddColumn( - name: "CurrentScanningType", - table: "Settings", - type: "character varying(50)", - maxLength: 50, - nullable: false, - defaultValue: ""); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropColumn( - name: "CurrentScanningPage", - table: "Settings"); - - migrationBuilder.DropColumn( - name: "CurrentScanningType", - table: "Settings"); - } - } -} diff --git a/FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs b/FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs deleted file mode 100644 index eb123c2..0000000 --- a/FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs +++ /dev/null @@ -1,107 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace FinlyticAssets.Migrations -{ - /// - public partial class FixIsinKey : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropForeignKey( - name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin", - table: "AssetEntityTagEntity"); - - migrationBuilder.DropPrimaryKey( - name: "PK_TradeRepublicAssets", - table: "TradeRepublicAssets"); - - migrationBuilder.DropPrimaryKey( - name: "PK_AssetEntityTagEntity", - table: "AssetEntityTagEntity"); - - migrationBuilder.DropIndex( - name: "IX_AssetEntityTagEntity_TagsId", - table: "AssetEntityTagEntity"); - - migrationBuilder.AddColumn( - name: "AssetsInstrumentCategory", - table: "AssetEntityTagEntity", - type: "text", - nullable: false, - defaultValue: ""); - - migrationBuilder.AddPrimaryKey( - name: "PK_TradeRepublicAssets", - table: "TradeRepublicAssets", - columns: new[] { "Isin", "InstrumentCategory" }); - - migrationBuilder.AddPrimaryKey( - name: "PK_AssetEntityTagEntity", - table: "AssetEntityTagEntity", - columns: new[] { "TagsId", "AssetsIsin", "AssetsInstrumentCategory" }); - - migrationBuilder.CreateIndex( - name: "IX_AssetEntityTagEntity_AssetsIsin_AssetsInstrumentCategory", - table: "AssetEntityTagEntity", - columns: new[] { "AssetsIsin", "AssetsInstrumentCategory" }); - - migrationBuilder.AddForeignKey( - name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin_AssetsI~", - table: "AssetEntityTagEntity", - columns: new[] { "AssetsIsin", "AssetsInstrumentCategory" }, - principalTable: "TradeRepublicAssets", - principalColumns: new[] { "Isin", "InstrumentCategory" }, - onDelete: ReferentialAction.Cascade); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropForeignKey( - name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin_AssetsI~", - table: "AssetEntityTagEntity"); - - migrationBuilder.DropPrimaryKey( - name: "PK_TradeRepublicAssets", - table: "TradeRepublicAssets"); - - migrationBuilder.DropPrimaryKey( - name: "PK_AssetEntityTagEntity", - table: "AssetEntityTagEntity"); - - migrationBuilder.DropIndex( - name: "IX_AssetEntityTagEntity_AssetsIsin_AssetsInstrumentCategory", - table: "AssetEntityTagEntity"); - - migrationBuilder.DropColumn( - name: "AssetsInstrumentCategory", - table: "AssetEntityTagEntity"); - - migrationBuilder.AddPrimaryKey( - name: "PK_TradeRepublicAssets", - table: "TradeRepublicAssets", - column: "Isin"); - - migrationBuilder.AddPrimaryKey( - name: "PK_AssetEntityTagEntity", - table: "AssetEntityTagEntity", - columns: new[] { "AssetsIsin", "TagsId" }); - - migrationBuilder.CreateIndex( - name: "IX_AssetEntityTagEntity_TagsId", - table: "AssetEntityTagEntity", - column: "TagsId"); - - migrationBuilder.AddForeignKey( - name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin", - table: "AssetEntityTagEntity", - column: "AssetsIsin", - principalTable: "TradeRepublicAssets", - principalColumn: "Isin", - onDelete: ReferentialAction.Cascade); - } - } -} diff --git a/FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs b/FinlyticAssets/Migrations/20260801073314_Init.Designer.cs similarity index 93% rename from FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs rename to FinlyticAssets/Migrations/20260801073314_Init.Designer.cs index 79fb20b..4f1cca7 100644 --- a/FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs +++ b/FinlyticAssets/Migrations/20260801073314_Init.Designer.cs @@ -1,4 +1,4 @@ -// +// using System; using FinlyticAssets.Database; using Microsoft.EntityFrameworkCore; @@ -12,8 +12,8 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace FinlyticAssets.Migrations { [DbContext(typeof(AssetsDbContext))] - [Migration("20260628204647_FixIsinKey")] - partial class FixIsinKey + [Migration("20260801073314_Init")] + partial class Init { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -72,21 +72,9 @@ namespace FinlyticAssets.Migrations b.Property("InitBatchAssetUpdateDelay") .HasColumnType("integer"); - b.Property("MaxRandomUpdateDay") - .HasColumnType("integer"); - - b.Property("MinRandomUpdateDay") - .HasColumnType("integer"); - b.Property("TradeRepublicMaxRequestPageSize") .HasColumnType("integer"); - b.Property("UpdateDayTimeStart") - .HasColumnType("integer"); - - b.Property("UpdateDayTimeStop") - .HasColumnType("integer"); - b.HasKey("Id"); b.ToTable("Settings"); @@ -122,11 +110,10 @@ namespace FinlyticAssets.Migrations .IsRequired() .HasColumnType("text"); - b.Property("UpdateAt") - .HasColumnType("timestamp with time zone"); - b.HasKey("Isin", "InstrumentCategory"); + b.HasIndex("LastUpdatedAt"); + b.ToTable("TradeRepublicAssets"); b.HasDiscriminator("AssetType").HasValue("AssetEntity"); diff --git a/FinlyticAssets/Migrations/20260628184437_Init.cs b/FinlyticAssets/Migrations/20260801073314_Init.cs similarity index 84% rename from FinlyticAssets/Migrations/20260628184437_Init.cs rename to FinlyticAssets/Migrations/20260801073314_Init.cs index 8edc236..813eb3b 100644 --- a/FinlyticAssets/Migrations/20260628184437_Init.cs +++ b/FinlyticAssets/Migrations/20260801073314_Init.cs @@ -1,4 +1,4 @@ -using System; +using System; using Microsoft.EntityFrameworkCore.Migrations; #nullable disable @@ -17,15 +17,13 @@ namespace FinlyticAssets.Migrations { Id = table.Column(type: "uuid", nullable: false), FinishedInitialScan = table.Column(type: "boolean", nullable: false), - MinRandomUpdateDay = table.Column(type: "integer", nullable: false), - MaxRandomUpdateDay = table.Column(type: "integer", nullable: false), - UpdateDayTimeStart = table.Column(type: "integer", nullable: false), - UpdateDayTimeStop = table.Column(type: "integer", nullable: false), TradeRepublicMaxRequestPageSize = table.Column(type: "integer", nullable: false), AssetUpdateTypeDelay = table.Column(type: "integer", nullable: false), InitAssetUpdateTypeDelay = table.Column(type: "integer", nullable: false), InitBatchAssetUpdateDelay = table.Column(type: "integer", nullable: false), - BatchAssetUpdateDelay = table.Column(type: "integer", nullable: false) + BatchAssetUpdateDelay = table.Column(type: "integer", nullable: false), + CurrentScanningType = table.Column(type: "character varying(50)", maxLength: 50, nullable: false), + CurrentScanningPage = table.Column(type: "integer", nullable: false) }, constraints: table => { @@ -37,12 +35,11 @@ namespace FinlyticAssets.Migrations columns: table => new { Isin = table.Column(type: "text", nullable: false), + InstrumentCategory = table.Column(type: "text", nullable: false), Name = table.Column(type: "text", nullable: false), Type = table.Column(type: "text", nullable: false), - InstrumentCategory = table.Column(type: "text", nullable: false), HasCfd = table.Column(type: "boolean", nullable: false), ImageId = table.Column(type: "text", nullable: true), - UpdateAt = table.Column(type: "timestamp with time zone", nullable: false), LastUpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), AssetType = table.Column(type: "character varying(13)", maxLength: 13, nullable: false), BondIssuerName = table.Column(type: "text", nullable: true), @@ -61,7 +58,7 @@ namespace FinlyticAssets.Migrations }, constraints: table => { - table.PrimaryKey("PK_TradeRepublicAssets", x => x.Isin); + table.PrimaryKey("PK_TradeRepublicAssets", x => new { x.Isin, x.InstrumentCategory }); }); migrationBuilder.CreateTable( @@ -81,17 +78,18 @@ namespace FinlyticAssets.Migrations name: "AssetEntityTagEntity", columns: table => new { + TagsId = table.Column(type: "text", nullable: false), AssetsIsin = table.Column(type: "text", nullable: false), - TagsId = table.Column(type: "text", nullable: false) + AssetsInstrumentCategory = table.Column(type: "text", nullable: false) }, constraints: table => { - table.PrimaryKey("PK_AssetEntityTagEntity", x => new { x.AssetsIsin, x.TagsId }); + table.PrimaryKey("PK_AssetEntityTagEntity", x => new { x.TagsId, x.AssetsIsin, x.AssetsInstrumentCategory }); table.ForeignKey( - name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin", - column: x => x.AssetsIsin, + name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin_AssetsI~", + columns: x => new { x.AssetsIsin, x.AssetsInstrumentCategory }, principalTable: "TradeRepublicAssets", - principalColumn: "Isin", + principalColumns: new[] { "Isin", "InstrumentCategory" }, onDelete: ReferentialAction.Cascade); table.ForeignKey( name: "FK_AssetEntityTagEntity_TradeRepublicTags_TagsId", @@ -102,9 +100,14 @@ namespace FinlyticAssets.Migrations }); migrationBuilder.CreateIndex( - name: "IX_AssetEntityTagEntity_TagsId", + name: "IX_AssetEntityTagEntity_AssetsIsin_AssetsInstrumentCategory", table: "AssetEntityTagEntity", - column: "TagsId"); + columns: new[] { "AssetsIsin", "AssetsInstrumentCategory" }); + + migrationBuilder.CreateIndex( + name: "IX_TradeRepublicAssets_LastUpdatedAt", + table: "TradeRepublicAssets", + column: "LastUpdatedAt"); } /// diff --git a/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs b/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs index b2c32be..453a7c6 100644 --- a/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs +++ b/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs @@ -1,4 +1,4 @@ -// +// using System; using FinlyticAssets.Database; using Microsoft.EntityFrameworkCore; @@ -69,21 +69,9 @@ namespace FinlyticAssets.Migrations b.Property("InitBatchAssetUpdateDelay") .HasColumnType("integer"); - b.Property("MaxRandomUpdateDay") - .HasColumnType("integer"); - - b.Property("MinRandomUpdateDay") - .HasColumnType("integer"); - b.Property("TradeRepublicMaxRequestPageSize") .HasColumnType("integer"); - b.Property("UpdateDayTimeStart") - .HasColumnType("integer"); - - b.Property("UpdateDayTimeStop") - .HasColumnType("integer"); - b.HasKey("Id"); b.ToTable("Settings"); @@ -119,11 +107,10 @@ namespace FinlyticAssets.Migrations .IsRequired() .HasColumnType("text"); - b.Property("UpdateAt") - .HasColumnType("timestamp with time zone"); - b.HasKey("Isin", "InstrumentCategory"); + b.HasIndex("LastUpdatedAt"); + b.ToTable("TradeRepublicAssets"); b.HasDiscriminator("AssetType").HasValue("AssetEntity"); diff --git a/FinlyticAssets/Models/AssetIndex.cs b/FinlyticAssets/Models/AssetIndex.cs deleted file mode 100644 index 60df083..0000000 --- a/FinlyticAssets/Models/AssetIndex.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FinlyticAssets.Models; - -public record AssetIndex( - [property: JsonPropertyName("isin")] string Isin, - [property: JsonPropertyName("name")] string Name); \ No newline at end of file diff --git a/FinlyticAssets/Models/AssetsCount.cs b/FinlyticAssets/Models/AssetsCount.cs deleted file mode 100644 index b4b741d..0000000 --- a/FinlyticAssets/Models/AssetsCount.cs +++ /dev/null @@ -1,39 +0,0 @@ -using FinlyticCore.Models.Assets; - -namespace FinlyticAssets.Models; - -public class AssetsCount -{ - public int Stock { get; set; } - public int Funds { get; set; } - public int Derivatives { get; set; } - public int Crypto { get; set; } - public int Bond { get; set; } - - public int GetCountFromType(AssetType type) - { - return type switch - { - AssetType.Stock => Stock, - AssetType.Fund => Funds, - AssetType.Derivative => Derivatives, - AssetType.Crypto => Crypto, - AssetType.Bond => Bond, - _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) - }; - } - - - public void SetCountOfType(AssetType type, int count) - { - _ = type switch - { - AssetType.Stock => Stock = count, - AssetType.Fund => Funds = count, - AssetType.Derivative => Derivatives = count, - AssetType.Crypto => Crypto = count, - AssetType.Bond => Bond = count, - _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) - }; - } -} \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs deleted file mode 100644 index b92f683..0000000 --- a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs +++ /dev/null @@ -1,139 +0,0 @@ -namespace FinlyticAssets.Models.DataToObject.TradeRepublic; - -using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Text.Json.Serialization; - -// 1. Der Response-Wrapper -public record TradeRepublicAssetResponse( - [property: JsonPropertyName("correlationId")] string CorrelationId, - [property: JsonPropertyName("resultCount")] int ResultCount, - [property: JsonPropertyName("results")] IList Results -); - -// 2. Das Tag-Objekt -public record TradeRepublicTag -{ - [JsonPropertyName("id")] public string Id { get; init; } = ""; - [JsonPropertyName("name")] public string Name { get; init; } = ""; - [JsonPropertyName("type")] public string Type { get; init; } = ""; -} - -// 3. Die Basisklasse MIT UNSEREM CUSTOM CONVERTER (Kein [JsonPolymorphic] mehr!) -[JsonConverter(typeof(TradeRepublicAssetConverter))] -public record TradeRepublicAsset -{ - [JsonPropertyName("isin")] public string Isin { get; init; } = ""; - [JsonPropertyName("name")] public string Name { get; init; } = ""; - [JsonPropertyName("type")] public string Type { get; init; } = ""; - [JsonPropertyName("instrumentCategory")] public string InstrumentCategory { get; init; } = ""; - [JsonPropertyName("hasCfd")] public bool HasCfd { get; init; } - [JsonPropertyName("imageId")] public string? ImageId { get; init; } - - [JsonPropertyName("tags")] - public IReadOnlyList Tags { get; init; } = Array.Empty(); -} - -// 4. Die spezifischen Klassen (inklusive Bond und Derivative aus deinem JSON!) - -public record TradeRepublicStock : TradeRepublicAsset -{ - [JsonPropertyName("derivativeProductCategories")] - public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); -} - -public record TradeRepublicCrypto : TradeRepublicAsset -{ - [JsonPropertyName("subtitle")] public string Subtitle { get; init; } = ""; - [JsonPropertyName("searchSubtitle")] public string SearchSubtitle { get; init; } = ""; -} - -public record TradeRepublicEtf : TradeRepublicAsset -{ - [JsonPropertyName("derivativeProductCategories")] - public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); - [JsonPropertyName("etfDescription")] public string EtfDescription { get; init; } = ""; - [JsonPropertyName("mappedEtfIndexName")] public string MappedEtfIndexName { get; init; } = ""; - [JsonPropertyName("subtitle")] public string Subtitle { get; init; } = ""; - [JsonPropertyName("searchSubtitle")] public string SearchSubtitle { get; init; } = ""; -} - -public record TradeRepublicSynthetic : TradeRepublicAsset -{ - [JsonPropertyName("derivativeProductCategories")] - public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); -} - -// NEU: Anleihen -public record TradeRepublicBond : TradeRepublicAsset -{ - [JsonPropertyName("bondIssuerName")] public string BondIssuerName { get; init; } = ""; - [JsonPropertyName("searchSubtitle")] public string SearchSubtitle { get; init; } = ""; -} - -// NEU: Derivate (Hebeleffekte etc.) -public record TradeRepublicDerivative : TradeRepublicAsset -{ - [JsonPropertyName("derivativeProductCategories")] - public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); - - [JsonIgnore] - public string? UnderlyingIsin - { - get - { - // Wenn die ImageId z.B. "logos/US0378331005/v2" ist... - if (!string.IsNullOrEmpty(ImageId) && ImageId.StartsWith("logos/")) - { - var parts = ImageId.Split('/'); - if (parts.Length >= 2) - { - return parts[1]; // Gibt "US0378331005" zurück - } - } - return null; // Falls das Format mal anders ist - } - } -} - -// 5. Der Custom Converter - Die Maschine, die das JSON scannt und verteilt -public class TradeRepublicAssetConverter : JsonConverter -{ - public override TradeRepublicAsset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - using var doc = JsonDocument.ParseValue(ref reader); - var root = doc.RootElement; - - // Wir scannen nach instrumentType, egal wo im JSON es steht! - string? instrumentType = null; - if (root.TryGetProperty("instrumentType", out var typeElement)) - { - instrumentType = typeElement.GetString(); - } - - // Wir werfen das JSON gezielt in die richtige Klasse - TradeRepublicAsset? result = instrumentType switch - { - "stock" => JsonSerializer.Deserialize(root.GetRawText(), options), - "crypto" => JsonSerializer.Deserialize(root.GetRawText(), options), - "fund" => JsonSerializer.Deserialize(root.GetRawText(), options), - "synthetic" => JsonSerializer.Deserialize(root.GetRawText(), options), - "bond" => JsonSerializer.Deserialize(root.GetRawText(), options), - "derivative" => JsonSerializer.Deserialize(root.GetRawText(), options), - - // Wenn TR einen Typ schickt, den wir noch nicht kennen: Fallback nutzen! - _ => JsonSerializer.Deserialize(root.GetRawText(), options) - }; - - return result ?? new TradeRepublicAssetFallback(); - } - - public override void Write(Utf8JsonWriter writer, TradeRepublicAsset value, JsonSerializerOptions options) - { - JsonSerializer.Serialize(writer, value, value.GetType(), options); - } -} - -// Ein reiner Fallback-Record, der nur intern vom Converter genutzt wird -file record TradeRepublicAssetFallback : TradeRepublicAsset; \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs deleted file mode 100644 index 4b38484..0000000 --- a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.Text.Json.Serialization; - -namespace FinlyticAssets.Models.DataToObject.TradeRepublic; - -public record TradeRepublicConnectRequest( - [property: JsonPropertyName("clientId")] string ClientId = "app.traderepublic.com", - [property: JsonPropertyName("clientVersion")] string ClientVersion = "15.65.6", - [property: JsonPropertyName("locale")] string Locale = "en", - [property: JsonPropertyName("platformId")] string PlatformId = "webtrading", - [property: JsonPropertyName("platformVersion")] string PlatformVersion = "chrome - 149.0.0", - TradeRepublicHeaders? Headers = null -) -{ - [JsonPropertyName("__headers")] - public TradeRepublicHeaders Headers { get; init; } = Headers ?? new TradeRepublicHeaders(); -} \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs deleted file mode 100644 index 4923424..0000000 --- a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Text.Json.Serialization; -using FinlyticAssets.Util; - -namespace FinlyticAssets.Models.DataToObject.TradeRepublic; - -public record TradeRepublicHeaders( - [property: JsonPropertyName("traceparent")] string Traceparent -) -{ - public TradeRepublicHeaders() : this(StringCodeGenerator.GenerateTraceparent()) - {} -} \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs deleted file mode 100644 index fa71349..0000000 --- a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Text.Json.Serialization; -using FinlyticAssets.Util; - -namespace FinlyticAssets.Models.DataToObject.TradeRepublic; - -public record TradeRepublicFilter( - [property: JsonPropertyName("key")] string Key, - [property: JsonPropertyName("value")] string Value -); - -public record TradeRepublicSearchData( - [property: JsonPropertyName("q")] string Query = "", - [property: JsonPropertyName("page")] int Page = 1, - [property: JsonPropertyName("pageSize")] int PageSize = 50, - IReadOnlyList? Filter = null -) -{ - [JsonPropertyName("filter")] - public IReadOnlyList Filter { get; init; } = Filter ?? Array.Empty(); -} - -public record TradeRepublicSearchRequest( - [property: JsonPropertyName("data")] TradeRepublicSearchData Data, - [property: JsonPropertyName("type")] string Type = "neonSearch", - TradeRepublicHeaders? Headers = null -) -{ - [JsonPropertyName("__headers")] - public TradeRepublicHeaders Headers { get; init; } = Headers ?? new TradeRepublicHeaders(); -} \ No newline at end of file diff --git a/FinlyticAssets/Program.cs b/FinlyticAssets/Program.cs index 6442ea0..599d7ef 100644 --- a/FinlyticAssets/Program.cs +++ b/FinlyticAssets/Program.cs @@ -1,9 +1,10 @@ using System.Text.Json; using FinlyticAssets; using FinlyticAssets.Database; -using FinlyticAssets.Models.DataToObject.TradeRepublic; -using FinlyticAssets.Services; +using FinlyticCore.Services.TradeRepublic; using FinlyticAssets.Util; +using FinlyticAssets.Services; +using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; var builder = Host.CreateApplicationBuilder(args); @@ -19,10 +20,10 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); -builder.Services.AddHostedService(); - -builder.Services.AddHostedService(); +builder.Services.AddHostedService(); +builder.Services.AddHostedService(); var host = builder.Build(); @@ -32,6 +33,17 @@ using (var scope = host.Services.CreateScope()) { var context = scope.ServiceProvider.GetRequiredService(); await context.Database.MigrateAsync(); + + // Fix: Reset InitAssetUpdateTypeDelay from old default (3600s) to new default (0s = no delay). + // This ensures the scanner moves immediately to the next asset type during the initial scan. + var settingsService = scope.ServiceProvider.GetRequiredService(); + var settings = await settingsService.GetSettings(); + if (settings.InitAssetUpdateTypeDelay == 3600) + { + settings.InitAssetUpdateTypeDelay = 0; + await settingsService.SaveSettings(settings); + Console.WriteLine("[Startup] InitAssetUpdateTypeDelay reset from 3600 to 0."); + } } catch (Exception ex) { @@ -40,4 +52,4 @@ using (var scope = host.Services.CreateScope()) } } -host.Run(); \ No newline at end of file +host.Run(); diff --git a/FinlyticAssets/Project.md b/FinlyticAssets/Project.md new file mode 100644 index 0000000..d9af5f6 --- /dev/null +++ b/FinlyticAssets/Project.md @@ -0,0 +1,67 @@ +# Finlytic Assets Service + +Finlytic Assets is a scalable C# microservice designed for asset discovery, metadata ingestion, and quick querying. It interfaces directly with the Trade Republic API via WebSockets and exposes a high-performance RPC interface over MQTT to other services in the Finlytic ecosystem. + +--- + +## Architecture Overview + +```mermaid +graph TD + TR[Trade Republic WebSocket API] <-->|WS Protocol| TRS[TradeRepublicService] + TRS <-->|Ingest| ADS[AssetsFullScanService] + ADS <-->|Save / Update| DB[PostgreSQL Database] + DB -->|Trigger Index Update| AIS[AssetsIndexService] + AIS -->|Write Cache| Index[assets/index/index.json] + + MS[Other Finlytic Services] <-->|MQTT Request-Reply| MqttClient[AssetsMqttClient] + MqttClient <-->|Query Cache| DB +``` + +--- + +## Core Features & Workflows + +### 1. Automated WebSocket scraping +- **Continuous Scan**: The background worker (`AssetsFullScanService`) iterates through all supported asset types: + - **Stocks** (`StockEntity`) + - **Crypto** (`CryptoEntity`) + - **Derivatives** (`DerivativeEntity`) + - **Bonds** (`BondEntity`) + - **Funds/ETFs** (`EtfEntity`) +- **Resilient Reconnection**: Uses a robust custom socket wrapper (`TradeRepublicClient`) that closes automatically after 5 minutes of inactivity to mimic human interaction profiles and reconnects dynamically when a request is made. + +### 2. Scanner State Persistence & Crash Recovery +- **Resilient State**: The scan loop continuously persists progress to the database (`Settings` table), recording `CurrentScanningType` and `CurrentScanningPage`. +- **Exit Recovery**: Resumes exactly from the last saved page and asset type upon restart. + +### 3. Stealth Scheduling with Randomized Jitter +- **Dynamic Delays**: Scanner wait times are read dynamically from database configuration (`Settings`). +- **Jitter Offset**: Randomized offsets between batches and asset types to prevent rate limiting. + +### 4. Active vs. Dead Asset Tracking +- **Recency Filter**: Assets updated within 14 days are active. Inactive assets are flagged. + +### 5. MQTT RPC Interface +- **Get Asset by ISIN**: Subscribes to `services/request/assets_Get/#`. +- **Omnibox Search**: Subscribes to `services/request/assets_Search/#`. +- **JIT Fallback**: Performs JIT-lookup directly against the Trade Republic API for unknown ISINs. + +### 6. Local Indexing File (`index.json`) +- **Automatic Regeneration**: Serializes active assets list (`ISIN`, `Name`) to shared volume `assets/index/index.json`. + +--- + +## Feature Status + +### Implemented Features +- [x] Trade Republic WebSocket scanner for Stocks, Crypto, Derivatives, Bonds, ETFs. +- [x] State persistence & crash recovery in PostgreSQL. +- [x] JIT Trade Republic lookup fallback for new ISINs. +- [x] Shared local index file generation (`index.json`). +- [x] Zero-Allocation MQTT RPC handlers. +- [x] Pure Worker Service architecture (No Kestrel HTTP server). + +### Planned Features +- [ ] Real-time WebSocket price tick streaming over MQTT topic `finlytic/assets/ticks/{isin}`. +- [ ] Derivative option-chain Greeks calculator integration. diff --git a/FinlyticAssets/Services/AssetScannerBackgroundService.cs b/FinlyticAssets/Services/AssetScannerBackgroundService.cs new file mode 100644 index 0000000..2ba35e6 --- /dev/null +++ b/FinlyticAssets/Services/AssetScannerBackgroundService.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FinlyticAssets.Models; +using FinlyticCore.Models.Assets; +using FinlyticCore.Models.TradeRepublic; +using FinlyticCore.Services.TradeRepublic; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace FinlyticAssets.Services; + +/// +/// A background service that runs a continuous asset synchronization loop, +/// scanning Trade Republic to retrieve, update, and index all supported asset types. +/// +public class AssetScannerBackgroundService : BackgroundService +{ + private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly ILogger _logger; + + private AssetsCount? _assetsCount; + private AssetsCount? _currAssetsCount; + + public AssetScannerBackgroundService(IServiceScopeFactory serviceScopeFactory, ILogger logger) + { + _serviceScopeFactory = serviceScopeFactory; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("[{Channel}] AssetScannerBackgroundService has started.", "AssetsChannel"); + + try + { + using var scope = _serviceScopeFactory.CreateScope(); + var indexService = scope.ServiceProvider.GetRequiredService(); + _logger.LogInformation("[{Channel}] Building initial asset index on service startup...", "AssetsChannel"); + await indexService.ReCreateIndexFileAsync(stoppingToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "[{Channel}] Failed to build initial asset index on startup. Continuing service execution.", "AssetsChannel"); + } + + do + { + try + { + using var scope = _serviceScopeFactory.CreateScope(); + var tradeRepublicService = scope.ServiceProvider.GetRequiredService(); + var settingsService = scope.ServiceProvider.GetRequiredService(); + var assetsDbService = scope.ServiceProvider.GetRequiredService(); + var indexService = scope.ServiceProvider.GetRequiredService(); + + _logger.LogInformation("[{Channel}] Requesting total asset counts from Trade Republic...", "AssetsChannel"); + _assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken); + _currAssetsCount = new AssetsCount(); + + var initSettings = await settingsService.GetSettings(); + var isRecoveryMode = initSettings.CurrentScanningPage > 0; + + foreach (var type in Enum.GetValues()) + { + if (stoppingToken.IsCancellationRequested) break; + + if (isRecoveryMode) + { + if (type != initSettings.CurrentScanningType) + { + _logger.LogInformation("[{Channel}] Recovery: {AssetType} was already processed. Skipping.", "AssetsChannel", type); + continue; + } + isRecoveryMode = false; + } + else + { + var settings = await settingsService.GetSettings(); + settings.CurrentScanningType = type; + settings.CurrentScanningPage = 0; + await settingsService.SaveSettings(settings); + } + + _logger.LogInformation("[{Channel}] Processing asset type: {AssetType}...", "AssetsChannel", type); + await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken); + + var currentSettings = await settingsService.GetSettings(); + var delaySeconds = currentSettings.FinishedInitialScan + ? currentSettings.AssetUpdateTypeDelay + : currentSettings.InitAssetUpdateTypeDelay; + + if (delaySeconds > 0) + { + var jitter = Random.Shared.Next(0, Math.Min(15, delaySeconds)); + _logger.LogInformation("[{Channel}] Waiting {Delay}s before next asset type ({Type}).", "AssetsChannel", delaySeconds + jitter, type); + await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); + } + } + + var finalSettings = await settingsService.GetSettings(); + finalSettings.CurrentScanningPage = 0; + + if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested) + { + _logger.LogInformation("[{Channel}] Initial scan successfully completed. Switching FinishedInitialScan to true.", "AssetsChannel"); + finalSettings.FinishedInitialScan = true; + } + + await settingsService.SaveSettings(finalSettings); + + _logger.LogInformation("[{Channel}] Full scan cycle completed. Waiting 1 minute before starting the next cycle.", "AssetsChannel"); + await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + } + catch (Exception e) when (!stoppingToken.IsCancellationRequested) + { + _logger.LogError(e, "[{Channel}] An unhandled exception occurred in AssetScannerBackgroundService. Retrying in 10 seconds.", "AssetsChannel"); + try { await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); } catch { /* Ignore */ } + } + } while (!stoppingToken.IsCancellationRequested); + + _logger.LogInformation("[{Channel}] AssetScannerBackgroundService is stopping.", "AssetsChannel"); + } + + private async Task HandleAssetType( + AssetType type, + ITradeRepublicService tradeRepublicService, + ISettingsDbService settingsDbService, + IAssetsDbService assetsDbService, + IAssetsIndexService indexService, + CancellationToken stoppingToken) + { + var totalCount = _assetsCount?.GetCountFromType(type) ?? 0; + if (totalCount == 0) + { + _logger.LogWarning("[{Channel}] No assets found for type {AssetType}.", "AssetsChannel", type); + return; + } + + _currAssetsCount ??= new AssetsCount(); + var currentItemOffset = 0; + + var settings = await settingsDbService.GetSettings(); + var pageSize = Math.Clamp(settings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : settings.TradeRepublicMaxRequestPageSize, 1, 100); + + if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0) + { + currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize; + _logger.LogInformation("[{Channel}] Resuming full scan for {AssetType} from Page {Page} (Calculated Offset: {Offset}).", + "AssetsChannel", type, settings.CurrentScanningPage, currentItemOffset); + } + + while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested) + { + var currentSettings = await settingsDbService.GetSettings(); + pageSize = Math.Clamp(currentSettings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : currentSettings.TradeRepublicMaxRequestPageSize, 1, 100); + + var currentPage = (currentItemOffset / pageSize) + 1; + + currentSettings.CurrentScanningType = type; + currentSettings.CurrentScanningPage = currentPage; + await settingsDbService.SaveSettings(currentSettings); + + _logger.LogDebug("Fetching {AssetType} - Page {Page}. Numerical Offset: {Offset}/{Total}", + type, currentPage, currentItemOffset, totalCount); + + var assets = await tradeRepublicService.GetAssets(type, currentPage, pageSize, stoppingToken); + + // Keine Ergebnisse geliefert -> Katalogende erreicht + if (assets?.Results == null || assets.Results.Count == 0) + { + _logger.LogInformation("[{Channel}] Fetch for {AssetType} (Page {Page}) returned no results. Reached end of available assets.", "AssetsChannel", type, currentPage); + currentSettings.CurrentScanningPage = 0; + await settingsDbService.SaveSettings(currentSettings); + break; + } + + await ProcessAssets(assets.Results, assetsDbService, indexService, stoppingToken); + _currAssetsCount.SetCountOfType(type, _currAssetsCount.GetCountFromType(type) + assets.Results.Count); + + currentItemOffset += assets.Results.Count; + + // Unvollständige Seite -> Letzte Seite abgearbeitet + if (assets.Results.Count < pageSize) + { + _logger.LogInformation("[{Channel}] Reached the last page for {AssetType}.", "AssetsChannel", type); + currentSettings.CurrentScanningPage = 0; + await settingsDbService.SaveSettings(currentSettings); + break; + } + + var delaySeconds = currentSettings.FinishedInitialScan + ? currentSettings.BatchAssetUpdateDelay + : currentSettings.InitBatchAssetUpdateDelay; + + if (delaySeconds > 0) + { + // Angemessener Jitter (0 bis max. 5 Sek. bzw. kleiner als delaySeconds) + var maxJitter = Math.Min(5, delaySeconds); + var jitter = Random.Shared.Next(0, maxJitter + 1); + _logger.LogDebug("Waiting {Delay} seconds before the next batch.", delaySeconds + jitter); + await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); + } + } + + _logger.LogInformation("[{Channel}] Finished scanning {AssetType}. Total scanned in this cycle: {Count}/{Total}", + "AssetsChannel", type, _currAssetsCount.GetCountFromType(type), totalCount); + } + + private async Task ProcessAssets( + IList assets, + IAssetsDbService assetsDbService, + IAssetsIndexService indexService, + CancellationToken stoppingToken) + { + if (assets == null || assets.Count == 0) return; + + var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets); + _logger.LogInformation("[{Channel}] [Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", "AssetsChannel", assets.Count, changedRows); + + if (changedRows > 0) + { + _logger.LogInformation("[{Channel}] Database modifications detected. Recreating the asset index file...", "AssetsChannel"); + await indexService.ReCreateIndexFileAsync(stoppingToken); + } + } +} \ No newline at end of file diff --git a/FinlyticAssets/Services/AssetsDbService.cs b/FinlyticAssets/Services/AssetsDbService.cs index 11537b7..20f8055 100644 --- a/FinlyticAssets/Services/AssetsDbService.cs +++ b/FinlyticAssets/Services/AssetsDbService.cs @@ -1,7 +1,8 @@ using FinlyticAssets.Database; using FinlyticAssets.Entities; -using FinlyticAssets.Models.DataToObject.TradeRepublic; using FinlyticCore.Entities.Assets; +using FinlyticCore.Models.TradeRepublic; +using FinlyticCore.Services.TradeRepublic; using Microsoft.EntityFrameworkCore; namespace FinlyticAssets.Services; @@ -11,63 +12,15 @@ namespace FinlyticAssets.Services; /// public interface IAssetsDbService { - - /// - /// Retrieves all active assets from the database that have been updated within the last 14 days, including their associated tags. - /// Assets older than 14 days are filtered out as they are considered de-listed or inactive. - /// - /// A task that represents the asynchronous operation. The task result contains a list of all active instances. public Task> GetAllValidAssetsAsync(); - - /// - /// Adds a new asset or updates an existing one based on the composite key of ISIN and InstrumentType. - /// - /// The incoming asset data transfer object from the API. - /// A task that represents the asynchronous operation. The task result contains true if the asset was updated or created; otherwise, false. public Task AddOrUpdateAssetAsync(TradeRepublicAsset dtoAsset); - - /// - /// Batches processing for a collection of asset items, returning the total amount of modified or newly added entries. - /// - /// The collection of incoming asset objects to process. - /// A task that represents the asynchronous operation. The task result contains the count of changed or added entries. public Task AddOrUpdateAssetsAsync(IEnumerable dtoAssets); - - /// - /// Retrieves all assets matching a specific International Securities Identification Number (ISIN). - /// Can return multiple entities (e.g., both the stock and the derivative tracking asset for the same ISIN). - /// - /// The ISIN value to look up. - /// A task that represents the asynchronous operation. The task result contains a list of matching instances. public Task> GetAssetsByIsinAsync(string isin); - - /// - /// Retrieves all assets matching a specific ISIN only if they have been updated within the last 14 days. - /// Assets older than 14 days are considered de-listed or inactive. - /// - /// The ISIN value to look up. - /// A task that represents the asynchronous operation. The task result contains a list of matching active instances. public Task> GetValidAssetsByIsinAsync(string isin); - - /// - /// Scans the database for active assets matching a combined, comma-separated search query. - /// The search applies an AND-logic approach where every extracted keyword must be found within an asset's ISIN, name, or tags. - /// Evaluates local records first and utilizes a targeted JIT-fallback to the external API for any search term formatted as a valid, completely unknown ISIN. - /// - /// A comma-separated string containing the keywords, ISINs, or tags to search for (e.g., "Siemens, Medic" or "IE00B4L5Y983, ETF"). - /// A task that represents the asynchronous operation. The task result contains a list of all affected active instances. public Task> FindAffectedActiveAssetsAsync(string searchQuery); - - /// - /// Deletes all asset records associated with a specific ISIN from the database. - /// - /// - /// Use with caution. For standard maintenance and handling de-listed instruments, - /// rely on the 14-day recency filter provided by instead of hard deletion. - /// - /// The ISIN value of the assets to remove. - /// A task that represents the asynchronous operation. The task result contains true if any records were successfully deleted; otherwise, false. + public Task UpdateAssetImageIdAsync(string isin, string imageId); public Task DeleteAssetAsync(string isin); + public Task> GetDiscoveryAssetsAsync(int limit = 15); } /// @@ -76,33 +29,77 @@ public class AssetsDbService : IAssetsDbService private readonly AssetsDbContext _context; private readonly ITradeRepublicService _tradeRepublicService; private readonly ILogger _logger; - private readonly Random _random = new(); - /// - /// Initializes a new instance of the class. - /// - public AssetsDbService(AssetsDbContext context, ILogger logger, - ITradeRepublicService tradeRepublicService) + public AssetsDbService(AssetsDbContext context, ILogger logger, ITradeRepublicService tradeRepublicService) { _context = context; _logger = logger; _tradeRepublicService = tradeRepublicService; } - /// - public async Task> GetAllValidAssetsAsync() + /// Inherits documentation from interface. + public async Task> GetDiscoveryAssetsAsync(int limit = 15) { - var cutoff = DateTime.UtcNow.AddDays(-14); - - var existingEntity = await _context.TradeRepublicAssets + var cutoff = DateTime.UtcNow.AddDays(-90); + + var validAssets = await _context.TradeRepublicAssets + .AsNoTracking() .Include(a => a.Tags) - .Where(a => a.UpdateAt >= cutoff) + .Where(a => a.LastUpdatedAt >= cutoff && !string.IsNullOrEmpty(a.Name)) .ToListAsync(); - return existingEntity; + if (validAssets.Count == 0) return []; + + var scored = validAssets + .Select(a => new + { + Asset = a, + Score = (a.Tags?.Count ?? 0) * 10 + + (a.HasCfd ? 5 : 0) + + (string.IsNullOrEmpty(a.ImageId) ? 0 : 15) + + (a.Name.Length > 3 ? 5 : 0) + }) + .OrderByDescending(x => x.Score) + .ThenByDescending(x => x.Asset.LastUpdatedAt) + .ToList(); + + var result = new List(); + var grouped = scored.GroupBy(x => x.Asset.Type).ToList(); + + int index = 0; + while (result.Count < limit && grouped.Any(g => g.Any())) + { + bool addedAny = false; + foreach (var group in grouped) + { + var item = group.Skip(index).FirstOrDefault(); + if (item != null) + { + result.Add(item.Asset); + addedAny = true; + if (result.Count >= limit) break; + } + } + index++; + if (!addedAny) break; + } + + return result; } - - /// + + /// Inherits documentation from interface. + public async Task> GetAllValidAssetsAsync() + { + var cutoff = DateTime.UtcNow.AddDays(-90); + + return await _context.TradeRepublicAssets + .AsNoTracking() + .Include(a => a.Tags) + .Where(a => a.LastUpdatedAt >= cutoff) + .ToListAsync(); + } + + /// Inherits documentation from interface. public async Task AddOrUpdateAssetAsync(TradeRepublicAsset dtoAsset) { var existingEntity = await _context.TradeRepublicAssets @@ -110,7 +107,20 @@ public class AssetsDbService : IAssetsDbService .FirstOrDefaultAsync(a => a.Isin == dtoAsset.Isin); var now = DateTime.UtcNow; - var nextUpdateScheduledAt = await CalculateNextUpdateDateAsync(now); + + var mappedTags = new List(); + foreach (var tagDto in dtoAsset.Tags ?? Array.Empty()) + { + var existingTag = await _context.TradeRepublicTags.FirstOrDefaultAsync(t => t.Id == tagDto.Id); + if (existingTag == null) + { + _logger.LogTrace("Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id); + existingTag = new TagEntity { Id = tagDto.Id, Name = tagDto.Name, Type = tagDto.Type }; + await _context.TradeRepublicTags.AddAsync(existingTag); + } + + mappedTags.Add(existingTag); + } if (existingEntity == null) { @@ -118,53 +128,125 @@ public class AssetsDbService : IAssetsDbService var newEntity = MapDtoToEntity(dtoAsset); newEntity.LastUpdatedAt = now; - newEntity.UpdateAt = nextUpdateScheduledAt; - - newEntity.Tags = await MapTagsAsync(dtoAsset.Tags); + newEntity.Tags = mappedTags; await _context.TradeRepublicAssets.AddAsync(newEntity); await _context.SaveChangesAsync(); return true; } - _logger.LogDebug("Asset with ISIN {Isin} exists. Merging properties and updating database record.", - dtoAsset.Isin); + _logger.LogDebug("Asset with ISIN {Isin} exists. Merging properties and updating database record.", dtoAsset.Isin); existingEntity.Name = dtoAsset.Name; existingEntity.Type = dtoAsset.Type; existingEntity.InstrumentCategory = dtoAsset.InstrumentCategory; existingEntity.HasCfd = dtoAsset.HasCfd; existingEntity.ImageId = dtoAsset.ImageId; - existingEntity.LastUpdatedAt = now; - existingEntity.UpdateAt = nextUpdateScheduledAt; UpdateSubtypeProperties(existingEntity, dtoAsset); - existingEntity.Tags = await MapTagsAsync(dtoAsset.Tags); + existingEntity.Tags = mappedTags; _context.TradeRepublicAssets.Update(existingEntity); await _context.SaveChangesAsync(); return true; } - /// + /// Inherits documentation from interface. public async Task AddOrUpdateAssetsAsync(IEnumerable dtoAssets) { + var assetsList = dtoAssets.ToList(); + if (assetsList.Count == 0) return 0; + + var now = DateTime.UtcNow; int changedCount = 0; - foreach (var dto in dtoAssets) + var isins = assetsList.Select(a => a.Isin).Distinct().ToList(); + var existingAssets = await _context.TradeRepublicAssets + .Include(a => a.Tags) + .Where(a => isins.Contains(a.Isin)) + .ToDictionaryAsync(a => a.Isin); + + var tagIds = assetsList + .SelectMany(a => a.Tags ?? Array.Empty()) + .Select(t => t.Id) + .Distinct() + .ToList(); + + var tagCache = await _context.TradeRepublicTags + .Where(t => tagIds.Contains(t.Id)) + .ToDictionaryAsync(t => t.Id); + + foreach (var dto in assetsList) { - var isChanged = await AddOrUpdateAssetAsync(dto); + var isChanged = false; + var existingEntity = existingAssets.GetValueOrDefault(dto.Isin); + + var mappedTags = new List(); + foreach (var tagDto in dto.Tags ?? Array.Empty()) + { + if (!tagCache.TryGetValue(tagDto.Id, out var tagEntity)) + { + _logger.LogTrace("Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id); + tagEntity = new TagEntity { Id = tagDto.Id, Name = tagDto.Name, Type = tagDto.Type }; + await _context.TradeRepublicTags.AddAsync(tagEntity); + tagCache.Add(tagDto.Id, tagEntity); + } + mappedTags.Add(tagEntity); + } + + if (existingEntity == null) + { + _logger.LogDebug("Asset with ISIN {Isin} not found. Mapping and creating a new entity.", dto.Isin); + + var newEntity = MapDtoToEntity(dto); + newEntity.LastUpdatedAt = now; + newEntity.Tags = mappedTags; + + await _context.TradeRepublicAssets.AddAsync(newEntity); + isChanged = true; + } + else + { + _logger.LogDebug("Asset with ISIN {Isin} exists. Merging properties and updating database record.", dto.Isin); + + if (existingEntity.Name != dto.Name || + existingEntity.Type != dto.Type || + existingEntity.InstrumentCategory != dto.InstrumentCategory || + existingEntity.HasCfd != dto.HasCfd || + existingEntity.ImageId != dto.ImageId || + !existingEntity.Tags.SequenceEqual(mappedTags)) + { + existingEntity.Name = dto.Name; + existingEntity.Type = dto.Type; + existingEntity.InstrumentCategory = dto.InstrumentCategory; + existingEntity.HasCfd = dto.HasCfd; + existingEntity.ImageId = dto.ImageId; + existingEntity.LastUpdatedAt = now; + + UpdateSubtypeProperties(existingEntity, dto); + existingEntity.Tags = mappedTags; + + _context.TradeRepublicAssets.Update(existingEntity); + isChanged = true; + } + } + if (isChanged) { changedCount++; } } + if (changedCount > 0) + { + await _context.SaveChangesAsync(); + } + return changedCount; } - /// + /// Inherits documentation from interface. public async Task> GetAssetsByIsinAsync(string isin) { var localAssets = await _context.TradeRepublicAssets @@ -172,26 +254,30 @@ public class AssetsDbService : IAssetsDbService .Where(a => a.Isin == isin) .ToListAsync(); - if (localAssets.Any()) + if (localAssets.Count > 0) { return localAssets; } + // JIT-Fetch via API var trAssetDto = await _tradeRepublicService.GetAsset(isin); - if (trAssetDto != null) + if (trAssetDto?.Results != null && trAssetDto.Results.Count > 0) { foreach (var asset in trAssetDto.Results) { - _ = await AddOrUpdateAssetAsync(asset); + await AddOrUpdateAssetAsync(asset); } - return await GetAssetsByIsinAsync(isin); + return await _context.TradeRepublicAssets + .Include(a => a.Tags) + .Where(a => a.Isin == isin) + .ToListAsync(); } return []; } - /// + /// Inherits documentation from interface. public async Task> GetValidAssetsByIsinAsync(string isin) { var cutoff = DateTime.UtcNow.AddDays(-14); @@ -199,23 +285,20 @@ public class AssetsDbService : IAssetsDbService return await _context.TradeRepublicAssets .AsNoTracking() .Include(a => a.Tags) - .Where(a => a.Isin == isin && a.UpdateAt >= cutoff) + .Where(a => a.Isin == isin && a.LastUpdatedAt >= cutoff) .ToListAsync(); } - /// + /// Inherits documentation from interface. public async Task> FindAffectedActiveAssetsAsync(string searchQuery) { - if (string.IsNullOrWhiteSpace(searchQuery)) - { - return []; - } + if (string.IsNullOrWhiteSpace(searchQuery)) return []; var cutoff = DateTime.UtcNow.AddDays(-14); - var searchTerms = searchQuery + var searchTerms = searchQuery .Split(',') - .Select(t => t.Trim().ToLower()) + .Select(t => t.Trim()) .Where(t => !string.IsNullOrEmpty(t)) .Distinct() .ToList(); @@ -224,17 +307,17 @@ public class AssetsDbService : IAssetsDbService var query = _context.TradeRepublicAssets .AsNoTracking() - .Where(a => a.UpdateAt >= cutoff) + .Where(a => a.LastUpdatedAt >= cutoff) .Include(a => a.Tags) .AsQueryable(); - foreach (var term in searchTerms) { + var lowerTerm = term.ToLower(); query = query.Where(a => - a.Isin.ToLower().Contains(term) || - a.Name.ToLower().Contains(term) || - a.Tags.Any(tag => tag.Name.ToLower().Contains(term))); + a.Isin.ToLower().Contains(lowerTerm) || + a.Name.ToLower().Contains(lowerTerm) || + a.Tags.Any(tag => tag.Name.ToLower().Contains(lowerTerm))); } var localAssets = await query.ToListAsync(); @@ -244,12 +327,12 @@ public class AssetsDbService : IAssetsDbService .Select(t => t.ToUpper()) .ToList(); - if (possibleIsins.Any()) + if (possibleIsins.Count > 0) { var foundIsins = localAssets.Select(a => a.Isin).ToHashSet(); var missingIsins = possibleIsins.Where(isin => !foundIsins.Contains(isin)).ToList(); - if (missingIsins.Any()) + if (missingIsins.Count > 0) { var fetchedNewAsset = false; foreach (var missingIsin in missingIsins) @@ -276,57 +359,42 @@ public class AssetsDbService : IAssetsDbService return localAssets; } - /// + /// Inherits documentation from interface. + public async Task UpdateAssetImageIdAsync(string isin, string imageId) + { + var existingAssets = await _context.TradeRepublicAssets + .Where(a => a.Isin == isin) + .ToListAsync(); + + if (existingAssets.Count > 0) + { + foreach (var asset in existingAssets) + { + asset.ImageId = imageId; + } + await _context.SaveChangesAsync(); + _logger.LogInformation("[{Channel}] Updated ImageId for ISIN {Isin} in database to '{ImageId}'", "AssetsChannel", isin, imageId); + } + } + + /// Inherits documentation from interface. public async Task DeleteAssetAsync(string isin) { var asset = await _context.TradeRepublicAssets.FirstOrDefaultAsync(a => a.Isin == isin); if (asset == null) { - _logger.LogWarning("Delete execution cancelled. Asset with ISIN {Isin} does not exist.", isin); + _logger.LogWarning("[{Channel}] Delete execution cancelled. Asset with ISIN {Isin} does not exist.", "AssetsChannel", isin); return false; } _context.TradeRepublicAssets.Remove(asset); await _context.SaveChangesAsync(); - _logger.LogInformation("Asset with ISIN {Isin} has been successfully deleted.", isin); + _logger.LogInformation("[{Channel}] Asset with ISIN {Isin} has been successfully deleted.", "AssetsChannel", isin); return true; } #region Helper & Mapping Methods - /// - /// Calculates the next synchronization/update date for an asset using configured parameters and a random offset. - /// - /// The baseline date to add the offsets to. - /// A task representing the asynchronous operation, returning a DateTime representing the next scheduled update time (UTC). - private async Task CalculateNextUpdateDateAsync(DateTime baseDate) - { - var settings = await _context.Set().FirstOrDefaultAsync() ?? new Settings(); - - int randomDays = _random.Next(settings.MinRandomUpdateDay, settings.MaxRandomUpdateDay + 1); - var targetDate = baseDate.AddDays(randomDays); - - int randomHour = _random.Next(settings.UpdateDayTimeStart, settings.UpdateDayTimeStop); - int randomMinute = _random.Next(0, 60); - int randomSecond = _random.Next(0, 60); - - return new DateTime( - targetDate.Year, - targetDate.Month, - targetDate.Day, - randomHour, - randomMinute, - randomSecond, - DateTimeKind.Utc - ); - } - - /// - /// Maps a raw Trade Republic asset data transfer object (DTO) to its matching database entity subtype. - /// - /// The source Trade Republic asset data transfer object. - /// A newly created subtype instance of mapped with the DTO properties. - /// Thrown when the DTO type is unrecognized or unsupported. private AssetEntity MapDtoToEntity(TradeRepublicAsset dto) { AssetEntity entity = dto switch @@ -356,12 +424,6 @@ public class AssetsDbService : IAssetsDbService return PopulateBaseProperties(entity, dto); } - /// - /// Populates common base properties of a database asset entity using a Trade Republic DTO. - /// - /// The target database entity. - /// The source Trade Republic DTO. - /// The updated database asset entity. private AssetEntity PopulateBaseProperties(AssetEntity entity, TradeRepublicAsset dto) { entity.Name = dto.Name; @@ -372,11 +434,6 @@ public class AssetsDbService : IAssetsDbService return entity; } - /// - /// Merges/updates the subtype-specific properties from a Trade Republic DTO into an existing database entity. - /// - /// The existing database entity to update. - /// The source Trade Republic DTO. private void UpdateSubtypeProperties(AssetEntity entity, TradeRepublicAsset dto) { switch (entity) @@ -409,29 +466,5 @@ public class AssetsDbService : IAssetsDbService } } - /// - /// Maps a list of Trade Republic tags to database entity instances, registering new tags in the database context if they do not yet exist. - /// - /// The read-only collection of Trade Republic tags. - /// A task representing the asynchronous operation, returning the list of mapped database tag entities. - private async Task> MapTagsAsync(IReadOnlyList dtos) - { - var tags = new List(); - foreach (var tagDto in dtos) - { - var existingTag = await _context.TradeRepublicTags.FirstOrDefaultAsync(t => t.Id == tagDto.Id); - if (existingTag == null) - { - _logger.LogTrace("Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id); - existingTag = new TagEntity { Id = tagDto.Id, Name = tagDto.Name, Type = tagDto.Type }; - await _context.TradeRepublicTags.AddAsync(existingTag); - } - - tags.Add(existingTag); - } - - return tags; - } - #endregion } \ No newline at end of file diff --git a/FinlyticAssets/Services/AssetsFullScanService.cs b/FinlyticAssets/Services/AssetsFullScanService.cs deleted file mode 100644 index 0bc22e7..0000000 --- a/FinlyticAssets/Services/AssetsFullScanService.cs +++ /dev/null @@ -1,231 +0,0 @@ -using FinlyticAssets.Models; -using FinlyticAssets.Models.DataToObject.TradeRepublic; -using FinlyticCore.Models.Assets; - -namespace FinlyticAssets.Services; - -/// -/// A background service that runs a continuous asset synchronization loop, -/// scanning Trade Republic to retrieve, update, and index all supported asset types. -/// -public class AssetsFullScanService : BackgroundService -{ - private readonly IServiceScopeFactory _serviceScopeFactory; - private readonly ILogger _logger; - - private AssetsCount? _assetsCount; - private AssetsCount? _currAssetsCount; - - /// - /// Initializes a new instance of the class. - /// - /// Factory used to create service scopes for database and API requests. - /// Logger for service lifecycle and scanning progress messages. - public AssetsFullScanService(IServiceScopeFactory serviceScopeFactory, ILogger logger) - { - _serviceScopeFactory = serviceScopeFactory; - _logger = logger; - } - - /// - /// Executes the background scanning task, handling initial startup, recovery, and periodic full-scan cycles. - /// - /// Triggered when the host is shutting down. - /// A task that represents the background operation. - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("AssetsFullScanService has started."); - - try - { - using (var scope = _serviceScopeFactory.CreateScope()) - { - var indexService = scope.ServiceProvider.GetRequiredService(); - _logger.LogInformation("Building initial asset index on service startup..."); - await indexService.ReCreateIndexFileAsync(stoppingToken); - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to build initial asset index on startup. Continuing service execution."); - } - - do - { - try - { - using (var scope = _serviceScopeFactory.CreateScope()) - { - var tradeRepublicService = scope.ServiceProvider.GetRequiredService(); - var settingsService = scope.ServiceProvider.GetRequiredService(); - var assetsDbService = scope.ServiceProvider.GetRequiredService(); - var indexService = scope.ServiceProvider.GetRequiredService(); - - _logger.LogInformation("Requesting total asset counts from Trade Republic..."); - _assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken); - _currAssetsCount = new AssetsCount(); - - var initSettings = await settingsService.GetSettings(); - var isRecoveryMode = initSettings.CurrentScanningPage > 0; - - foreach (var type in Enum.GetValues()) - { - if (stoppingToken.IsCancellationRequested) break; - - if (isRecoveryMode) - { - if (type != initSettings.CurrentScanningType) - { - _logger.LogInformation("Recovery: {AssetType} was already processed. Skipping.", type); - continue; - } - isRecoveryMode = false; - } - else - { - var settings = await settingsService.GetSettings(); - settings.CurrentScanningType = type; - settings.CurrentScanningPage = 0; - await settingsService.SaveSettings(settings); - } - - _logger.LogInformation("Processing asset type: {AssetType}...", type); - await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken); - - var currentSettings = await settingsService.GetSettings(); - var delaySeconds = currentSettings.FinishedInitialScan - ? currentSettings.AssetUpdateTypeDelay - : currentSettings.InitAssetUpdateTypeDelay; - - var jitter = Random.Shared.Next(0, 480); - _logger.LogDebug("Waiting {Delay} seconds before the next asset type.", delaySeconds + jitter); - await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); - } - - var finalSettings = await settingsService.GetSettings(); - finalSettings.CurrentScanningPage = 0; - - if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested) - { - _logger.LogInformation("Initial scan successfully completed. Switching FinishedInitialScan to true."); - finalSettings.FinishedInitialScan = true; - } - - await settingsService.SaveSettings(finalSettings); - } - - _logger.LogInformation("Full scan cycle completed. Waiting 1 minute before starting the next cycle."); - await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); - } - catch (Exception e) - { - _logger.LogError(e, "An unhandled exception occurred in AssetsFullScanService. Retrying in 10 seconds."); - try { await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); } catch { /* Ignore */ } - } - } while (!stoppingToken.IsCancellationRequested); - - _logger.LogInformation("AssetsFullScanService is stopping."); - } - - /// - /// Handles the scanning process for a specific asset type, iterating through its paginated results. - /// - /// The type of assets to process (e.g., Stock, Crypto). - /// Service to fetch asset data from Trade Republic. - /// Service to read and write application state/settings. - /// Service to insert or update assets in the database. - /// Service to recreate the local index file if needed. - /// Cancellation token monitored for cancellation requests. - /// A task representing the asynchronous operation. - private async Task HandleAssetType(AssetType type, ITradeRepublicService tradeRepublicService, - ISettingsDbService settingsDbService, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) - { - var totalCount = _assetsCount?.GetCountFromType(type) ?? 0; - if (totalCount == 0) - { - _logger.LogWarning("No assets found for type {AssetType}.", type); - return; - } - - _currAssetsCount ??= new AssetsCount(); - var currentItemOffset = 0; - - var settings = await settingsDbService.GetSettings(); - if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0) - { - var pageSize = settings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : settings.TradeRepublicMaxRequestPageSize; - if (pageSize > 100) pageSize = 100; - - currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize; - _logger.LogInformation("Resuming full scan for {AssetType} from Page {Page} (Offset: {Offset}).", - type, settings.CurrentScanningPage, currentItemOffset); - } - - while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested) - { - var currentSettings = await settingsDbService.GetSettings(); - var pageSize = currentSettings.TradeRepublicMaxRequestPageSize; - if (pageSize <= 0 || pageSize > 100) pageSize = 100; - - var currentPage = (currentItemOffset / pageSize) + 1; - - currentSettings.CurrentScanningType = type; - currentSettings.CurrentScanningPage = currentPage; - await settingsDbService.SaveSettings(currentSettings); - - _logger.LogDebug("Fetching {AssetType} - Page {Page} (Size: {PageSize}). Offset: {Offset}/{Total}", - type, currentPage, pageSize, currentItemOffset, totalCount); - - var assets = await tradeRepublicService.GetAssets(type, currentPage, pageSize, stoppingToken); - - if (assets?.Results == null || assets.Results.Count == 0) - { - _logger.LogWarning("Fetch for {AssetType} (Page {Page}) returned no results. Retrying in 5 seconds...", type, currentPage); - await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); - continue; - } - - await ProcessAssets(assets.Results, assetsDbService, indexService,stoppingToken); - _currAssetsCount.SetCountOfType(type, _currAssetsCount.GetCountFromType(type) + assets.Results.Count); - - currentItemOffset = currentPage * pageSize; - - if (assets.Results.Count < pageSize) - { - _logger.LogInformation("Reached the last page for {AssetType}.", type); - break; - } - - var delaySeconds = currentSettings.FinishedInitialScan - ? currentSettings.BatchAssetUpdateDelay - : currentSettings.InitBatchAssetUpdateDelay; - - var jitter = Random.Shared.Next(0, 360); - _logger.LogDebug("Waiting {Delay} seconds before the next batch.", delaySeconds + jitter); - await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); - } - } - - /// - /// Processes a list of fetched Trade Republic assets, updates them in the database, - /// and triggers an index recreation if changes were detected. - /// - /// The list of Trade Republic assets to process. - /// Service to insert or update assets in the database. - /// Service to recreate the local index file. - /// Cancellation token monitored for cancellation requests. - /// A task representing the asynchronous operation. - private async Task ProcessAssets(IList assets, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) - { - if (assets == null || assets.Count == 0) return; - - var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets); - _logger.LogInformation("[Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows); - - if (changedRows > 0) - { - _logger.LogInformation("Database modifications detected. Recreating the asset index file..."); - await indexService.ReCreateIndexFileAsync(stoppingToken); - } - } -} \ No newline at end of file diff --git a/FinlyticAssets/Services/AssetsIndexService.cs b/FinlyticAssets/Services/AssetsIndexService.cs index 611b904..4955462 100644 --- a/FinlyticAssets/Services/AssetsIndexService.cs +++ b/FinlyticAssets/Services/AssetsIndexService.cs @@ -6,7 +6,7 @@ using FinlyticAssets.Util; namespace FinlyticAssets.Services; /// -/// Provides methods for generating and maintaining the indexed asset reference file used for pre-filtering. +/// Provides methods for generating the indexed asset reference file and downloading local asset logos strictly on demand. /// public interface IAssetsIndexService { @@ -15,16 +15,22 @@ public interface IAssetsIndexService /// /// A token to monitor for cancellation requests. /// A task that represents the asynchronous operation. - public Task ReCreateIndexFileAsync(CancellationToken cancellationToken); + public Task ReCreateIndexFileAsync(CancellationToken cancellationToken = default); + + /// + /// Strictly On Demand: Downloads and saves the logo SVG for a requested ISIN into the local assets/logos folder. + /// + public Task DownloadAndSaveLogoAsync(string isin, CancellationToken cancellationToken = default); } /// -/// Implements the to maintain local asset index references. +/// Implements the to maintain local asset index references and logo file storage. /// public class AssetsIndexService : IAssetsIndexService { private readonly ILogger _logger; private readonly IAssetsDbService _assetsDbService; + private static readonly HttpClient _httpClient = new(); /// /// Initializes a new instance of the class. @@ -37,19 +43,26 @@ public class AssetsIndexService : IAssetsIndexService _assetsDbService = assetsDbService; } - /// - public async Task ReCreateIndexFileAsync(CancellationToken cancellationToken) + /// Inherits documentation from interface. + public async Task ReCreateIndexFileAsync(CancellationToken cancellationToken = default) { try { var assets = await _assetsDbService.GetAllValidAssetsAsync(); if (assets == null || !assets.Any()) { - _logger.LogWarning("No valid assets found in the database to index."); + _logger.LogWarning("[{Channel}] No valid assets found in the database to index.", "AssetsChannel"); return; } - var indexAssets = assets.Select(a => new AssetIndex(a.Isin, a.Name)).ToList(); + var indexAssets = assets + .DistinctBy(a => a.Isin) + .Select(a => { + string cleanIsin = a.Isin.Trim().ToUpperInvariant(); + // Point directly to our own local backend logo endpoint + string imageUrl = $"/api/v1/logo/{cleanIsin}"; + return new AssetIndex(cleanIsin, a.Name, imageUrl); + }).ToList(); var directoryPath = Volumes.IndexRelativePath; var filePath = Path.Combine(directoryPath, "index.json"); @@ -65,23 +78,68 @@ public class AssetsIndexService : IAssetsIndexService await JsonSerializer.SerializeAsync(fileStream, indexAssets, cancellationToken: cancellationToken); } - _logger.LogInformation("Successfully recreated asset index file with {Count} entries at {Path}", - indexAssets.Count, filePath); + _logger.LogInformation("[{Channel}] Successfully recreated asset index file with {Count} entries pointing to local logos at {Path}", + "AssetsChannel", indexAssets.Count, filePath); } catch (IOException ex) { - _logger.LogError(ex, "Disk I/O error occurred while writing the asset index file."); + _logger.LogError(ex, "[{Channel}] Disk I/O error occurred while writing the asset index file.", "AssetsChannel"); throw; } catch (JsonException ex) { - _logger.LogError(ex, "Failed to serialize the asset index data to JSON."); + _logger.LogError(ex, "[{Channel}] Failed to serialize the asset index data to JSON.", "AssetsChannel"); throw; } catch (Exception ex) { - _logger.LogError(ex, "An unexpected error occurred while recreating the asset index file."); + _logger.LogError(ex, "[{Channel}] An unexpected error occurred while recreating the asset index file.", "AssetsChannel"); throw; } } -} \ No newline at end of file + + /// Inherits documentation from interface. + public async Task DownloadAndSaveLogoAsync(string isin, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(isin)) return null; + + string cleanIsin = isin.Trim().ToUpperInvariant(); + string directoryPath = Volumes.LogosRelativePath; + string filePath = Path.Combine(directoryPath, $"{cleanIsin}.svg"); + + if (!Directory.Exists(directoryPath)) + { + Directory.CreateDirectory(directoryPath); + } + + if (File.Exists(filePath)) + { + return filePath; + } + + string targetUrl = $"https://assets.traderepublic.com/img/logos/{cleanIsin}/v2/dark.min.svg"; + + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, targetUrl); + request.Headers.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + request.Headers.TryAddWithoutValidation("Accept", "image/svg+xml,image/*,*/*"); + request.Headers.TryAddWithoutValidation("Referer", "https://traderepublic.com/"); + + using var response = await _httpClient.SendAsync(request, cancellationToken); + if (response.IsSuccessStatusCode) + { + byte[] data = await response.Content.ReadAsByteArrayAsync(cancellationToken); + await File.WriteAllBytesAsync(filePath, data, cancellationToken); + _logger.LogInformation("[{Channel}] Successfully saved logo SVG for ISIN {Isin} to {Path} on demand", "AssetsChannel", cleanIsin, filePath); + return filePath; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[{Channel}] Failed to download logo for ISIN {Isin} from {Url}", "AssetsChannel", cleanIsin, targetUrl); + } + + return null; + } +} diff --git a/FinlyticAssets/Services/LogoFetcherBackgroundService.cs b/FinlyticAssets/Services/LogoFetcherBackgroundService.cs new file mode 100644 index 0000000..76b1c1f --- /dev/null +++ b/FinlyticAssets/Services/LogoFetcherBackgroundService.cs @@ -0,0 +1,171 @@ +using System; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FinlyticAssets.Util; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace FinlyticAssets.Services; + +/// +/// Dedicated background service that periodically scans for missing asset logos in the local storage directory +/// and fetches them in batches from Trade Republic CDN. +/// Swaps missing/404 logos with a clean SVG placeholder image and triggers ReCreateIndexFileAsync. +/// +public class LogoFetcherBackgroundService : BackgroundService +{ + private readonly ILogger _logger; + private readonly IServiceScopeFactory _scopeFactory; + private readonly HttpClient _httpClient; + + private const string PlaceholderSvg = """ + + + + + + """; + + public LogoFetcherBackgroundService( + ILogger logger, + IServiceScopeFactory scopeFactory) + { + _logger = logger; + _scopeFactory = scopeFactory; + _httpClient = new HttpClient(); + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "image/svg+xml,image/*,*/*"); + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://traderepublic.com/"); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("[{Channel}] LogoFetcherBackgroundService started. Will fetch missing logos periodically.", "AssetsChannel"); + + await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await ProcessMissingLogosBatchAsync(stoppingToken); + } + catch (Exception ex) when (!stoppingToken.IsCancellationRequested) + { + _logger.LogError(ex, "[{Channel}] Error occurred while executing logo batch fetch.", "AssetsChannel"); + } + + try + { + await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + } + catch (OperationCanceledException) + { + break; + } + } + + _logger.LogInformation("[{Channel}] LogoFetcherBackgroundService stopped.", "AssetsChannel"); + } + + private async Task ProcessMissingLogosBatchAsync(CancellationToken stoppingToken) + { + using var scope = _scopeFactory.CreateScope(); + var dbService = scope.ServiceProvider.GetRequiredService(); + var indexService = scope.ServiceProvider.GetRequiredService(); + + var validAssets = await dbService.GetAllValidAssetsAsync(); + if (validAssets == null || !validAssets.Any()) return; + + string directoryPath = Volumes.LogosRelativePath; + if (!Directory.Exists(directoryPath)) + { + Directory.CreateDirectory(directoryPath); + } + + // ✅ Prüft sowohl DB-Eintrag ALS AUCH, ob die Datei bereits lokal existiert + var missingIsins = validAssets + .Select(a => a.Isin?.Trim().ToUpperInvariant()) + .Where(isin => !string.IsNullOrEmpty(isin)) + .Distinct() + .Where(isin => !File.Exists(Path.Combine(directoryPath, $"{isin}.svg"))) + .ToList(); + + if (missingIsins.Count == 0) + { + _logger.LogDebug("All asset logos are downloaded and up to date."); + return; + } + + var batchToFetch = missingIsins.Take(60).ToList(); + _logger.LogInformation("[{Channel}] Found {Count} missing logos on disk. Fetching bulk batch of {BatchSize} logos...", "AssetsChannel", missingIsins.Count, batchToFetch.Count); + + int successCount = 0; + + foreach (var isin in batchToFetch) + { + if (stoppingToken.IsCancellationRequested) break; + + string targetUrl = $"https://assets.traderepublic.com/img/logos/{isin}/v2/dark.min.svg"; + string filePath = Path.Combine(directoryPath, $"{isin}.svg"); + string dbImageEndpoint = $"/api/v1/logo/{isin}"; + + try + { + using var response = await _httpClient.GetAsync(targetUrl, stoppingToken); + if (response.IsSuccessStatusCode) + { + byte[] data = await response.Content.ReadAsByteArrayAsync(stoppingToken); + await File.WriteAllBytesAsync(filePath, data, stoppingToken); + successCount++; + } + else + { + _logger.LogWarning("[{Channel}] Logo not found on CDN for ISIN {Isin} (HTTP {StatusCode}). Saving SVG placeholder.", "AssetsChannel", isin, response.StatusCode); + byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg); + await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken); + successCount++; + } + + await dbService.UpdateAssetImageIdAsync(isin, dbImageEndpoint); + } + catch (Exception ex) when (!stoppingToken.IsCancellationRequested) + { + _logger.LogWarning(ex, "[{Channel}] Exception while downloading logo for ISIN {Isin} from {Url}. Saving SVG placeholder.", "AssetsChannel", isin, targetUrl); + try + { + byte[] placeholderData = Encoding.UTF8.GetBytes(PlaceholderSvg); + await File.WriteAllBytesAsync(filePath, placeholderData, stoppingToken); + successCount++; + + await dbService.UpdateAssetImageIdAsync(isin, dbImageEndpoint); + } + catch { } + } + + // Kurze Pause gegen Rate Limiting + await Task.Delay(50, stoppingToken); + } + + _logger.LogInformation("[{Channel}] Batch fetch complete. Successfully processed {SuccessCount}/{BatchSize} logos. Remaining missing: {Remaining}", + "AssetsChannel", successCount, batchToFetch.Count, missingIsins.Count - batchToFetch.Count); + + if (successCount > 0) + { + try + { + await indexService.ReCreateIndexFileAsync(stoppingToken); + _logger.LogInformation("[{Channel}] Successfully updated index.json after logo batch fetch.", "AssetsChannel"); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[{Channel}] Failed to update index.json after logo batch fetch.", "AssetsChannel"); + } + } + } +} \ No newline at end of file diff --git a/FinlyticAssets/Services/MqttConnectionService.cs b/FinlyticAssets/Services/MqttConnectionService.cs deleted file mode 100644 index cdc27b6..0000000 --- a/FinlyticAssets/Services/MqttConnectionService.cs +++ /dev/null @@ -1,52 +0,0 @@ -using FinlyticAssets.Util; -using FinlyticCore.Models; - -namespace FinlyticAssets.Services; - -/// -/// A hosted service responsible for managing the lifecycle of the MQTT client connection -/// when the application starts up and shuts down. -/// -public class MqttConnectionService : IHostedService -{ - private readonly AssetsMqttClient _mqttClient; - private readonly IConfiguration _configuration; - - /// - /// Initializes a new instance of the class. - /// - /// The MQTT client wrapper instance. - /// The application configuration provider. - public MqttConnectionService(AssetsMqttClient mqttClient, IConfiguration configuration) - { - _mqttClient = mqttClient; - _configuration = configuration; - } - - /// - /// Starts the MQTT client connection using settings resolved from configuration. - /// - /// A token to monitor for cancellation requests. - /// A task representing the asynchronous start operation. - public async Task StartAsync(CancellationToken cancellationToken) - { - var config = new MqttConfiguration() - { - Host = _configuration["MQTT__Host"]!, - Port = Convert.ToInt32(_configuration["MQTT__Port"]!), - ClientId = $"{_configuration["MQTT__ClientId"]!}_{Guid.NewGuid()}" - }; - - await _mqttClient.ConnectAsync(config); - } - - /// - /// Stops and disconnects the MQTT client connection. - /// - /// A token to monitor for cancellation requests. - /// A task representing the asynchronous stop operation. - public async Task StopAsync(CancellationToken cancellationToken) - { - await _mqttClient.DisconnectAsync(); - } -} \ No newline at end of file diff --git a/FinlyticAssets/Services/SettingsDbService.cs b/FinlyticAssets/Services/SettingsDbService.cs index 54b4a1d..0210523 100644 --- a/FinlyticAssets/Services/SettingsDbService.cs +++ b/FinlyticAssets/Services/SettingsDbService.cs @@ -1,4 +1,4 @@ -using FinlyticAssets.Database; +using FinlyticAssets.Database; using FinlyticAssets.Entities; using Microsoft.EntityFrameworkCore; @@ -28,6 +28,11 @@ public interface ISettingsDbService /// and tracked instance. /// public Task SaveSettings(Settings settings); + + /// + /// Updates settings from a key-value dictionary received via Admin Panel MQTT events. + /// + public Task UpdateSettingsFromDictionary(Dictionary dictionary); } /// @@ -47,25 +52,23 @@ public class SettingsDbService : ISettingsDbService _context = context; } - /// + /// Inherits documentation from interface. public async Task GetSettings() { var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync(); if (settings == null) { - settings = new Settings - { - Id = Guid.NewGuid() - }; - - await SaveSettings(settings); + settings = new Settings { Id = Guid.NewGuid() }; + _context.Settings.Add(settings); + await _context.SaveChangesAsync(); + _context.ChangeTracker.Clear(); } return settings; } - /// + /// Inherits documentation from interface. public async Task SaveSettings(Settings settings) { var existing = await _context.Settings.FirstOrDefaultAsync(); @@ -76,16 +79,47 @@ public class SettingsDbService : ISettingsDbService { settings.Id = Guid.NewGuid(); } - await _context.Settings.AddAsync(settings); + _context.Settings.Add(settings); await _context.SaveChangesAsync(); return settings; } else { - _context.Entry(existing).CurrentValues.SetValues(settings); - + existing.FinishedInitialScan = settings.FinishedInitialScan; + existing.TradeRepublicMaxRequestPageSize = settings.TradeRepublicMaxRequestPageSize; + existing.AssetUpdateTypeDelay = settings.AssetUpdateTypeDelay; + existing.InitAssetUpdateTypeDelay = settings.InitAssetUpdateTypeDelay; + existing.InitBatchAssetUpdateDelay = settings.InitBatchAssetUpdateDelay; + existing.BatchAssetUpdateDelay = settings.BatchAssetUpdateDelay; + existing.CurrentScanningType = settings.CurrentScanningType; + existing.CurrentScanningPage = settings.CurrentScanningPage; + _context.Settings.Update(existing); await _context.SaveChangesAsync(); return existing; } } -} \ No newline at end of file + + /// Inherits documentation from interface. + public async Task UpdateSettingsFromDictionary(Dictionary dictionary) + { + var settings = await GetSettings(); + + foreach (var (key, value) in dictionary) + { + if (string.Equals(key, "TradeRepublicMaxRequestPageSize", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ps)) + settings.TradeRepublicMaxRequestPageSize = ps; + else if (string.Equals(key, "AssetUpdateTypeDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var autd)) + settings.AssetUpdateTypeDelay = autd; + else if (string.Equals(key, "InitAssetUpdateTypeDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var iautd)) + settings.InitAssetUpdateTypeDelay = iautd; + else if (string.Equals(key, "BatchAssetUpdateDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var baud)) + settings.BatchAssetUpdateDelay = baud; + else if (string.Equals(key, "InitBatchAssetUpdateDelay", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ibaud)) + settings.InitBatchAssetUpdateDelay = ibaud; + else if (string.Equals(key, "FinishedInitialScan", StringComparison.OrdinalIgnoreCase) && bool.TryParse(value, out var fis)) + settings.FinishedInitialScan = fis; + } + + await SaveSettings(settings); + } +} diff --git a/FinlyticAssets/Services/TradeRepublicService.cs b/FinlyticAssets/Services/TradeRepublicService.cs deleted file mode 100644 index 58f277d..0000000 --- a/FinlyticAssets/Services/TradeRepublicService.cs +++ /dev/null @@ -1,240 +0,0 @@ -using System.Timers; -using FinlyticAssets.Models; -using FinlyticAssets.Models.DataToObject.TradeRepublic; -using FinlyticAssets.Util; -using FinlyticCore.Models.Assets; - -namespace FinlyticAssets.Services; - -public interface ITradeRepublicService -{ - /// - /// Holt die Anzahl der Assets pro Typ für die Paginierung des Initial-Scans. - /// - public Task GetAssetsCount(CancellationToken cancellationToken = default); - - /// - /// Holt eine spezifische Seite an Assets für den Initial-Scan. - /// - public Task GetAssets(AssetType type, int page, int pageSize, - CancellationToken cancellationToken = default); - - /// - /// Holt die aktuellen Stammdaten für eine spezifische ISIN (Gezieltes Update). - /// Gibt null zurück, wenn das Asset bei TR nicht mehr existiert. - /// - public Task GetAsset(string isin, CancellationToken cancellationToken = default); -} - -/// -/// Provides a managed service to interact with the Trade Republic API via WebSockets, -/// featuring an automatic inactivity timeout to mimic human behavior. -/// -public class TradeRepublicService : ITradeRepublicService, IDisposable -{ - private readonly TradeRepublicClient _client; - private readonly ILogger _logger; - private readonly System.Timers.Timer _inactivityTimer; - private readonly SemaphoreSlim _lock = new(1, 1); - - /// - /// Initializes a new instance of the class. - /// - /// The underlying managed WebSocket client. - /// The logger instance. - public TradeRepublicService(TradeRepublicClient client, ILogger logger) - { - _client = client; - _logger = logger; - - _inactivityTimer = new System.Timers.Timer(TimeSpan.FromSeconds(461).TotalMilliseconds); - _inactivityTimer.AutoReset = false; - _inactivityTimer.Elapsed += OnInactivityTimeout; - } - - /// - /// Ensures that the Trade Republic WebSocket client is connected, initiating a new connection if necessary. - /// Also handles resetting the inactivity timer. - /// - /// A task representing the asynchronous operation. - private async Task EnsureConnectedAsync() - { - await _lock.WaitAsync(); - try - { - _inactivityTimer.Stop(); - - if (!_client.IsConnected) - { - _logger.LogInformation("Trade Republic API is not connected. Establishing automated connection..."); - - // Nutzt den boolschen Rückgabewert von InitAsync - bool connected = await _client.InitAsync(); - - if (connected) - { - _logger.LogInformation("Successfully connected to Trade Republic API."); - } - else - { - _logger.LogWarning("Trade Republic API connection initialization failed (InitAsync returned false)."); - } - } - - _inactivityTimer.Start(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to establish a connection to the Trade Republic API."); - throw; - } - finally - { - _lock.Release(); - } - } - - /// - /// Retrieves the total count of available assets grouped by their types. - /// - /// A token to monitor for cancellation requests. - /// An object containing the metrics. - public async Task GetAssetsCount(CancellationToken cancellationToken = default) - { - await EnsureConnectedAsync(); - - var counts = new AssetsCount(); - - foreach (var type in Enum.GetValues()) - { - var reqData = new TradeRepublicSearchData() - { - Query = "", - Page = 1, - PageSize = 1, - Filter = - [ - new TradeRepublicFilter("type", type.ToString().ToLowerInvariant()), - new TradeRepublicFilter("jurisdiction", "DE"), - ] - }; - - - - var request = new TradeRepublicSearchRequest(Data: reqData); - var response = - await _client.SendRequestAsync(request); - - var count = response?.ResultCount ?? 0; - - counts.SetCountOfType(type, count); - - await Task.Delay(TimeSpan.FromMilliseconds(320), cancellationToken); - } - - return counts; - } - - /// - /// Retrieves a paginated chunk of assets filtered by a specific type. - /// - /// The type of assets to retrieve (e.g., Stock, Etf). - /// The zero-based page index. - /// The number of elements per page. - /// A token to monitor for cancellation requests. - /// A containing the elements, or null if the request fails. - public async Task GetAssets(AssetType type, int page, int pageSize, - CancellationToken cancellationToken = default) - { - await EnsureConnectedAsync(); - - var reqData = new TradeRepublicSearchData() - { - Query = "", - Page = page, - PageSize = pageSize, - Filter = - [ - new TradeRepublicFilter("type", type.ToString().ToLowerInvariant()), - new TradeRepublicFilter("jurisdiction", "DE"), - ] - }; - - var request = new TradeRepublicSearchRequest(Data: reqData); - - return await _client.SendRequestAsync(request); - } - - /// - /// Retrieves the static metadata for a single specific asset via its ISIN. - /// - /// The International Securities Identification Number of the target asset. - /// A token to monitor for cancellation requests. - /// A containing instrument details, or null if the asset is not found. - public async Task GetAsset(string isin, CancellationToken cancellationToken = default) - { - try - { - await EnsureConnectedAsync(); - - - var reqData = new TradeRepublicSearchData() - { - Query = isin, - Page = 1, - PageSize = 1, - Filter = - [ - new TradeRepublicFilter("jurisdiction", "DE"), - ] - }; - - var request = new TradeRepublicSearchRequest(Data: reqData); - - return await _client.SendRequestAsync(request); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error while fetching asset metadata for ISIN {Isin}", isin); - return null; - } - } - - /// - /// Event handler executed when the inactivity timer expires. - /// Gracefully disconnects the WebSocket client. - /// - /// The source of the event. - /// An EventData object that contains the event data. - private async void OnInactivityTimeout(object? sender, ElapsedEventArgs e) - { - try - { - await _lock.WaitAsync(); - - if (!_client.IsConnected) return; - - _logger.LogInformation("No active requests detected for 5 minutes. Automatically disconnecting WebSocket."); - await _client.DisconnectAsync(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error during automatic inactivity disconnect procedure."); - //ignore - } - finally - { - _lock.Release(); - } - } - - /// - /// Disposes the underlying timer and synchronization primitives. - /// - public void Dispose() - { - _inactivityTimer.Dispose(); - _lock.Dispose(); - GC.SuppressFinalize(this); - } -} \ No newline at end of file diff --git a/FinlyticAssets/Util/AssetsMqttClient.cs b/FinlyticAssets/Util/AssetsMqttClient.cs index a6d6e62..c3cfbe2 100644 --- a/FinlyticAssets/Util/AssetsMqttClient.cs +++ b/FinlyticAssets/Util/AssetsMqttClient.cs @@ -1,46 +1,74 @@ -using System.Text.Json; +using System.Text.Json; using FinlyticAssets.Services; using FinlyticCore.Entities.Assets; +using FinlyticCore.Models; using FinlyticCore.Models.Assets; using FinlyticCore.Util; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace FinlyticAssets.Util; /// /// Represents a managed MQTT client acting as a server-side RPC provider within the asset microservice. -/// It subscribes to request topics, processes incoming JSON payloads via the database service, and publishes -/// the requested asset entities back to the corresponding response topic. +/// It subscribes to request topics, processes incoming JSON payloads via the database and index service, +/// and publishes the requested asset entities or logo files back to the response topic. +/// Also implements to manage its own lifecycle connections. /// -public class AssetsMqttClient : ManagedMqttClient +public class AssetsMqttClient( + ILogger logger, + IServiceScopeFactory scopeFactory, + IConfiguration configuration) : ManagedMqttClient(logger), IHostedService { - private readonly IServiceScopeFactory _scopeFactory; - private readonly ILogger _logger; - + + /// - /// Initializes a new instance of the class. + /// Starts the MQTT client and connects to the configured broker. /// - /// The logger used to record connection, error, and status messages. - /// The database service used for querying and validating assets. - public AssetsMqttClient(ILogger logger, IServiceScopeFactory scopeFactory) - : base(logger) + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous start operation. + public async Task StartAsync(CancellationToken cancellationToken) { - _scopeFactory = scopeFactory; - _logger = logger; + var config = new MqttConfiguration() + { + Host = configuration["MQTT:Host"] ?? configuration["MQTT__Host"]!, + Port = Convert.ToInt32(configuration["MQTT:Port"] ?? configuration["MQTT__Port"]!), + ClientId = $"{(configuration["MQTT:ClientId"] ?? configuration["MQTT__ClientId"]!)}_{Guid.NewGuid()}" + }; + + await ConnectAsync(config); + } + + /// + /// Gracefully stops and disconnects the MQTT client. + /// + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous stop operation. + public async Task StopAsync(CancellationToken cancellationToken) + { + await DisconnectAsync(); } /// /// Invoked automatically once the connection to the MQTT broker is successfully established or restored. - /// Registers the required wildcard subscriptions for incoming asset validation and search requests. + /// Registers subscriptions for asset validation, search, and logo download requests. /// /// A representing the asynchronous subscription operation. protected override async Task OnConnectedAsync() { await SubscribeAsync("services/request/assets_Get/#"); await SubscribeAsync("services/request/assets_Search/#"); + await SubscribeAsync("services/request/assets_GetDiscovery/#"); + await SubscribeAsync("services/request/assets_FetchLogo/#"); + await SubscribeAsync("services/request/health_Ping/#"); + await SubscribeAsync("services/config/updated/#"); } + /// - /// Processes incoming messages on the subscribed topics, executes the corresponding database query, + /// Processes incoming messages on the subscribed topics, executes the corresponding service methods, /// and publishes the result to the response topic while preserving the correlation ID. /// /// The MQTT topic on which the message was received. @@ -48,46 +76,144 @@ public class AssetsMqttClient : ManagedMqttClient /// A representing the asynchronous message processing operation. protected override async Task OnMessageReceivedAsync(string topic, string payload) { + if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase)) + { + await HandleConfigUpdatedAsync(topic, payload); + return; + } + var segments = topic.Split('/'); if (segments.Length < 4) return; var channel = segments[2]; - var correlationId = segments[3]; + var correlationId = segments[segments.Length - 1]; + + if (topic.Contains("health_Ping", StringComparison.OrdinalIgnoreCase)) + { + await HandleHealthPingAsync(topic, segments, correlationId); + return; + } try { - List responseData = []; - - using var scope = _scopeFactory.CreateScope(); + using var scope = scopeFactory.CreateScope(); var dbService = scope.ServiceProvider.GetRequiredService(); + var indexService = scope.ServiceProvider.GetRequiredService(); + + if (channel == "assets_FetchLogo") + { + await HandleFetchLogoAsync(payload, correlationId, indexService); + return; + } + + List responseData = []; switch (channel) { case "assets_Get": - var validReq = JsonSerializer.Deserialize(payload); - if (validReq != null) - { - responseData = await dbService.GetValidAssetsByIsinAsync(validReq.Isin); - } + responseData = await HandleAssetsGetAsync(payload, dbService); break; - case "assets_Search": - var searchReq = JsonSerializer.Deserialize(payload); - if (searchReq != null) - { - responseData = await dbService.FindAffectedActiveAssetsAsync(searchReq.SearchQuery); - } + responseData = await HandleAssetsSearchAsync(payload, dbService); + break; + case "assets_GetDiscovery": + responseData = await HandleAssetsGetDiscoveryAsync(payload, dbService); break; } - { - string responseTopic = $"services/response/{channel}/{correlationId}"; - await PublishAsync(responseTopic, responseData.ToDtoList()); - } + string defaultResponseTopic = $"services/response/{channel}/{correlationId}"; + await PublishAsync(defaultResponseTopic, responseData.ToDtoList()); } catch (Exception ex) { OnError(ex); } } -} \ No newline at end of file + + private async Task HandleConfigUpdatedAsync(string topic, string payload) + { + if (!topic.EndsWith("FinlyticAssets", StringComparison.OrdinalIgnoreCase)) + return; + + logger.LogInformation("[{Channel}] [AssetsMqttClient] Received config update event for FinlyticAssets.", "AssetsChannel"); + try + { + var updatePayload = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.ServiceConfigUpdatePayload); + if (updatePayload?.Settings != null && updatePayload.Settings.Count > 0) + { + using var scope = scopeFactory.CreateScope(); + var settingsDb = scope.ServiceProvider.GetRequiredService(); + await settingsDb.UpdateSettingsFromDictionary(updatePayload.Settings); + logger.LogInformation("[{Channel}] [AssetsMqttClient] Persisted {Count} updated settings to FinlyticAssets database.", "AssetsChannel", updatePayload.Settings.Count); + } + } + catch (Exception ex) + { + logger.LogError(ex, "[{Channel}] [AssetsMqttClient] Error processing MQTT config update event.", "AssetsChannel"); + } + } + + private async Task HandleHealthPingAsync(string topic, string[] segments, string correlationId) + { + bool isForMe = segments.Length >= 5 + ? segments[3].Equals("FinlyticAssets", StringComparison.OrdinalIgnoreCase) + : topic.Contains("FinlyticAssets", StringComparison.OrdinalIgnoreCase); + + if (isForMe) + { + string respTopic = $"services/response/health_Ping/{correlationId}"; + await PublishAsync(respTopic, new FinlyticCore.Dtos.ServiceHealthResponse("FinlyticAssets", "Online", DateTime.UtcNow, "Connected")); + logger.LogInformation("[{Channel}] [AssetsMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", "AssetsChannel", correlationId); + } + } + + private async Task HandleFetchLogoAsync(string payload, string correlationId, IAssetsIndexService indexService) + { + var req = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.IsinRequest); + string? isin = req?.Isin; + string? savedPath = null; + + if (!string.IsNullOrEmpty(isin)) + { + savedPath = await indexService.DownloadAndSaveLogoAsync(isin); + } + + string responseTopic = $"services/response/assets_FetchLogo/{correlationId}"; + await PublishAsync(responseTopic, new FinlyticCore.Dtos.FetchLogoResponse(isin, savedPath, savedPath != null)); + } + + private async Task> HandleAssetsGetAsync(string payload, IAssetsDbService dbService) + { + var validReq = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.GetValidAssetRequest); + if (validReq != null) + { + return await dbService.GetValidAssetsByIsinAsync(validReq.Isin); + } + return []; + } + + private async Task> HandleAssetsSearchAsync(string payload, IAssetsDbService dbService) + { + var searchReq = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.SearchAssetsRequest); + if (searchReq != null) + { + return await dbService.FindAffectedActiveAssetsAsync(searchReq.SearchQuery); + } + return []; + } + + private async Task> HandleAssetsGetDiscoveryAsync(string payload, IAssetsDbService dbService) + { + int limit = 15; + if (!string.IsNullOrWhiteSpace(payload)) + { + try + { + var discReq = JsonSerializer.Deserialize(payload, FinlyticJsonSerializerContext.Default.GetDiscoveryAssetsRequest); + if (discReq != null && discReq.Limit > 0) limit = discReq.Limit; + } + catch { } + } + return await dbService.GetDiscoveryAssetsAsync(limit); + } +} diff --git a/FinlyticAssets/Util/StringCodeGenerator.cs b/FinlyticAssets/Util/StringCodeGenerator.cs index 5ddbd27..818ccd5 100644 --- a/FinlyticAssets/Util/StringCodeGenerator.cs +++ b/FinlyticAssets/Util/StringCodeGenerator.cs @@ -1,8 +1,11 @@ -namespace FinlyticAssets.Util; +namespace FinlyticAssets.Util; public class StringCodeGenerator { + /// + /// Generates a W3C traceparent string for telemetry tracking. + /// public static string GenerateTraceparent() { var traceId = Guid.NewGuid().ToString("N"); @@ -11,4 +14,4 @@ public class StringCodeGenerator return $"00-{traceId}-{spanId}-01"; } -} \ No newline at end of file +} diff --git a/FinlyticAssets/Util/TradeRepublicClient.cs b/FinlyticAssets/Util/TradeRepublicClient.cs deleted file mode 100644 index b5dc32c..0000000 --- a/FinlyticAssets/Util/TradeRepublicClient.cs +++ /dev/null @@ -1,252 +0,0 @@ -using System.Collections.Concurrent; -using System.Text.Json; -using FinlyticAssets.Models.DataToObject.TradeRepublic; - -namespace FinlyticAssets.Util; - -/// -/// A managed WebSocket client designed to communicate with the Trade Republic API. -/// Handles asynchronous requests, generic serialization, and automatic subscription management. -/// -public class TradeRepublicClient : ManagedWebSocket -{ - private readonly ILogger _logger; - private int _currentSub; - private readonly ConcurrentDictionary> _pendingRequests = new(); - - /// - /// Wird ausgelöst, wenn Trade Republic asynchrone Updates (z.B. Live-Preise) schickt, - /// auf die niemand aktiv per SendRequestAsync wartet. - /// - public event Action? UnhandledMessageReceived; - - /// - /// Wird ausgelöst, wenn Trade Republic Systemnachrichten oder Fehler ohne ID schickt. - /// - public event Action? SystemMessageReceived; - - /// - /// Initializes a new instance of the TradeRepublicClient. - /// Call InitAsync() afterwards to establish the connection. - /// - /// The logger instance for tracking socket events and errors. - public TradeRepublicClient(ILogger logger) - { - _logger = logger; - } - - /// - /// Asynchronously establishes the WebSocket connection to the Trade Republic API. - /// - public async Task InitAsync() - { - await ConnectAsync("wss://api.traderepublic.com/", TimeSpan.FromSeconds(10)); - - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - _pendingRequests.TryAdd(-1, tcs); - - try - { - var json = JsonSerializer.Serialize(new TradeRepublicConnectRequest()); - await SendAsync($"connect 34 {json}"); - - var res = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(5)); - - if (string.IsNullOrWhiteSpace(res.Type)) return false; - - var isConnected = res.Type == "connected"; - - if (isConnected) - { - _logger.LogInformation("WebSocket connection to Trade Republic established."); - } - - return isConnected; - } - catch (TimeoutException) - { - _pendingRequests.TryRemove(-1, out _); - _logger.LogWarning("Timeout while waiting for response to ID {Id}.", -1); - return false; - } - catch (TaskCanceledException) - { - _logger.LogWarning( - "Trade Republic immediately rejected the request for ID {Id} (e.g., invalid ISIN or access denied).", - -1); - return false; - } - } - - /// - /// Sends a strongly-typed request to the API and waits for the corresponding response. - /// Automatically handles the subscription ID and unsubscribes after completion or failure. - /// - /// The expected type of the response payload. - /// The type of the request payload. - /// The request data to be serialized and sent. - /// The deserialized response object, or null if the request timed out or was canceled. - public async Task SendRequestAsync(TRequest request) - where TResponse : class where TRequest : class - { - var tempSub = Interlocked.Increment(ref _currentSub); - - var msg = $"sub {tempSub} {JsonSerializer.Serialize(request)}"; - - Console.WriteLine(msg); - - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - _pendingRequests.TryAdd(tempSub, tcs); - - await SendAsync(msg); - - try - { - var res = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(5)); - - if (res.Type == null) - { - _logger.LogWarning("Trade Republic rejected the request for ID {Id} with message type '{Type}'.", tempSub, res.Type); - return null; - } - - if (!res.Type.Contains('A')) - { - return null; - } - - if (string.IsNullOrWhiteSpace(res.Data)) return null; - - if (typeof(TResponse) == typeof(string)) - { - return res.Data as TResponse; - } - else - { - return JsonSerializer.Deserialize(res.Data); - } - } - catch (TimeoutException) - { - _pendingRequests.TryRemove(tempSub, out _); - _logger.LogWarning("Timeout while waiting for response to ID {Id}.", tempSub); - return null; - } - catch (TaskCanceledException) - { - _logger.LogWarning( - "Trade Republic immediately rejected the request for ID {Id} (e.g., invalid ISIN or access denied).", - tempSub); - return null; - } - finally - { - if (IsConnected) - { - try - { - await SendAsync($"unsub {tempSub}"); - } - catch - { - //ignore - } - } - } - } -/// -/// Processes incoming WebSocket messages, extracting the JSON payload and resolving pending tasks. -/// -/// The raw text message received from the server. -protected override void OnMessageReceived(string message) -{ - if (string.IsNullOrWhiteSpace(message)) return; - - // 1. Handshake-Nachricht direkt abfangen - if (message == "connected") - { - if (_pendingRequests.TryRemove(-1, out var tcs)) - { - tcs.SetResult(new ReceivedMessage(-1, "connected", null)); - } - return; - } - - // 2. Erstes Leerzeichen finden, um die ID zu isolieren - var firstSpaceIndex = message.IndexOf(' '); - if (firstSpaceIndex <= 0) - { - _logger.LogWarning("Unknown message format received: {Message}", message); - SystemMessageReceived?.Invoke(message); - return; - } - - var idString = message[..firstSpaceIndex]; - if (!int.TryParse(idString, out var responseId)) - { - SystemMessageReceived?.Invoke(message); - return; - } - - // Der Rest nach der ID (z. B. "A {...}" oder "C") - var remainder = message[firstSpaceIndex..].Trim(); - - // 3. Nachrichtentyp ("A", "C", etc.) und JSON-Inhalt sauber trennen - var nextSpaceIndex = remainder.IndexOf(' '); - string msgType; - string? json = null; - - if (nextSpaceIndex == -1) - { - // Kein weiteres Leerzeichen vorhanden (wie bei "2 C") - msgType = remainder; - } - else - { - // Typ und JSON trennen (wie bei "2 A {...}") - msgType = remainder[..nextSpaceIndex].Trim(); - json = remainder[nextSpaceIndex..].Trim(); - } - - // 4. KORREKTUR: "C" signalisiert nur das Ende des Datenstroms auf dieser ID. - // Wir ignorieren es, da die Daten bereits im Typ "A" übertragen wurden. - if (msgType == "C") - { - _logger.LogDebug("Trade Republic closed subscription channel for ID {ResponseId}.", responseId); - return; - } - - // 5. Task auflösen, falls jemand auf diese ID wartet - if (_pendingRequests.TryRemove(responseId, out var pendingTcs)) - { - pendingTcs.SetResult(new ReceivedMessage(responseId, msgType, json)); - } - else - { - UnhandledMessageReceived?.Invoke(new ReceivedMessage(responseId, msgType, json)); - } -} - - /// - /// Determines whether the incoming message is a keep-alive echo response. - /// - /// The raw text message. - /// True if the message is an echo response; otherwise, false. - protected override bool IsKeepAliveMessage(string message) - { - return message.StartsWith("echo"); - } - - /// - /// Sends a periodic keep-alive echo to maintain the WebSocket connection. - /// - protected override Task SendLifeMessageAsync() - { - var echo = $"echo {DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - return SendAsync(echo); - } -} - -public record ReceivedMessage(int? Sub, string? Type, string? Data); \ No newline at end of file diff --git a/FinlyticAssets/Util/Volumes.cs b/FinlyticAssets/Util/Volumes.cs index aa71727..ba36fb2 100644 --- a/FinlyticAssets/Util/Volumes.cs +++ b/FinlyticAssets/Util/Volumes.cs @@ -1,4 +1,4 @@ -namespace FinlyticAssets.Util; +namespace FinlyticAssets.Util; public class Volumes { @@ -6,4 +6,9 @@ public class Volumes /// Der relative Pfad für die schlanke Index-Datei (ISINs + Namen) zur Asset-Erkennung. /// public const string IndexRelativePath = "assets/index"; -} \ No newline at end of file + + /// + /// Der relative Pfad für lokal gespeicherte Asset-Logos (nach ISIN benannt). + /// + public const string LogosRelativePath = "assets/logos"; +} diff --git a/FinlyticAssets/appsettings.json b/FinlyticAssets/appsettings.json index b2dcdb6..4e8c9f7 100644 --- a/FinlyticAssets/appsettings.json +++ b/FinlyticAssets/appsettings.json @@ -2,7 +2,8 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.Hosting.Lifetime": "Information" + "Microsoft.Hosting.Lifetime": "Information", + "Microsoft.EntityFrameworkCore.Database.Command": "Warning" } } }