From 6337e63a77a4d05bfb68e14f750006387dc90bc0 Mon Sep 17 00:00:00 2001 From: Kleidukos Date: Sun, 19 Jul 2026 12:13:14 +0200 Subject: [PATCH] Init --- .dockerignore | 25 + .gitignore | 32 ++ Finlytic.sln | 33 ++ FinlyticAssets/Database/AssetsDbContext.cs | 79 ++++ FinlyticAssets/Dockerfile | 21 + FinlyticAssets/Entities/Settings.cs | 88 ++++ FinlyticAssets/FinlyticAssets.csproj | 37 ++ .../20260628184437_Init.Designer.cs | 285 ++++++++++++ .../Migrations/20260628184437_Init.cs | 126 +++++ ...260628195842_AddedScannerState.Designer.cs | 293 ++++++++++++ .../20260628195842_AddedScannerState.cs | 41 ++ .../20260628204647_FixIsinKey.Designer.cs | 295 ++++++++++++ .../Migrations/20260628204647_FixIsinKey.cs | 107 +++++ .../AssetsDbContextModelSnapshot.cs | 292 ++++++++++++ FinlyticAssets/Models/AssetIndex.cs | 7 + FinlyticAssets/Models/AssetsCount.cs | 39 ++ .../TradeRepublicAssetResponse.cs | 139 ++++++ .../TradeRepublicConnectRequest.cs | 16 + .../TradeRepublic/TradeRepublicHeaders.cs | 12 + .../TradeRepublicSearchRequest.cs | 30 ++ FinlyticAssets/Program.cs | 43 ++ FinlyticAssets/Properties/launchSettings.json | 12 + FinlyticAssets/Services/AssetsDbService.cs | 437 ++++++++++++++++++ .../Services/AssetsFullScanService.cs | 231 +++++++++ FinlyticAssets/Services/AssetsIndexService.cs | 87 ++++ .../Services/MqttConnectionService.cs | 52 +++ FinlyticAssets/Services/SettingsDbService.cs | 91 ++++ .../Services/TradeRepublicService.cs | 240 ++++++++++ FinlyticAssets/Util/AssetsMqttClient.cs | 93 ++++ FinlyticAssets/Util/StringCodeGenerator.cs | 14 + FinlyticAssets/Util/TradeRepublicClient.cs | 252 ++++++++++ FinlyticAssets/Util/Volumes.cs | 9 + FinlyticAssets/appsettings.json | 8 + FinlyticCore/FinlyticCore.csproj | 14 + FinlyticCore/Models/MqttConfiguration.cs | 32 ++ FinlyticCore/Util/AssetMapper.cs | 65 +++ FinlyticCore/Util/ManagedMqttClient.cs | 307 ++++++++++++ FinlyticCore/Util/ManagedWebSocket.cs | 234 ++++++++++ FinlyticNews/Dockerfile | 21 + FinlyticNews/FinlyticNews.csproj | 26 ++ FinlyticNews/Program.cs | 5 + FinlyticNews/Properties/launchSettings.json | 12 + FinlyticNews/appsettings.json | 8 + Project.md | 66 +++ Rules.md | 20 + compose.yaml | 35 ++ 46 files changed, 4411 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Finlytic.sln create mode 100644 FinlyticAssets/Database/AssetsDbContext.cs create mode 100644 FinlyticAssets/Dockerfile create mode 100644 FinlyticAssets/Entities/Settings.cs create mode 100644 FinlyticAssets/FinlyticAssets.csproj create mode 100644 FinlyticAssets/Migrations/20260628184437_Init.Designer.cs create mode 100644 FinlyticAssets/Migrations/20260628184437_Init.cs create mode 100644 FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs create mode 100644 FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs create mode 100644 FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs create mode 100644 FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs create mode 100644 FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs create mode 100644 FinlyticAssets/Models/AssetIndex.cs create mode 100644 FinlyticAssets/Models/AssetsCount.cs create mode 100644 FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs create mode 100644 FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs create mode 100644 FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs create mode 100644 FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs create mode 100644 FinlyticAssets/Program.cs create mode 100644 FinlyticAssets/Properties/launchSettings.json create mode 100644 FinlyticAssets/Services/AssetsDbService.cs create mode 100644 FinlyticAssets/Services/AssetsFullScanService.cs create mode 100644 FinlyticAssets/Services/AssetsIndexService.cs create mode 100644 FinlyticAssets/Services/MqttConnectionService.cs create mode 100644 FinlyticAssets/Services/SettingsDbService.cs create mode 100644 FinlyticAssets/Services/TradeRepublicService.cs create mode 100644 FinlyticAssets/Util/AssetsMqttClient.cs create mode 100644 FinlyticAssets/Util/StringCodeGenerator.cs create mode 100644 FinlyticAssets/Util/TradeRepublicClient.cs create mode 100644 FinlyticAssets/Util/Volumes.cs create mode 100644 FinlyticAssets/appsettings.json create mode 100644 FinlyticCore/FinlyticCore.csproj create mode 100644 FinlyticCore/Models/MqttConfiguration.cs create mode 100644 FinlyticCore/Util/AssetMapper.cs create mode 100644 FinlyticCore/Util/ManagedMqttClient.cs create mode 100644 FinlyticCore/Util/ManagedWebSocket.cs create mode 100644 FinlyticNews/Dockerfile create mode 100644 FinlyticNews/FinlyticNews.csproj create mode 100644 FinlyticNews/Program.cs create mode 100644 FinlyticNews/Properties/launchSettings.json create mode 100644 FinlyticNews/appsettings.json create mode 100644 Project.md create mode 100644 Rules.md create mode 100644 compose.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cd967fc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.idea +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ac79b40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +## Build outputs +**/bin/ +**/obj/ + +## Rider / JetBrains / VS Code / Visual Studio +.idea/ +.vs/ +.vscode/ +*.sln.DotSettings.user +*.csproj.user +_ReSharper.*/ +*.resharper.user + +## OS Files +.DS_Store +Thumbs.db +desktop.ini + +## Local assets volume & generated cache files +assets/ +**/assets/index/ + +## Secrets & local environment files +.env +*.env.local +appsettings.Development.json + +## User-specific files +*.user +*.suo +*.userosscache +*.sln.docstates diff --git a/Finlytic.sln b/Finlytic.sln new file mode 100644 index 0000000..4b4788a --- /dev/null +++ b/Finlytic.sln @@ -0,0 +1,33 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticAssets", "FinlyticAssets\FinlyticAssets.csproj", "{B5AC2C29-44D5-4538-815F-F02CDDE9D01F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{F6AC7FB2-7669-42B3-8155-7E6F76873C21}" + ProjectSection(SolutionItems) = preProject + compose.yaml = compose.yaml + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticCore", "FinlyticCore\FinlyticCore.csproj", "{03E2ECBB-0FF9-43E4-994E-F6A522860AD5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticNews", "FinlyticNews\FinlyticNews.csproj", "{03B4D920-6173-44E8-A1E2-8945D8393CEA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|Any CPU.Build.0 = Release|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|Any CPU.Build.0 = Release|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/FinlyticAssets/Database/AssetsDbContext.cs b/FinlyticAssets/Database/AssetsDbContext.cs new file mode 100644 index 0000000..d1e59cf --- /dev/null +++ b/FinlyticAssets/Database/AssetsDbContext.cs @@ -0,0 +1,79 @@ +using FinlyticAssets.Entities; +using FinlyticCore.Entities.Assets; +using FinlyticCore.Models.Assets; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; + +namespace FinlyticAssets.Database; + +public class AssetsDbContext : DbContext +{ + public AssetsDbContext(DbContextOptions options) : base(options) + { + } + + public DbSet Settings { get; set; } + public DbSet TradeRepublicAssets { get; set; } + public DbSet TradeRepublicTags { get; set; } + + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.HasKey(e => new {e.Isin, e.InstrumentCategory}); + + entity.HasDiscriminator("AssetType") + .HasValue("Stock") + .HasValue("Crypto") + .HasValue("Etf") + .HasValue("Synthetic") + .HasValue("Bond") + .HasValue("Derivative"); + }); + + var stringListConverter = + new Microsoft.EntityFrameworkCore.Storage.ValueConversion.ValueConverter, string>( + v => System.Text.Json.JsonSerializer.Serialize(v, (System.Text.Json.JsonSerializerOptions?)null), + v => System.Text.Json.JsonSerializer.Deserialize>(v, + (System.Text.Json.JsonSerializerOptions?)null) ?? new List() + ); + + var stringListComparer = new ValueComparer>( + (c1, c2) => c1 != null && c2 != null ? c1.SequenceEqual(c2) : c1 == c2, + c => c.Aggregate(0, (a, v) => HashCode.Combine(a, v.GetHashCode())), + c => c.ToList() + ); + + modelBuilder.Entity() + .Property(e => e.DerivativeProductCategories) + .HasConversion(stringListConverter, stringListComparer); + + modelBuilder.Entity() + .Property(e => e.DerivativeProductCategories) + .HasConversion(stringListConverter, stringListComparer); + + modelBuilder.Entity() + .Property(e => e.DerivativeProductCategories) + .HasConversion(stringListConverter, stringListComparer); + + modelBuilder.Entity() + .Property(e => e.DerivativeProductCategories) + .HasConversion(stringListConverter, stringListComparer); + + modelBuilder.Entity(entity => { entity.HasKey(e => e.Id); }); + + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Id); + + entity.Property(e => e.CurrentScanningType) + .HasConversion() + .HasMaxLength(50); + }); + + modelBuilder.Entity() + .HasMany(a => a.Tags) + .WithMany(t => t.Assets); + } +} \ No newline at end of file diff --git a/FinlyticAssets/Dockerfile b/FinlyticAssets/Dockerfile new file mode 100644 index 0000000..7f498f5 --- /dev/null +++ b/FinlyticAssets/Dockerfile @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/dotnet/runtime:10.0 AS base +USER $APP_UID +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["FinlyticAssets/FinlyticAssets.csproj", "FinlyticAssets/"] +RUN dotnet restore "FinlyticAssets/FinlyticAssets.csproj" +COPY . . +WORKDIR "/src/FinlyticAssets" +RUN dotnet build "./FinlyticAssets.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./FinlyticAssets.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "FinlyticAssets.dll"] diff --git a/FinlyticAssets/Entities/Settings.cs b/FinlyticAssets/Entities/Settings.cs new file mode 100644 index 0000000..3a1f609 --- /dev/null +++ b/FinlyticAssets/Entities/Settings.cs @@ -0,0 +1,88 @@ +using System; +using System.ComponentModel.DataAnnotations; +using FinlyticAssets.Models; +using FinlyticCore.Models.Assets; + +namespace FinlyticAssets.Entities; + +/// +/// Represents the global synchronization and timing configurations for the Trade Republic asset scanner. +/// +public class Settings +{ + /// + /// Gets or sets the unique identifier for the settings record. + /// + [Key] + public Guid Id { get; set; } + + /// + /// Gets or sets a value indicating whether the very first full scan of all assets has been completed. + /// Used to switch from fast initial discovery delays to stealthy incremental update delays. + /// + public bool FinishedInitialScan { get; set; } + + /// + /// Gets or sets the minimum number of days to wait before an asset becomes eligible for re-validation. + /// Combined with to achieve a flat 2-to-3-month rotation cycle. + /// + public int MinRandomUpdateDay { get; set; } = 60; + + /// + /// Gets or sets the maximum number of days to wait before an asset must be re-validated (roughly 3 months). + /// + public int MaxRandomUpdateDay { get; set; } = 90; + + /// + /// Gets or sets the earliest hour (0-23) of the day when background synchronization is allowed to execute. + /// Prevents unusual nocturnal API traffic. + /// + public int UpdateDayTimeStart { get; set; } = 8; + + /// + /// Gets or sets the latest hour (0-23) of the day when background synchronization is allowed to execute. + /// + public int UpdateDayTimeStop { get; set; } = 21; + + /// + /// Gets or sets the maximum number of assets requested per single API pagination call. + /// Values around 100 look like standard dynamic-scrolling payloads from a real client device. + /// + public int TradeRepublicMaxRequestPageSize { get; set; } = 100; + + /// + /// Gets or sets the idle delay in seconds between switching from one full asset category to another (e.g., from Stocks to Crypto) + /// during standard maintenance mode. (Default 12000s = ~3.3 hours). + /// + public int AssetUpdateTypeDelay { get; set; } = 12000; + + /// + /// Gets or sets the idle delay in seconds between switching asset categories during the initial setup scan. + /// Faster than standard mode but kept high enough to prevent early rate limiting. (Default 3600s = 1 hour). + /// + public int InitAssetUpdateTypeDelay { get; set; } = 3600; + + /// + /// Gets or sets the baseline delay in seconds between sequential page requests of the same asset type during the initial setup scan. + /// (Default 120s = 2 minutes). + /// + public int InitBatchAssetUpdateDelay { get; set; } = 120; + + /// + /// Gets or sets the standard baseline delay in seconds between sequential page requests of the same asset type during recurring incremental updates. + /// Spreads pagination widely over time to blend into regular human traffic profiles. (Default 600s = 10 minutes). + /// + public int BatchAssetUpdateDelay { get; set; } = 600; + + /// + /// Gets or sets the asset type that is currently being processed by the full scan. + /// Acts as a live pointer for recovery after a service interruption. + /// + public AssetType CurrentScanningType { get; set; } = AssetType.Stock; + + /// + /// Gets or sets the page number of the that is currently being fetched or was just processed. + /// Trade Republic uses 1-based pagination. A value of 0 means the scan is currently idle or between types. + /// + public int CurrentScanningPage { get; set; } = 0; +} \ No newline at end of file diff --git a/FinlyticAssets/FinlyticAssets.csproj b/FinlyticAssets/FinlyticAssets.csproj new file mode 100644 index 0000000..e89e094 --- /dev/null +++ b/FinlyticAssets/FinlyticAssets.csproj @@ -0,0 +1,37 @@ + + + + net10.0 + enable + enable + dotnet-FinlyticAssets-a64604ac-cbde-4284-838a-2173f640d19a + Linux + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + .dockerignore + + + + + + + diff --git a/FinlyticAssets/Migrations/20260628184437_Init.Designer.cs b/FinlyticAssets/Migrations/20260628184437_Init.Designer.cs new file mode 100644 index 0000000..a89b6a3 --- /dev/null +++ b/FinlyticAssets/Migrations/20260628184437_Init.Designer.cs @@ -0,0 +1,285 @@ +// +using System; +using FinlyticAssets.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + [DbContext(typeof(AssetsDbContext))] + [Migration("20260628184437_Init")] + partial class Init + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.Property("AssetsIsin") + .HasColumnType("text"); + + b.Property("TagsId") + .HasColumnType("text"); + + b.HasKey("AssetsIsin", "TagsId"); + + b.HasIndex("TagsId"); + + b.ToTable("AssetEntityTagEntity"); + }); + + modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("BatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("FinishedInitialScan") + .HasColumnType("boolean"); + + b.Property("InitAssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("InitBatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("MaxRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("MinRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("TradeRepublicMaxRequestPageSize") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStart") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStop") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("AssetType") + .IsRequired() + .HasMaxLength(13) + .HasColumnType("character varying(13)"); + + b.Property("HasCfd") + .HasColumnType("boolean"); + + b.Property("ImageId") + .HasColumnType("text"); + + b.Property("InstrumentCategory") + .IsRequired() + .HasColumnType("text"); + + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Isin"); + + b.ToTable("TradeRepublicAssets"); + + b.HasDiscriminator("AssetType").HasValue("AssetEntity"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("TradeRepublicTags"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("BondIssuerName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Bond"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("SearchSubtitle") + .HasColumnName("CryptoEntity_SearchSubtitle"); + }); + + b.HasDiscriminator().HasValue("Crypto"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("UnderlyingIsin") + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Derivative"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("EtfDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("MappedEtfIndexName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("EtfEntity_DerivativeProductCategories"); + + t.Property("SearchSubtitle") + .HasColumnName("EtfEntity_SearchSubtitle"); + + t.Property("Subtitle") + .HasColumnName("EtfEntity_Subtitle"); + }); + + b.HasDiscriminator().HasValue("Etf"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("StockEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Stock"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("SyntheticEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Synthetic"); + }); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) + .WithMany() + .HasForeignKey("AssetsIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticAssets/Migrations/20260628184437_Init.cs b/FinlyticAssets/Migrations/20260628184437_Init.cs new file mode 100644 index 0000000..8edc236 --- /dev/null +++ b/FinlyticAssets/Migrations/20260628184437_Init.cs @@ -0,0 +1,126 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + /// + public partial class Init : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "Settings", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + FinishedInitialScan = table.Column(type: "boolean", nullable: false), + MinRandomUpdateDay = table.Column(type: "integer", nullable: false), + MaxRandomUpdateDay = table.Column(type: "integer", nullable: false), + UpdateDayTimeStart = table.Column(type: "integer", nullable: false), + UpdateDayTimeStop = table.Column(type: "integer", nullable: false), + TradeRepublicMaxRequestPageSize = table.Column(type: "integer", nullable: false), + AssetUpdateTypeDelay = table.Column(type: "integer", nullable: false), + InitAssetUpdateTypeDelay = table.Column(type: "integer", nullable: false), + InitBatchAssetUpdateDelay = table.Column(type: "integer", nullable: false), + BatchAssetUpdateDelay = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Settings", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "TradeRepublicAssets", + columns: table => new + { + Isin = table.Column(type: "text", nullable: false), + Name = table.Column(type: "text", nullable: false), + Type = table.Column(type: "text", nullable: false), + InstrumentCategory = table.Column(type: "text", nullable: false), + HasCfd = table.Column(type: "boolean", nullable: false), + ImageId = table.Column(type: "text", nullable: true), + UpdateAt = table.Column(type: "timestamp with time zone", nullable: false), + LastUpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), + AssetType = table.Column(type: "character varying(13)", maxLength: 13, nullable: false), + BondIssuerName = table.Column(type: "text", nullable: true), + SearchSubtitle = table.Column(type: "text", nullable: true), + Subtitle = table.Column(type: "text", nullable: true), + CryptoEntity_SearchSubtitle = table.Column(type: "text", nullable: true), + DerivativeProductCategories = table.Column(type: "text", nullable: true), + UnderlyingIsin = table.Column(type: "text", nullable: true), + EtfEntity_DerivativeProductCategories = table.Column(type: "text", nullable: true), + EtfDescription = table.Column(type: "text", nullable: true), + MappedEtfIndexName = table.Column(type: "text", nullable: true), + EtfEntity_Subtitle = table.Column(type: "text", nullable: true), + EtfEntity_SearchSubtitle = table.Column(type: "text", nullable: true), + StockEntity_DerivativeProductCategories = table.Column(type: "text", nullable: true), + SyntheticEntity_DerivativeProductCategories = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_TradeRepublicAssets", x => x.Isin); + }); + + migrationBuilder.CreateTable( + name: "TradeRepublicTags", + columns: table => new + { + Id = table.Column(type: "text", nullable: false), + Name = table.Column(type: "text", nullable: false), + Type = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TradeRepublicTags", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "AssetEntityTagEntity", + columns: table => new + { + AssetsIsin = table.Column(type: "text", nullable: false), + TagsId = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AssetEntityTagEntity", x => new { x.AssetsIsin, x.TagsId }); + table.ForeignKey( + name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin", + column: x => x.AssetsIsin, + principalTable: "TradeRepublicAssets", + principalColumn: "Isin", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AssetEntityTagEntity_TradeRepublicTags_TagsId", + column: x => x.TagsId, + principalTable: "TradeRepublicTags", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_AssetEntityTagEntity_TagsId", + table: "AssetEntityTagEntity", + column: "TagsId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AssetEntityTagEntity"); + + migrationBuilder.DropTable( + name: "Settings"); + + migrationBuilder.DropTable( + name: "TradeRepublicAssets"); + + migrationBuilder.DropTable( + name: "TradeRepublicTags"); + } + } +} diff --git a/FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs b/FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs new file mode 100644 index 0000000..f9d85d2 --- /dev/null +++ b/FinlyticAssets/Migrations/20260628195842_AddedScannerState.Designer.cs @@ -0,0 +1,293 @@ +// +using System; +using FinlyticAssets.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + [DbContext(typeof(AssetsDbContext))] + [Migration("20260628195842_AddedScannerState")] + partial class AddedScannerState + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.Property("AssetsIsin") + .HasColumnType("text"); + + b.Property("TagsId") + .HasColumnType("text"); + + b.HasKey("AssetsIsin", "TagsId"); + + b.HasIndex("TagsId"); + + b.ToTable("AssetEntityTagEntity"); + }); + + modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("BatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("CurrentScanningPage") + .HasColumnType("integer"); + + b.Property("CurrentScanningType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("FinishedInitialScan") + .HasColumnType("boolean"); + + b.Property("InitAssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("InitBatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("MaxRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("MinRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("TradeRepublicMaxRequestPageSize") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStart") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStop") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("AssetType") + .IsRequired() + .HasMaxLength(13) + .HasColumnType("character varying(13)"); + + b.Property("HasCfd") + .HasColumnType("boolean"); + + b.Property("ImageId") + .HasColumnType("text"); + + b.Property("InstrumentCategory") + .IsRequired() + .HasColumnType("text"); + + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Isin"); + + b.ToTable("TradeRepublicAssets"); + + b.HasDiscriminator("AssetType").HasValue("AssetEntity"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("TradeRepublicTags"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("BondIssuerName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Bond"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("SearchSubtitle") + .HasColumnName("CryptoEntity_SearchSubtitle"); + }); + + b.HasDiscriminator().HasValue("Crypto"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("UnderlyingIsin") + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Derivative"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("EtfDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("MappedEtfIndexName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("EtfEntity_DerivativeProductCategories"); + + t.Property("SearchSubtitle") + .HasColumnName("EtfEntity_SearchSubtitle"); + + t.Property("Subtitle") + .HasColumnName("EtfEntity_Subtitle"); + }); + + b.HasDiscriminator().HasValue("Etf"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("StockEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Stock"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("SyntheticEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Synthetic"); + }); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) + .WithMany() + .HasForeignKey("AssetsIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs b/FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs new file mode 100644 index 0000000..fec561b --- /dev/null +++ b/FinlyticAssets/Migrations/20260628195842_AddedScannerState.cs @@ -0,0 +1,41 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + /// + public partial class AddedScannerState : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "CurrentScanningPage", + table: "Settings", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "CurrentScanningType", + table: "Settings", + type: "character varying(50)", + maxLength: 50, + nullable: false, + defaultValue: ""); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "CurrentScanningPage", + table: "Settings"); + + migrationBuilder.DropColumn( + name: "CurrentScanningType", + table: "Settings"); + } + } +} diff --git a/FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs b/FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs new file mode 100644 index 0000000..79fb20b --- /dev/null +++ b/FinlyticAssets/Migrations/20260628204647_FixIsinKey.Designer.cs @@ -0,0 +1,295 @@ +// +using System; +using FinlyticAssets.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + [DbContext(typeof(AssetsDbContext))] + [Migration("20260628204647_FixIsinKey")] + partial class FixIsinKey + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.Property("TagsId") + .HasColumnType("text"); + + b.Property("AssetsIsin") + .HasColumnType("text"); + + b.Property("AssetsInstrumentCategory") + .HasColumnType("text"); + + b.HasKey("TagsId", "AssetsIsin", "AssetsInstrumentCategory"); + + b.HasIndex("AssetsIsin", "AssetsInstrumentCategory"); + + b.ToTable("AssetEntityTagEntity"); + }); + + modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("BatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("CurrentScanningPage") + .HasColumnType("integer"); + + b.Property("CurrentScanningType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("FinishedInitialScan") + .HasColumnType("boolean"); + + b.Property("InitAssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("InitBatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("MaxRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("MinRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("TradeRepublicMaxRequestPageSize") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStart") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStop") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("InstrumentCategory") + .HasColumnType("text"); + + b.Property("AssetType") + .IsRequired() + .HasMaxLength(13) + .HasColumnType("character varying(13)"); + + b.Property("HasCfd") + .HasColumnType("boolean"); + + b.Property("ImageId") + .HasColumnType("text"); + + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Isin", "InstrumentCategory"); + + b.ToTable("TradeRepublicAssets"); + + b.HasDiscriminator("AssetType").HasValue("AssetEntity"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("TradeRepublicTags"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("BondIssuerName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Bond"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("SearchSubtitle") + .HasColumnName("CryptoEntity_SearchSubtitle"); + }); + + b.HasDiscriminator().HasValue("Crypto"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("UnderlyingIsin") + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Derivative"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("EtfDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("MappedEtfIndexName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("EtfEntity_DerivativeProductCategories"); + + t.Property("SearchSubtitle") + .HasColumnName("EtfEntity_SearchSubtitle"); + + t.Property("Subtitle") + .HasColumnName("EtfEntity_Subtitle"); + }); + + b.HasDiscriminator().HasValue("Etf"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("StockEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Stock"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("SyntheticEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Synthetic"); + }); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) + .WithMany() + .HasForeignKey("AssetsIsin", "AssetsInstrumentCategory") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs b/FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs new file mode 100644 index 0000000..eb123c2 --- /dev/null +++ b/FinlyticAssets/Migrations/20260628204647_FixIsinKey.cs @@ -0,0 +1,107 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + /// + public partial class FixIsinKey : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin", + table: "AssetEntityTagEntity"); + + migrationBuilder.DropPrimaryKey( + name: "PK_TradeRepublicAssets", + table: "TradeRepublicAssets"); + + migrationBuilder.DropPrimaryKey( + name: "PK_AssetEntityTagEntity", + table: "AssetEntityTagEntity"); + + migrationBuilder.DropIndex( + name: "IX_AssetEntityTagEntity_TagsId", + table: "AssetEntityTagEntity"); + + migrationBuilder.AddColumn( + name: "AssetsInstrumentCategory", + table: "AssetEntityTagEntity", + type: "text", + nullable: false, + defaultValue: ""); + + migrationBuilder.AddPrimaryKey( + name: "PK_TradeRepublicAssets", + table: "TradeRepublicAssets", + columns: new[] { "Isin", "InstrumentCategory" }); + + migrationBuilder.AddPrimaryKey( + name: "PK_AssetEntityTagEntity", + table: "AssetEntityTagEntity", + columns: new[] { "TagsId", "AssetsIsin", "AssetsInstrumentCategory" }); + + migrationBuilder.CreateIndex( + name: "IX_AssetEntityTagEntity_AssetsIsin_AssetsInstrumentCategory", + table: "AssetEntityTagEntity", + columns: new[] { "AssetsIsin", "AssetsInstrumentCategory" }); + + migrationBuilder.AddForeignKey( + name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin_AssetsI~", + table: "AssetEntityTagEntity", + columns: new[] { "AssetsIsin", "AssetsInstrumentCategory" }, + principalTable: "TradeRepublicAssets", + principalColumns: new[] { "Isin", "InstrumentCategory" }, + onDelete: ReferentialAction.Cascade); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin_AssetsI~", + table: "AssetEntityTagEntity"); + + migrationBuilder.DropPrimaryKey( + name: "PK_TradeRepublicAssets", + table: "TradeRepublicAssets"); + + migrationBuilder.DropPrimaryKey( + name: "PK_AssetEntityTagEntity", + table: "AssetEntityTagEntity"); + + migrationBuilder.DropIndex( + name: "IX_AssetEntityTagEntity_AssetsIsin_AssetsInstrumentCategory", + table: "AssetEntityTagEntity"); + + migrationBuilder.DropColumn( + name: "AssetsInstrumentCategory", + table: "AssetEntityTagEntity"); + + migrationBuilder.AddPrimaryKey( + name: "PK_TradeRepublicAssets", + table: "TradeRepublicAssets", + column: "Isin"); + + migrationBuilder.AddPrimaryKey( + name: "PK_AssetEntityTagEntity", + table: "AssetEntityTagEntity", + columns: new[] { "AssetsIsin", "TagsId" }); + + migrationBuilder.CreateIndex( + name: "IX_AssetEntityTagEntity_TagsId", + table: "AssetEntityTagEntity", + column: "TagsId"); + + migrationBuilder.AddForeignKey( + name: "FK_AssetEntityTagEntity_TradeRepublicAssets_AssetsIsin", + table: "AssetEntityTagEntity", + column: "AssetsIsin", + principalTable: "TradeRepublicAssets", + principalColumn: "Isin", + onDelete: ReferentialAction.Cascade); + } + } +} diff --git a/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs b/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs new file mode 100644 index 0000000..b2c32be --- /dev/null +++ b/FinlyticAssets/Migrations/AssetsDbContextModelSnapshot.cs @@ -0,0 +1,292 @@ +// +using System; +using FinlyticAssets.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticAssets.Migrations +{ + [DbContext(typeof(AssetsDbContext))] + partial class AssetsDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.Property("TagsId") + .HasColumnType("text"); + + b.Property("AssetsIsin") + .HasColumnType("text"); + + b.Property("AssetsInstrumentCategory") + .HasColumnType("text"); + + b.HasKey("TagsId", "AssetsIsin", "AssetsInstrumentCategory"); + + b.HasIndex("AssetsIsin", "AssetsInstrumentCategory"); + + b.ToTable("AssetEntityTagEntity"); + }); + + modelBuilder.Entity("FinlyticAssets.Entities.Settings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("BatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("CurrentScanningPage") + .HasColumnType("integer"); + + b.Property("CurrentScanningType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("FinishedInitialScan") + .HasColumnType("boolean"); + + b.Property("InitAssetUpdateTypeDelay") + .HasColumnType("integer"); + + b.Property("InitBatchAssetUpdateDelay") + .HasColumnType("integer"); + + b.Property("MaxRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("MinRandomUpdateDay") + .HasColumnType("integer"); + + b.Property("TradeRepublicMaxRequestPageSize") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStart") + .HasColumnType("integer"); + + b.Property("UpdateDayTimeStop") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.AssetEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("InstrumentCategory") + .HasColumnType("text"); + + b.Property("AssetType") + .IsRequired() + .HasMaxLength(13) + .HasColumnType("character varying(13)"); + + b.Property("HasCfd") + .HasColumnType("boolean"); + + b.Property("ImageId") + .HasColumnType("text"); + + b.Property("LastUpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Isin", "InstrumentCategory"); + + b.ToTable("TradeRepublicAssets"); + + b.HasDiscriminator("AssetType").HasValue("AssetEntity"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.TagEntity", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("TradeRepublicTags"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.BondEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("BondIssuerName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Bond"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.CryptoEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("SearchSubtitle") + .HasColumnName("CryptoEntity_SearchSubtitle"); + }); + + b.HasDiscriminator().HasValue("Crypto"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.DerivativeEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("UnderlyingIsin") + .HasColumnType("text"); + + b.HasDiscriminator().HasValue("Derivative"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.EtfEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.Property("EtfDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("MappedEtfIndexName") + .IsRequired() + .HasColumnType("text"); + + b.Property("SearchSubtitle") + .IsRequired() + .HasColumnType("text"); + + b.Property("Subtitle") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("EtfEntity_DerivativeProductCategories"); + + t.Property("SearchSubtitle") + .HasColumnName("EtfEntity_SearchSubtitle"); + + t.Property("Subtitle") + .HasColumnName("EtfEntity_Subtitle"); + }); + + b.HasDiscriminator().HasValue("Etf"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.StockEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("StockEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Stock"); + }); + + modelBuilder.Entity("FinlyticCore.Entities.Assets.SyntheticEntity", b => + { + b.HasBaseType("FinlyticCore.Entities.Assets.AssetEntity"); + + b.Property("DerivativeProductCategories") + .IsRequired() + .HasColumnType("text"); + + b.ToTable("TradeRepublicAssets", t => + { + t.Property("DerivativeProductCategories") + .HasColumnName("SyntheticEntity_DerivativeProductCategories"); + }); + + b.HasDiscriminator().HasValue("Synthetic"); + }); + + modelBuilder.Entity("AssetEntityTagEntity", b => + { + b.HasOne("FinlyticCore.Entities.Assets.TagEntity", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("FinlyticCore.Entities.Assets.AssetEntity", null) + .WithMany() + .HasForeignKey("AssetsIsin", "AssetsInstrumentCategory") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticAssets/Models/AssetIndex.cs b/FinlyticAssets/Models/AssetIndex.cs new file mode 100644 index 0000000..60df083 --- /dev/null +++ b/FinlyticAssets/Models/AssetIndex.cs @@ -0,0 +1,7 @@ +using System.Text.Json.Serialization; + +namespace FinlyticAssets.Models; + +public record AssetIndex( + [property: JsonPropertyName("isin")] string Isin, + [property: JsonPropertyName("name")] string Name); \ No newline at end of file diff --git a/FinlyticAssets/Models/AssetsCount.cs b/FinlyticAssets/Models/AssetsCount.cs new file mode 100644 index 0000000..b4b741d --- /dev/null +++ b/FinlyticAssets/Models/AssetsCount.cs @@ -0,0 +1,39 @@ +using FinlyticCore.Models.Assets; + +namespace FinlyticAssets.Models; + +public class AssetsCount +{ + public int Stock { get; set; } + public int Funds { get; set; } + public int Derivatives { get; set; } + public int Crypto { get; set; } + public int Bond { get; set; } + + public int GetCountFromType(AssetType type) + { + return type switch + { + AssetType.Stock => Stock, + AssetType.Fund => Funds, + AssetType.Derivative => Derivatives, + AssetType.Crypto => Crypto, + AssetType.Bond => Bond, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) + }; + } + + + public void SetCountOfType(AssetType type, int count) + { + _ = type switch + { + AssetType.Stock => Stock = count, + AssetType.Fund => Funds = count, + AssetType.Derivative => Derivatives = count, + AssetType.Crypto => Crypto = count, + AssetType.Bond => Bond = count, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) + }; + } +} \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs new file mode 100644 index 0000000..b92f683 --- /dev/null +++ b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicAssetResponse.cs @@ -0,0 +1,139 @@ +namespace FinlyticAssets.Models.DataToObject.TradeRepublic; + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +// 1. Der Response-Wrapper +public record TradeRepublicAssetResponse( + [property: JsonPropertyName("correlationId")] string CorrelationId, + [property: JsonPropertyName("resultCount")] int ResultCount, + [property: JsonPropertyName("results")] IList Results +); + +// 2. Das Tag-Objekt +public record TradeRepublicTag +{ + [JsonPropertyName("id")] public string Id { get; init; } = ""; + [JsonPropertyName("name")] public string Name { get; init; } = ""; + [JsonPropertyName("type")] public string Type { get; init; } = ""; +} + +// 3. Die Basisklasse MIT UNSEREM CUSTOM CONVERTER (Kein [JsonPolymorphic] mehr!) +[JsonConverter(typeof(TradeRepublicAssetConverter))] +public record TradeRepublicAsset +{ + [JsonPropertyName("isin")] public string Isin { get; init; } = ""; + [JsonPropertyName("name")] public string Name { get; init; } = ""; + [JsonPropertyName("type")] public string Type { get; init; } = ""; + [JsonPropertyName("instrumentCategory")] public string InstrumentCategory { get; init; } = ""; + [JsonPropertyName("hasCfd")] public bool HasCfd { get; init; } + [JsonPropertyName("imageId")] public string? ImageId { get; init; } + + [JsonPropertyName("tags")] + public IReadOnlyList Tags { get; init; } = Array.Empty(); +} + +// 4. Die spezifischen Klassen (inklusive Bond und Derivative aus deinem JSON!) + +public record TradeRepublicStock : TradeRepublicAsset +{ + [JsonPropertyName("derivativeProductCategories")] + public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); +} + +public record TradeRepublicCrypto : TradeRepublicAsset +{ + [JsonPropertyName("subtitle")] public string Subtitle { get; init; } = ""; + [JsonPropertyName("searchSubtitle")] public string SearchSubtitle { get; init; } = ""; +} + +public record TradeRepublicEtf : TradeRepublicAsset +{ + [JsonPropertyName("derivativeProductCategories")] + public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); + [JsonPropertyName("etfDescription")] public string EtfDescription { get; init; } = ""; + [JsonPropertyName("mappedEtfIndexName")] public string MappedEtfIndexName { get; init; } = ""; + [JsonPropertyName("subtitle")] public string Subtitle { get; init; } = ""; + [JsonPropertyName("searchSubtitle")] public string SearchSubtitle { get; init; } = ""; +} + +public record TradeRepublicSynthetic : TradeRepublicAsset +{ + [JsonPropertyName("derivativeProductCategories")] + public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); +} + +// NEU: Anleihen +public record TradeRepublicBond : TradeRepublicAsset +{ + [JsonPropertyName("bondIssuerName")] public string BondIssuerName { get; init; } = ""; + [JsonPropertyName("searchSubtitle")] public string SearchSubtitle { get; init; } = ""; +} + +// NEU: Derivate (Hebeleffekte etc.) +public record TradeRepublicDerivative : TradeRepublicAsset +{ + [JsonPropertyName("derivativeProductCategories")] + public IReadOnlyList DerivativeProductCategories { get; init; } = Array.Empty(); + + [JsonIgnore] + public string? UnderlyingIsin + { + get + { + // Wenn die ImageId z.B. "logos/US0378331005/v2" ist... + if (!string.IsNullOrEmpty(ImageId) && ImageId.StartsWith("logos/")) + { + var parts = ImageId.Split('/'); + if (parts.Length >= 2) + { + return parts[1]; // Gibt "US0378331005" zurück + } + } + return null; // Falls das Format mal anders ist + } + } +} + +// 5. Der Custom Converter - Die Maschine, die das JSON scannt und verteilt +public class TradeRepublicAssetConverter : JsonConverter +{ + public override TradeRepublicAsset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using var doc = JsonDocument.ParseValue(ref reader); + var root = doc.RootElement; + + // Wir scannen nach instrumentType, egal wo im JSON es steht! + string? instrumentType = null; + if (root.TryGetProperty("instrumentType", out var typeElement)) + { + instrumentType = typeElement.GetString(); + } + + // Wir werfen das JSON gezielt in die richtige Klasse + TradeRepublicAsset? result = instrumentType switch + { + "stock" => JsonSerializer.Deserialize(root.GetRawText(), options), + "crypto" => JsonSerializer.Deserialize(root.GetRawText(), options), + "fund" => JsonSerializer.Deserialize(root.GetRawText(), options), + "synthetic" => JsonSerializer.Deserialize(root.GetRawText(), options), + "bond" => JsonSerializer.Deserialize(root.GetRawText(), options), + "derivative" => JsonSerializer.Deserialize(root.GetRawText(), options), + + // Wenn TR einen Typ schickt, den wir noch nicht kennen: Fallback nutzen! + _ => JsonSerializer.Deserialize(root.GetRawText(), options) + }; + + return result ?? new TradeRepublicAssetFallback(); + } + + public override void Write(Utf8JsonWriter writer, TradeRepublicAsset value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, value.GetType(), options); + } +} + +// Ein reiner Fallback-Record, der nur intern vom Converter genutzt wird +file record TradeRepublicAssetFallback : TradeRepublicAsset; \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs new file mode 100644 index 0000000..4b38484 --- /dev/null +++ b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicConnectRequest.cs @@ -0,0 +1,16 @@ +using System.Text.Json.Serialization; + +namespace FinlyticAssets.Models.DataToObject.TradeRepublic; + +public record TradeRepublicConnectRequest( + [property: JsonPropertyName("clientId")] string ClientId = "app.traderepublic.com", + [property: JsonPropertyName("clientVersion")] string ClientVersion = "15.65.6", + [property: JsonPropertyName("locale")] string Locale = "en", + [property: JsonPropertyName("platformId")] string PlatformId = "webtrading", + [property: JsonPropertyName("platformVersion")] string PlatformVersion = "chrome - 149.0.0", + TradeRepublicHeaders? Headers = null +) +{ + [JsonPropertyName("__headers")] + public TradeRepublicHeaders Headers { get; init; } = Headers ?? new TradeRepublicHeaders(); +} \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs new file mode 100644 index 0000000..4923424 --- /dev/null +++ b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicHeaders.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; +using FinlyticAssets.Util; + +namespace FinlyticAssets.Models.DataToObject.TradeRepublic; + +public record TradeRepublicHeaders( + [property: JsonPropertyName("traceparent")] string Traceparent +) +{ + public TradeRepublicHeaders() : this(StringCodeGenerator.GenerateTraceparent()) + {} +} \ No newline at end of file diff --git a/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs new file mode 100644 index 0000000..fa71349 --- /dev/null +++ b/FinlyticAssets/Models/DataToObject/TradeRepublic/TradeRepublicSearchRequest.cs @@ -0,0 +1,30 @@ +using System.Text.Json.Serialization; +using FinlyticAssets.Util; + +namespace FinlyticAssets.Models.DataToObject.TradeRepublic; + +public record TradeRepublicFilter( + [property: JsonPropertyName("key")] string Key, + [property: JsonPropertyName("value")] string Value +); + +public record TradeRepublicSearchData( + [property: JsonPropertyName("q")] string Query = "", + [property: JsonPropertyName("page")] int Page = 1, + [property: JsonPropertyName("pageSize")] int PageSize = 50, + IReadOnlyList? Filter = null +) +{ + [JsonPropertyName("filter")] + public IReadOnlyList Filter { get; init; } = Filter ?? Array.Empty(); +} + +public record TradeRepublicSearchRequest( + [property: JsonPropertyName("data")] TradeRepublicSearchData Data, + [property: JsonPropertyName("type")] string Type = "neonSearch", + TradeRepublicHeaders? Headers = null +) +{ + [JsonPropertyName("__headers")] + public TradeRepublicHeaders Headers { get; init; } = Headers ?? new TradeRepublicHeaders(); +} \ No newline at end of file diff --git a/FinlyticAssets/Program.cs b/FinlyticAssets/Program.cs new file mode 100644 index 0000000..6442ea0 --- /dev/null +++ b/FinlyticAssets/Program.cs @@ -0,0 +1,43 @@ +using System.Text.Json; +using FinlyticAssets; +using FinlyticAssets.Database; +using FinlyticAssets.Models.DataToObject.TradeRepublic; +using FinlyticAssets.Services; +using FinlyticAssets.Util; +using Microsoft.EntityFrameworkCore; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services.AddDbContext(options => + options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection"))); + +builder.Services.AddScoped(); + +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +builder.Services.AddSingleton(); + +builder.Services.AddHostedService(); + +builder.Services.AddHostedService(); + +var host = builder.Build(); + +using (var scope = host.Services.CreateScope()) +{ + try + { + var context = scope.ServiceProvider.GetRequiredService(); + await context.Database.MigrateAsync(); + } + catch (Exception ex) + { + Console.WriteLine($"Critical error during database migration: {ex.Message}"); + Console.WriteLine(ex.StackTrace); + } +} + +host.Run(); \ No newline at end of file diff --git a/FinlyticAssets/Properties/launchSettings.json b/FinlyticAssets/Properties/launchSettings.json new file mode 100644 index 0000000..59e7b5d --- /dev/null +++ b/FinlyticAssets/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "FinlyticAssets": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/FinlyticAssets/Services/AssetsDbService.cs b/FinlyticAssets/Services/AssetsDbService.cs new file mode 100644 index 0000000..11537b7 --- /dev/null +++ b/FinlyticAssets/Services/AssetsDbService.cs @@ -0,0 +1,437 @@ +using FinlyticAssets.Database; +using FinlyticAssets.Entities; +using FinlyticAssets.Models.DataToObject.TradeRepublic; +using FinlyticCore.Entities.Assets; +using Microsoft.EntityFrameworkCore; + +namespace FinlyticAssets.Services; + +/// +/// Defines database operations for managing Trade Republic asset entities. +/// +public interface IAssetsDbService +{ + + /// + /// Retrieves all active assets from the database that have been updated within the last 14 days, including their associated tags. + /// Assets older than 14 days are filtered out as they are considered de-listed or inactive. + /// + /// A task that represents the asynchronous operation. The task result contains a list of all active instances. + public Task> GetAllValidAssetsAsync(); + + /// + /// Adds a new asset or updates an existing one based on the composite key of ISIN and InstrumentType. + /// + /// The incoming asset data transfer object from the API. + /// A task that represents the asynchronous operation. The task result contains true if the asset was updated or created; otherwise, false. + public Task AddOrUpdateAssetAsync(TradeRepublicAsset dtoAsset); + + /// + /// Batches processing for a collection of asset items, returning the total amount of modified or newly added entries. + /// + /// The collection of incoming asset objects to process. + /// A task that represents the asynchronous operation. The task result contains the count of changed or added entries. + public Task AddOrUpdateAssetsAsync(IEnumerable dtoAssets); + + /// + /// Retrieves all assets matching a specific International Securities Identification Number (ISIN). + /// Can return multiple entities (e.g., both the stock and the derivative tracking asset for the same ISIN). + /// + /// The ISIN value to look up. + /// A task that represents the asynchronous operation. The task result contains a list of matching instances. + public Task> GetAssetsByIsinAsync(string isin); + + /// + /// Retrieves all assets matching a specific ISIN only if they have been updated within the last 14 days. + /// Assets older than 14 days are considered de-listed or inactive. + /// + /// The ISIN value to look up. + /// A task that represents the asynchronous operation. The task result contains a list of matching active instances. + public Task> GetValidAssetsByIsinAsync(string isin); + + /// + /// Scans the database for active assets matching a combined, comma-separated search query. + /// The search applies an AND-logic approach where every extracted keyword must be found within an asset's ISIN, name, or tags. + /// Evaluates local records first and utilizes a targeted JIT-fallback to the external API for any search term formatted as a valid, completely unknown ISIN. + /// + /// A comma-separated string containing the keywords, ISINs, or tags to search for (e.g., "Siemens, Medic" or "IE00B4L5Y983, ETF"). + /// A task that represents the asynchronous operation. The task result contains a list of all affected active instances. + public Task> FindAffectedActiveAssetsAsync(string searchQuery); + + /// + /// Deletes all asset records associated with a specific ISIN from the database. + /// + /// + /// Use with caution. For standard maintenance and handling de-listed instruments, + /// rely on the 14-day recency filter provided by instead of hard deletion. + /// + /// The ISIN value of the assets to remove. + /// A task that represents the asynchronous operation. The task result contains true if any records were successfully deleted; otherwise, false. + public Task DeleteAssetAsync(string isin); +} + +/// +public class AssetsDbService : IAssetsDbService +{ + private readonly AssetsDbContext _context; + private readonly ITradeRepublicService _tradeRepublicService; + private readonly ILogger _logger; + private readonly Random _random = new(); + + /// + /// Initializes a new instance of the class. + /// + public AssetsDbService(AssetsDbContext context, ILogger logger, + ITradeRepublicService tradeRepublicService) + { + _context = context; + _logger = logger; + _tradeRepublicService = tradeRepublicService; + } + + /// + public async Task> GetAllValidAssetsAsync() + { + var cutoff = DateTime.UtcNow.AddDays(-14); + + var existingEntity = await _context.TradeRepublicAssets + .Include(a => a.Tags) + .Where(a => a.UpdateAt >= cutoff) + .ToListAsync(); + + return existingEntity; + } + + /// + public async Task AddOrUpdateAssetAsync(TradeRepublicAsset dtoAsset) + { + var existingEntity = await _context.TradeRepublicAssets + .Include(a => a.Tags) + .FirstOrDefaultAsync(a => a.Isin == dtoAsset.Isin); + + var now = DateTime.UtcNow; + var nextUpdateScheduledAt = await CalculateNextUpdateDateAsync(now); + + if (existingEntity == null) + { + _logger.LogDebug("Asset with ISIN {Isin} not found. Mapping and creating a new entity.", dtoAsset.Isin); + + var newEntity = MapDtoToEntity(dtoAsset); + newEntity.LastUpdatedAt = now; + newEntity.UpdateAt = nextUpdateScheduledAt; + + newEntity.Tags = await MapTagsAsync(dtoAsset.Tags); + + await _context.TradeRepublicAssets.AddAsync(newEntity); + await _context.SaveChangesAsync(); + return true; + } + + _logger.LogDebug("Asset with ISIN {Isin} exists. Merging properties and updating database record.", + dtoAsset.Isin); + + existingEntity.Name = dtoAsset.Name; + existingEntity.Type = dtoAsset.Type; + existingEntity.InstrumentCategory = dtoAsset.InstrumentCategory; + existingEntity.HasCfd = dtoAsset.HasCfd; + existingEntity.ImageId = dtoAsset.ImageId; + + existingEntity.LastUpdatedAt = now; + existingEntity.UpdateAt = nextUpdateScheduledAt; + + UpdateSubtypeProperties(existingEntity, dtoAsset); + existingEntity.Tags = await MapTagsAsync(dtoAsset.Tags); + + _context.TradeRepublicAssets.Update(existingEntity); + await _context.SaveChangesAsync(); + return true; + } + + /// + public async Task AddOrUpdateAssetsAsync(IEnumerable dtoAssets) + { + int changedCount = 0; + + foreach (var dto in dtoAssets) + { + var isChanged = await AddOrUpdateAssetAsync(dto); + if (isChanged) + { + changedCount++; + } + } + + return changedCount; + } + + /// + public async Task> GetAssetsByIsinAsync(string isin) + { + var localAssets = await _context.TradeRepublicAssets + .Include(a => a.Tags) + .Where(a => a.Isin == isin) + .ToListAsync(); + + if (localAssets.Any()) + { + return localAssets; + } + + var trAssetDto = await _tradeRepublicService.GetAsset(isin); + if (trAssetDto != null) + { + foreach (var asset in trAssetDto.Results) + { + _ = await AddOrUpdateAssetAsync(asset); + } + + return await GetAssetsByIsinAsync(isin); + } + + return []; + } + + /// + public async Task> GetValidAssetsByIsinAsync(string isin) + { + var cutoff = DateTime.UtcNow.AddDays(-14); + + return await _context.TradeRepublicAssets + .AsNoTracking() + .Include(a => a.Tags) + .Where(a => a.Isin == isin && a.UpdateAt >= cutoff) + .ToListAsync(); + } + + /// + public async Task> FindAffectedActiveAssetsAsync(string searchQuery) + { + if (string.IsNullOrWhiteSpace(searchQuery)) + { + return []; + } + + var cutoff = DateTime.UtcNow.AddDays(-14); + + var searchTerms = searchQuery + .Split(',') + .Select(t => t.Trim().ToLower()) + .Where(t => !string.IsNullOrEmpty(t)) + .Distinct() + .ToList(); + + if (searchTerms.Count == 0) return []; + + var query = _context.TradeRepublicAssets + .AsNoTracking() + .Where(a => a.UpdateAt >= cutoff) + .Include(a => a.Tags) + .AsQueryable(); + + + foreach (var term in searchTerms) + { + query = query.Where(a => + a.Isin.ToLower().Contains(term) || + a.Name.ToLower().Contains(term) || + a.Tags.Any(tag => tag.Name.ToLower().Contains(term))); + } + + var localAssets = await query.ToListAsync(); + + var possibleIsins = searchTerms + .Where(t => t.Length == 12 && char.IsLetter(t[0]) && char.IsLetter(t[1])) + .Select(t => t.ToUpper()) + .ToList(); + + if (possibleIsins.Any()) + { + var foundIsins = localAssets.Select(a => a.Isin).ToHashSet(); + var missingIsins = possibleIsins.Where(isin => !foundIsins.Contains(isin)).ToList(); + + if (missingIsins.Any()) + { + var fetchedNewAsset = false; + foreach (var missingIsin in missingIsins) + { + var trAssetDto = await _tradeRepublicService.GetAsset(missingIsin); + if (trAssetDto?.Results != null) + { + foreach (var asset in trAssetDto.Results) + { + await AddOrUpdateAssetAsync(asset); + } + + fetchedNewAsset = true; + } + } + + if (fetchedNewAsset) + { + return await query.ToListAsync(); + } + } + } + + return localAssets; + } + + /// + public async Task DeleteAssetAsync(string isin) + { + var asset = await _context.TradeRepublicAssets.FirstOrDefaultAsync(a => a.Isin == isin); + if (asset == null) + { + _logger.LogWarning("Delete execution cancelled. Asset with ISIN {Isin} does not exist.", isin); + return false; + } + + _context.TradeRepublicAssets.Remove(asset); + await _context.SaveChangesAsync(); + _logger.LogInformation("Asset with ISIN {Isin} has been successfully deleted.", isin); + return true; + } + + #region Helper & Mapping Methods + + /// + /// Calculates the next synchronization/update date for an asset using configured parameters and a random offset. + /// + /// The baseline date to add the offsets to. + /// A task representing the asynchronous operation, returning a DateTime representing the next scheduled update time (UTC). + private async Task CalculateNextUpdateDateAsync(DateTime baseDate) + { + var settings = await _context.Set().FirstOrDefaultAsync() ?? new Settings(); + + int randomDays = _random.Next(settings.MinRandomUpdateDay, settings.MaxRandomUpdateDay + 1); + var targetDate = baseDate.AddDays(randomDays); + + int randomHour = _random.Next(settings.UpdateDayTimeStart, settings.UpdateDayTimeStop); + int randomMinute = _random.Next(0, 60); + int randomSecond = _random.Next(0, 60); + + return new DateTime( + targetDate.Year, + targetDate.Month, + targetDate.Day, + randomHour, + randomMinute, + randomSecond, + DateTimeKind.Utc + ); + } + + /// + /// Maps a raw Trade Republic asset data transfer object (DTO) to its matching database entity subtype. + /// + /// The source Trade Republic asset data transfer object. + /// A newly created subtype instance of mapped with the DTO properties. + /// Thrown when the DTO type is unrecognized or unsupported. + private AssetEntity MapDtoToEntity(TradeRepublicAsset dto) + { + AssetEntity entity = dto switch + { + TradeRepublicStock stock => new StockEntity + { Isin = stock.Isin, DerivativeProductCategories = stock.DerivativeProductCategories.ToList() }, + TradeRepublicCrypto crypto => new CryptoEntity + { Isin = crypto.Isin, Subtitle = crypto.Subtitle, SearchSubtitle = crypto.SearchSubtitle }, + TradeRepublicEtf etf => new EtfEntity + { + Isin = etf.Isin, EtfDescription = etf.EtfDescription, MappedEtfIndexName = etf.MappedEtfIndexName, + Subtitle = etf.Subtitle, SearchSubtitle = etf.SearchSubtitle, + DerivativeProductCategories = etf.DerivativeProductCategories.ToList() + }, + TradeRepublicSynthetic synth => new SyntheticEntity + { Isin = synth.Isin, DerivativeProductCategories = synth.DerivativeProductCategories.ToList() }, + TradeRepublicBond bond => new BondEntity + { Isin = bond.Isin, BondIssuerName = bond.BondIssuerName, SearchSubtitle = bond.SearchSubtitle }, + TradeRepublicDerivative deriv => new DerivativeEntity + { + Isin = deriv.Isin, UnderlyingIsin = deriv.UnderlyingIsin, + DerivativeProductCategories = deriv.DerivativeProductCategories.ToList() + }, + _ => throw new NotSupportedException($"Type {dto.GetType().Name} is not supported.") + }; + + return PopulateBaseProperties(entity, dto); + } + + /// + /// Populates common base properties of a database asset entity using a Trade Republic DTO. + /// + /// The target database entity. + /// The source Trade Republic DTO. + /// The updated database asset entity. + private AssetEntity PopulateBaseProperties(AssetEntity entity, TradeRepublicAsset dto) + { + entity.Name = dto.Name; + entity.Type = dto.Type; + entity.InstrumentCategory = dto.InstrumentCategory; + entity.HasCfd = dto.HasCfd; + entity.ImageId = dto.ImageId; + return entity; + } + + /// + /// Merges/updates the subtype-specific properties from a Trade Republic DTO into an existing database entity. + /// + /// The existing database entity to update. + /// The source Trade Republic DTO. + private void UpdateSubtypeProperties(AssetEntity entity, TradeRepublicAsset dto) + { + switch (entity) + { + case StockEntity stockEntity when dto is TradeRepublicStock stockDto: + stockEntity.DerivativeProductCategories = stockDto.DerivativeProductCategories.ToList(); + break; + case CryptoEntity cryptoEntity when dto is TradeRepublicCrypto cryptoDto: + cryptoEntity.Subtitle = cryptoDto.Subtitle; + cryptoEntity.SearchSubtitle = cryptoDto.SearchSubtitle; + break; + case EtfEntity etfEntity when dto is TradeRepublicEtf etfDto: + etfEntity.DerivativeProductCategories = etfDto.DerivativeProductCategories.ToList(); + etfEntity.EtfDescription = etfDto.EtfDescription; + etfEntity.MappedEtfIndexName = etfDto.MappedEtfIndexName; + etfEntity.Subtitle = etfDto.Subtitle; + etfEntity.SearchSubtitle = etfDto.SearchSubtitle; + break; + case SyntheticEntity synthEntity when dto is TradeRepublicSynthetic synthDto: + synthEntity.DerivativeProductCategories = synthDto.DerivativeProductCategories.ToList(); + break; + case BondEntity bondEntity when dto is TradeRepublicBond bondDto: + bondEntity.BondIssuerName = bondDto.BondIssuerName; + bondEntity.SearchSubtitle = bondDto.SearchSubtitle; + break; + case DerivativeEntity derivEntity when dto is TradeRepublicDerivative derivDto: + derivEntity.DerivativeProductCategories = derivDto.DerivativeProductCategories.ToList(); + derivEntity.UnderlyingIsin = derivDto.UnderlyingIsin; + break; + } + } + + /// + /// Maps a list of Trade Republic tags to database entity instances, registering new tags in the database context if they do not yet exist. + /// + /// The read-only collection of Trade Republic tags. + /// A task representing the asynchronous operation, returning the list of mapped database tag entities. + private async Task> MapTagsAsync(IReadOnlyList dtos) + { + var tags = new List(); + foreach (var tagDto in dtos) + { + var existingTag = await _context.TradeRepublicTags.FirstOrDefaultAsync(t => t.Id == tagDto.Id); + if (existingTag == null) + { + _logger.LogTrace("Creating missing asset tag object in storage cache context. ID: {TagId}", tagDto.Id); + existingTag = new TagEntity { Id = tagDto.Id, Name = tagDto.Name, Type = tagDto.Type }; + await _context.TradeRepublicTags.AddAsync(existingTag); + } + + tags.Add(existingTag); + } + + return tags; + } + + #endregion +} \ No newline at end of file diff --git a/FinlyticAssets/Services/AssetsFullScanService.cs b/FinlyticAssets/Services/AssetsFullScanService.cs new file mode 100644 index 0000000..0bc22e7 --- /dev/null +++ b/FinlyticAssets/Services/AssetsFullScanService.cs @@ -0,0 +1,231 @@ +using FinlyticAssets.Models; +using FinlyticAssets.Models.DataToObject.TradeRepublic; +using FinlyticCore.Models.Assets; + +namespace FinlyticAssets.Services; + +/// +/// A background service that runs a continuous asset synchronization loop, +/// scanning Trade Republic to retrieve, update, and index all supported asset types. +/// +public class AssetsFullScanService : BackgroundService +{ + private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly ILogger _logger; + + private AssetsCount? _assetsCount; + private AssetsCount? _currAssetsCount; + + /// + /// Initializes a new instance of the class. + /// + /// Factory used to create service scopes for database and API requests. + /// Logger for service lifecycle and scanning progress messages. + public AssetsFullScanService(IServiceScopeFactory serviceScopeFactory, ILogger logger) + { + _serviceScopeFactory = serviceScopeFactory; + _logger = logger; + } + + /// + /// Executes the background scanning task, handling initial startup, recovery, and periodic full-scan cycles. + /// + /// Triggered when the host is shutting down. + /// A task that represents the background operation. + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("AssetsFullScanService has started."); + + try + { + using (var scope = _serviceScopeFactory.CreateScope()) + { + var indexService = scope.ServiceProvider.GetRequiredService(); + _logger.LogInformation("Building initial asset index on service startup..."); + await indexService.ReCreateIndexFileAsync(stoppingToken); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to build initial asset index on startup. Continuing service execution."); + } + + do + { + try + { + using (var scope = _serviceScopeFactory.CreateScope()) + { + var tradeRepublicService = scope.ServiceProvider.GetRequiredService(); + var settingsService = scope.ServiceProvider.GetRequiredService(); + var assetsDbService = scope.ServiceProvider.GetRequiredService(); + var indexService = scope.ServiceProvider.GetRequiredService(); + + _logger.LogInformation("Requesting total asset counts from Trade Republic..."); + _assetsCount = await tradeRepublicService.GetAssetsCount(stoppingToken); + _currAssetsCount = new AssetsCount(); + + var initSettings = await settingsService.GetSettings(); + var isRecoveryMode = initSettings.CurrentScanningPage > 0; + + foreach (var type in Enum.GetValues()) + { + if (stoppingToken.IsCancellationRequested) break; + + if (isRecoveryMode) + { + if (type != initSettings.CurrentScanningType) + { + _logger.LogInformation("Recovery: {AssetType} was already processed. Skipping.", type); + continue; + } + isRecoveryMode = false; + } + else + { + var settings = await settingsService.GetSettings(); + settings.CurrentScanningType = type; + settings.CurrentScanningPage = 0; + await settingsService.SaveSettings(settings); + } + + _logger.LogInformation("Processing asset type: {AssetType}...", type); + await HandleAssetType(type, tradeRepublicService, settingsService, assetsDbService, indexService, stoppingToken); + + var currentSettings = await settingsService.GetSettings(); + var delaySeconds = currentSettings.FinishedInitialScan + ? currentSettings.AssetUpdateTypeDelay + : currentSettings.InitAssetUpdateTypeDelay; + + var jitter = Random.Shared.Next(0, 480); + _logger.LogDebug("Waiting {Delay} seconds before the next asset type.", delaySeconds + jitter); + await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); + } + + var finalSettings = await settingsService.GetSettings(); + finalSettings.CurrentScanningPage = 0; + + if (!finalSettings.FinishedInitialScan && !stoppingToken.IsCancellationRequested) + { + _logger.LogInformation("Initial scan successfully completed. Switching FinishedInitialScan to true."); + finalSettings.FinishedInitialScan = true; + } + + await settingsService.SaveSettings(finalSettings); + } + + _logger.LogInformation("Full scan cycle completed. Waiting 1 minute before starting the next cycle."); + await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + } + catch (Exception e) + { + _logger.LogError(e, "An unhandled exception occurred in AssetsFullScanService. Retrying in 10 seconds."); + try { await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); } catch { /* Ignore */ } + } + } while (!stoppingToken.IsCancellationRequested); + + _logger.LogInformation("AssetsFullScanService is stopping."); + } + + /// + /// Handles the scanning process for a specific asset type, iterating through its paginated results. + /// + /// The type of assets to process (e.g., Stock, Crypto). + /// Service to fetch asset data from Trade Republic. + /// Service to read and write application state/settings. + /// Service to insert or update assets in the database. + /// Service to recreate the local index file if needed. + /// Cancellation token monitored for cancellation requests. + /// A task representing the asynchronous operation. + private async Task HandleAssetType(AssetType type, ITradeRepublicService tradeRepublicService, + ISettingsDbService settingsDbService, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) + { + var totalCount = _assetsCount?.GetCountFromType(type) ?? 0; + if (totalCount == 0) + { + _logger.LogWarning("No assets found for type {AssetType}.", type); + return; + } + + _currAssetsCount ??= new AssetsCount(); + var currentItemOffset = 0; + + var settings = await settingsDbService.GetSettings(); + if (settings.CurrentScanningType == type && settings.CurrentScanningPage > 0) + { + var pageSize = settings.TradeRepublicMaxRequestPageSize <= 0 ? 50 : settings.TradeRepublicMaxRequestPageSize; + if (pageSize > 100) pageSize = 100; + + currentItemOffset = (settings.CurrentScanningPage - 1) * pageSize; + _logger.LogInformation("Resuming full scan for {AssetType} from Page {Page} (Offset: {Offset}).", + type, settings.CurrentScanningPage, currentItemOffset); + } + + while (currentItemOffset < totalCount && !stoppingToken.IsCancellationRequested) + { + var currentSettings = await settingsDbService.GetSettings(); + var pageSize = currentSettings.TradeRepublicMaxRequestPageSize; + if (pageSize <= 0 || pageSize > 100) pageSize = 100; + + var currentPage = (currentItemOffset / pageSize) + 1; + + currentSettings.CurrentScanningType = type; + currentSettings.CurrentScanningPage = currentPage; + await settingsDbService.SaveSettings(currentSettings); + + _logger.LogDebug("Fetching {AssetType} - Page {Page} (Size: {PageSize}). Offset: {Offset}/{Total}", + type, currentPage, pageSize, currentItemOffset, totalCount); + + var assets = await tradeRepublicService.GetAssets(type, currentPage, pageSize, stoppingToken); + + if (assets?.Results == null || assets.Results.Count == 0) + { + _logger.LogWarning("Fetch for {AssetType} (Page {Page}) returned no results. Retrying in 5 seconds...", type, currentPage); + await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); + continue; + } + + await ProcessAssets(assets.Results, assetsDbService, indexService,stoppingToken); + _currAssetsCount.SetCountOfType(type, _currAssetsCount.GetCountFromType(type) + assets.Results.Count); + + currentItemOffset = currentPage * pageSize; + + if (assets.Results.Count < pageSize) + { + _logger.LogInformation("Reached the last page for {AssetType}.", type); + break; + } + + var delaySeconds = currentSettings.FinishedInitialScan + ? currentSettings.BatchAssetUpdateDelay + : currentSettings.InitBatchAssetUpdateDelay; + + var jitter = Random.Shared.Next(0, 360); + _logger.LogDebug("Waiting {Delay} seconds before the next batch.", delaySeconds + jitter); + await Task.Delay(TimeSpan.FromSeconds(delaySeconds + jitter), stoppingToken); + } + } + + /// + /// Processes a list of fetched Trade Republic assets, updates them in the database, + /// and triggers an index recreation if changes were detected. + /// + /// The list of Trade Republic assets to process. + /// Service to insert or update assets in the database. + /// Service to recreate the local index file. + /// Cancellation token monitored for cancellation requests. + /// A task representing the asynchronous operation. + private async Task ProcessAssets(IList assets, IAssetsDbService assetsDbService, IAssetsIndexService indexService, CancellationToken stoppingToken) + { + if (assets == null || assets.Count == 0) return; + + var changedRows = await assetsDbService.AddOrUpdateAssetsAsync(assets); + _logger.LogInformation("[Scan] {Count} assets passed to the DB service. {Changed} modifications/inserts executed.", assets.Count, changedRows); + + if (changedRows > 0) + { + _logger.LogInformation("Database modifications detected. Recreating the asset index file..."); + await indexService.ReCreateIndexFileAsync(stoppingToken); + } + } +} \ No newline at end of file diff --git a/FinlyticAssets/Services/AssetsIndexService.cs b/FinlyticAssets/Services/AssetsIndexService.cs new file mode 100644 index 0000000..611b904 --- /dev/null +++ b/FinlyticAssets/Services/AssetsIndexService.cs @@ -0,0 +1,87 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using FinlyticAssets.Models; +using FinlyticAssets.Util; + +namespace FinlyticAssets.Services; + +/// +/// Provides methods for generating and maintaining the indexed asset reference file used for pre-filtering. +/// +public interface IAssetsIndexService +{ + /// + /// Recreates the index file containing basic asset identifiers (ISIN and Name) for all active, valid assets. + /// + /// A token to monitor for cancellation requests. + /// A task that represents the asynchronous operation. + public Task ReCreateIndexFileAsync(CancellationToken cancellationToken); +} + +/// +/// Implements the to maintain local asset index references. +/// +public class AssetsIndexService : IAssetsIndexService +{ + private readonly ILogger _logger; + private readonly IAssetsDbService _assetsDbService; + + /// + /// Initializes a new instance of the class. + /// + /// The logger for documenting indexing events and errors. + /// The database service to query the assets from. + public AssetsIndexService(ILogger logger, IAssetsDbService assetsDbService) + { + _logger = logger; + _assetsDbService = assetsDbService; + } + + /// + public async Task ReCreateIndexFileAsync(CancellationToken cancellationToken) + { + try + { + var assets = await _assetsDbService.GetAllValidAssetsAsync(); + if (assets == null || !assets.Any()) + { + _logger.LogWarning("No valid assets found in the database to index."); + return; + } + + var indexAssets = assets.Select(a => new AssetIndex(a.Isin, a.Name)).ToList(); + + var directoryPath = Volumes.IndexRelativePath; + var filePath = Path.Combine(directoryPath, "index.json"); + + if (!Directory.Exists(directoryPath)) + { + Directory.CreateDirectory(directoryPath); + } + + using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, + useAsync: true)) + { + await JsonSerializer.SerializeAsync(fileStream, indexAssets, cancellationToken: cancellationToken); + } + + _logger.LogInformation("Successfully recreated asset index file with {Count} entries at {Path}", + indexAssets.Count, filePath); + } + catch (IOException ex) + { + _logger.LogError(ex, "Disk I/O error occurred while writing the asset index file."); + throw; + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to serialize the asset index data to JSON."); + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "An unexpected error occurred while recreating the asset index file."); + throw; + } + } +} \ No newline at end of file diff --git a/FinlyticAssets/Services/MqttConnectionService.cs b/FinlyticAssets/Services/MqttConnectionService.cs new file mode 100644 index 0000000..cdc27b6 --- /dev/null +++ b/FinlyticAssets/Services/MqttConnectionService.cs @@ -0,0 +1,52 @@ +using FinlyticAssets.Util; +using FinlyticCore.Models; + +namespace FinlyticAssets.Services; + +/// +/// A hosted service responsible for managing the lifecycle of the MQTT client connection +/// when the application starts up and shuts down. +/// +public class MqttConnectionService : IHostedService +{ + private readonly AssetsMqttClient _mqttClient; + private readonly IConfiguration _configuration; + + /// + /// Initializes a new instance of the class. + /// + /// The MQTT client wrapper instance. + /// The application configuration provider. + public MqttConnectionService(AssetsMqttClient mqttClient, IConfiguration configuration) + { + _mqttClient = mqttClient; + _configuration = configuration; + } + + /// + /// Starts the MQTT client connection using settings resolved from configuration. + /// + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous start operation. + public async Task StartAsync(CancellationToken cancellationToken) + { + var config = new MqttConfiguration() + { + Host = _configuration["MQTT__Host"]!, + Port = Convert.ToInt32(_configuration["MQTT__Port"]!), + ClientId = $"{_configuration["MQTT__ClientId"]!}_{Guid.NewGuid()}" + }; + + await _mqttClient.ConnectAsync(config); + } + + /// + /// Stops and disconnects the MQTT client connection. + /// + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous stop operation. + public async Task StopAsync(CancellationToken cancellationToken) + { + await _mqttClient.DisconnectAsync(); + } +} \ No newline at end of file diff --git a/FinlyticAssets/Services/SettingsDbService.cs b/FinlyticAssets/Services/SettingsDbService.cs new file mode 100644 index 0000000..54b4a1d --- /dev/null +++ b/FinlyticAssets/Services/SettingsDbService.cs @@ -0,0 +1,91 @@ +using FinlyticAssets.Database; +using FinlyticAssets.Entities; +using Microsoft.EntityFrameworkCore; + +namespace FinlyticAssets.Services; + +/// +/// Defines the business logic for managing global application settings. +/// Supports retrieving and updating (upserting) the central single-row configuration record. +/// +public interface ISettingsDbService +{ + /// + /// Retrieves the current global settings from the database. + /// + /// + /// A task that represents the asynchronous operation. The task result contains the current . + /// If no settings exist in the database yet, a new instance initialized with default values is returned. + /// + public Task GetSettings(); + + /// + /// Persists the provided settings by updating the existing record or inserting the first one if the table is empty. + /// + /// The new configuration values to be persisted. + /// + /// A task that represents the asynchronous operation. The task result contains the freshly saved + /// and tracked instance. + /// + public Task SaveSettings(Settings settings); +} + +/// +/// Implements the utilizing Entity Framework Core. +/// This service is designed for a single-row table architecture to maintain stateful global configurations. +/// +public class SettingsDbService : ISettingsDbService +{ + private readonly AssetsDbContext _context; + + /// + /// Initializes a new instance of the class with the required database context. + /// + /// The EF Core context used to access the assets database. + public SettingsDbService(AssetsDbContext context) + { + _context = context; + } + + /// + public async Task GetSettings() + { + var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync(); + + if (settings == null) + { + settings = new Settings + { + Id = Guid.NewGuid() + }; + + await SaveSettings(settings); + } + + return settings; + } + + /// + public async Task SaveSettings(Settings settings) + { + var existing = await _context.Settings.FirstOrDefaultAsync(); + + if (existing == null) + { + if (settings.Id == Guid.Empty) + { + settings.Id = Guid.NewGuid(); + } + await _context.Settings.AddAsync(settings); + await _context.SaveChangesAsync(); + return settings; + } + else + { + _context.Entry(existing).CurrentValues.SetValues(settings); + + await _context.SaveChangesAsync(); + return existing; + } + } +} \ No newline at end of file diff --git a/FinlyticAssets/Services/TradeRepublicService.cs b/FinlyticAssets/Services/TradeRepublicService.cs new file mode 100644 index 0000000..58f277d --- /dev/null +++ b/FinlyticAssets/Services/TradeRepublicService.cs @@ -0,0 +1,240 @@ +using System.Timers; +using FinlyticAssets.Models; +using FinlyticAssets.Models.DataToObject.TradeRepublic; +using FinlyticAssets.Util; +using FinlyticCore.Models.Assets; + +namespace FinlyticAssets.Services; + +public interface ITradeRepublicService +{ + /// + /// Holt die Anzahl der Assets pro Typ für die Paginierung des Initial-Scans. + /// + public Task GetAssetsCount(CancellationToken cancellationToken = default); + + /// + /// Holt eine spezifische Seite an Assets für den Initial-Scan. + /// + public Task GetAssets(AssetType type, int page, int pageSize, + CancellationToken cancellationToken = default); + + /// + /// Holt die aktuellen Stammdaten für eine spezifische ISIN (Gezieltes Update). + /// Gibt null zurück, wenn das Asset bei TR nicht mehr existiert. + /// + public Task GetAsset(string isin, CancellationToken cancellationToken = default); +} + +/// +/// Provides a managed service to interact with the Trade Republic API via WebSockets, +/// featuring an automatic inactivity timeout to mimic human behavior. +/// +public class TradeRepublicService : ITradeRepublicService, IDisposable +{ + private readonly TradeRepublicClient _client; + private readonly ILogger _logger; + private readonly System.Timers.Timer _inactivityTimer; + private readonly SemaphoreSlim _lock = new(1, 1); + + /// + /// Initializes a new instance of the class. + /// + /// The underlying managed WebSocket client. + /// The logger instance. + public TradeRepublicService(TradeRepublicClient client, ILogger logger) + { + _client = client; + _logger = logger; + + _inactivityTimer = new System.Timers.Timer(TimeSpan.FromSeconds(461).TotalMilliseconds); + _inactivityTimer.AutoReset = false; + _inactivityTimer.Elapsed += OnInactivityTimeout; + } + + /// + /// Ensures that the Trade Republic WebSocket client is connected, initiating a new connection if necessary. + /// Also handles resetting the inactivity timer. + /// + /// A task representing the asynchronous operation. + private async Task EnsureConnectedAsync() + { + await _lock.WaitAsync(); + try + { + _inactivityTimer.Stop(); + + if (!_client.IsConnected) + { + _logger.LogInformation("Trade Republic API is not connected. Establishing automated connection..."); + + // Nutzt den boolschen Rückgabewert von InitAsync + bool connected = await _client.InitAsync(); + + if (connected) + { + _logger.LogInformation("Successfully connected to Trade Republic API."); + } + else + { + _logger.LogWarning("Trade Republic API connection initialization failed (InitAsync returned false)."); + } + } + + _inactivityTimer.Start(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to establish a connection to the Trade Republic API."); + throw; + } + finally + { + _lock.Release(); + } + } + + /// + /// Retrieves the total count of available assets grouped by their types. + /// + /// A token to monitor for cancellation requests. + /// An object containing the metrics. + public async Task GetAssetsCount(CancellationToken cancellationToken = default) + { + await EnsureConnectedAsync(); + + var counts = new AssetsCount(); + + foreach (var type in Enum.GetValues()) + { + var reqData = new TradeRepublicSearchData() + { + Query = "", + Page = 1, + PageSize = 1, + Filter = + [ + new TradeRepublicFilter("type", type.ToString().ToLowerInvariant()), + new TradeRepublicFilter("jurisdiction", "DE"), + ] + }; + + + + var request = new TradeRepublicSearchRequest(Data: reqData); + var response = + await _client.SendRequestAsync(request); + + var count = response?.ResultCount ?? 0; + + counts.SetCountOfType(type, count); + + await Task.Delay(TimeSpan.FromMilliseconds(320), cancellationToken); + } + + return counts; + } + + /// + /// Retrieves a paginated chunk of assets filtered by a specific type. + /// + /// The type of assets to retrieve (e.g., Stock, Etf). + /// The zero-based page index. + /// The number of elements per page. + /// A token to monitor for cancellation requests. + /// A containing the elements, or null if the request fails. + public async Task GetAssets(AssetType type, int page, int pageSize, + CancellationToken cancellationToken = default) + { + await EnsureConnectedAsync(); + + var reqData = new TradeRepublicSearchData() + { + Query = "", + Page = page, + PageSize = pageSize, + Filter = + [ + new TradeRepublicFilter("type", type.ToString().ToLowerInvariant()), + new TradeRepublicFilter("jurisdiction", "DE"), + ] + }; + + var request = new TradeRepublicSearchRequest(Data: reqData); + + return await _client.SendRequestAsync(request); + } + + /// + /// Retrieves the static metadata for a single specific asset via its ISIN. + /// + /// The International Securities Identification Number of the target asset. + /// A token to monitor for cancellation requests. + /// A containing instrument details, or null if the asset is not found. + public async Task GetAsset(string isin, CancellationToken cancellationToken = default) + { + try + { + await EnsureConnectedAsync(); + + + var reqData = new TradeRepublicSearchData() + { + Query = isin, + Page = 1, + PageSize = 1, + Filter = + [ + new TradeRepublicFilter("jurisdiction", "DE"), + ] + }; + + var request = new TradeRepublicSearchRequest(Data: reqData); + + return await _client.SendRequestAsync(request); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error while fetching asset metadata for ISIN {Isin}", isin); + return null; + } + } + + /// + /// Event handler executed when the inactivity timer expires. + /// Gracefully disconnects the WebSocket client. + /// + /// The source of the event. + /// An EventData object that contains the event data. + private async void OnInactivityTimeout(object? sender, ElapsedEventArgs e) + { + try + { + await _lock.WaitAsync(); + + if (!_client.IsConnected) return; + + _logger.LogInformation("No active requests detected for 5 minutes. Automatically disconnecting WebSocket."); + await _client.DisconnectAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during automatic inactivity disconnect procedure."); + //ignore + } + finally + { + _lock.Release(); + } + } + + /// + /// Disposes the underlying timer and synchronization primitives. + /// + public void Dispose() + { + _inactivityTimer.Dispose(); + _lock.Dispose(); + GC.SuppressFinalize(this); + } +} \ No newline at end of file diff --git a/FinlyticAssets/Util/AssetsMqttClient.cs b/FinlyticAssets/Util/AssetsMqttClient.cs new file mode 100644 index 0000000..a6d6e62 --- /dev/null +++ b/FinlyticAssets/Util/AssetsMqttClient.cs @@ -0,0 +1,93 @@ +using System.Text.Json; +using FinlyticAssets.Services; +using FinlyticCore.Entities.Assets; +using FinlyticCore.Models.Assets; +using FinlyticCore.Util; + +namespace FinlyticAssets.Util; + +/// +/// Represents a managed MQTT client acting as a server-side RPC provider within the asset microservice. +/// It subscribes to request topics, processes incoming JSON payloads via the database service, and publishes +/// the requested asset entities back to the corresponding response topic. +/// +public class AssetsMqttClient : ManagedMqttClient +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The logger used to record connection, error, and status messages. + /// The database service used for querying and validating assets. + public AssetsMqttClient(ILogger logger, IServiceScopeFactory scopeFactory) + : base(logger) + { + _scopeFactory = scopeFactory; + _logger = logger; + } + + /// + /// Invoked automatically once the connection to the MQTT broker is successfully established or restored. + /// Registers the required wildcard subscriptions for incoming asset validation and search requests. + /// + /// A representing the asynchronous subscription operation. + protected override async Task OnConnectedAsync() + { + await SubscribeAsync("services/request/assets_Get/#"); + await SubscribeAsync("services/request/assets_Search/#"); + } + + /// + /// Processes incoming messages on the subscribed topics, executes the corresponding database query, + /// and publishes the result to the response topic while preserving the correlation ID. + /// + /// The MQTT topic on which the message was received. + /// The incoming message as a UTF-8 encoded JSON string. + /// A representing the asynchronous message processing operation. + protected override async Task OnMessageReceivedAsync(string topic, string payload) + { + var segments = topic.Split('/'); + if (segments.Length < 4) return; + + var channel = segments[2]; + var correlationId = segments[3]; + + try + { + List responseData = []; + + using var scope = _scopeFactory.CreateScope(); + var dbService = scope.ServiceProvider.GetRequiredService(); + + switch (channel) + { + case "assets_Get": + var validReq = JsonSerializer.Deserialize(payload); + if (validReq != null) + { + responseData = await dbService.GetValidAssetsByIsinAsync(validReq.Isin); + } + break; + + case "assets_Search": + var searchReq = JsonSerializer.Deserialize(payload); + if (searchReq != null) + { + responseData = await dbService.FindAffectedActiveAssetsAsync(searchReq.SearchQuery); + } + break; + } + + { + string responseTopic = $"services/response/{channel}/{correlationId}"; + await PublishAsync(responseTopic, responseData.ToDtoList()); + } + } + catch (Exception ex) + { + OnError(ex); + } + } +} \ No newline at end of file diff --git a/FinlyticAssets/Util/StringCodeGenerator.cs b/FinlyticAssets/Util/StringCodeGenerator.cs new file mode 100644 index 0000000..5ddbd27 --- /dev/null +++ b/FinlyticAssets/Util/StringCodeGenerator.cs @@ -0,0 +1,14 @@ +namespace FinlyticAssets.Util; + +public class StringCodeGenerator +{ + + public static string GenerateTraceparent() + { + var traceId = Guid.NewGuid().ToString("N"); + var spanId = Guid.NewGuid().ToString("N").Substring(0, 16); + + return $"00-{traceId}-{spanId}-01"; + } + +} \ No newline at end of file diff --git a/FinlyticAssets/Util/TradeRepublicClient.cs b/FinlyticAssets/Util/TradeRepublicClient.cs new file mode 100644 index 0000000..b5dc32c --- /dev/null +++ b/FinlyticAssets/Util/TradeRepublicClient.cs @@ -0,0 +1,252 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using FinlyticAssets.Models.DataToObject.TradeRepublic; + +namespace FinlyticAssets.Util; + +/// +/// A managed WebSocket client designed to communicate with the Trade Republic API. +/// Handles asynchronous requests, generic serialization, and automatic subscription management. +/// +public class TradeRepublicClient : ManagedWebSocket +{ + private readonly ILogger _logger; + private int _currentSub; + private readonly ConcurrentDictionary> _pendingRequests = new(); + + /// + /// Wird ausgelöst, wenn Trade Republic asynchrone Updates (z.B. Live-Preise) schickt, + /// auf die niemand aktiv per SendRequestAsync wartet. + /// + public event Action? UnhandledMessageReceived; + + /// + /// Wird ausgelöst, wenn Trade Republic Systemnachrichten oder Fehler ohne ID schickt. + /// + public event Action? SystemMessageReceived; + + /// + /// Initializes a new instance of the TradeRepublicClient. + /// Call InitAsync() afterwards to establish the connection. + /// + /// The logger instance for tracking socket events and errors. + public TradeRepublicClient(ILogger logger) + { + _logger = logger; + } + + /// + /// Asynchronously establishes the WebSocket connection to the Trade Republic API. + /// + public async Task InitAsync() + { + await ConnectAsync("wss://api.traderepublic.com/", TimeSpan.FromSeconds(10)); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + _pendingRequests.TryAdd(-1, tcs); + + try + { + var json = JsonSerializer.Serialize(new TradeRepublicConnectRequest()); + await SendAsync($"connect 34 {json}"); + + var res = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + if (string.IsNullOrWhiteSpace(res.Type)) return false; + + var isConnected = res.Type == "connected"; + + if (isConnected) + { + _logger.LogInformation("WebSocket connection to Trade Republic established."); + } + + return isConnected; + } + catch (TimeoutException) + { + _pendingRequests.TryRemove(-1, out _); + _logger.LogWarning("Timeout while waiting for response to ID {Id}.", -1); + return false; + } + catch (TaskCanceledException) + { + _logger.LogWarning( + "Trade Republic immediately rejected the request for ID {Id} (e.g., invalid ISIN or access denied).", + -1); + return false; + } + } + + /// + /// Sends a strongly-typed request to the API and waits for the corresponding response. + /// Automatically handles the subscription ID and unsubscribes after completion or failure. + /// + /// The expected type of the response payload. + /// The type of the request payload. + /// The request data to be serialized and sent. + /// The deserialized response object, or null if the request timed out or was canceled. + public async Task SendRequestAsync(TRequest request) + where TResponse : class where TRequest : class + { + var tempSub = Interlocked.Increment(ref _currentSub); + + var msg = $"sub {tempSub} {JsonSerializer.Serialize(request)}"; + + Console.WriteLine(msg); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + _pendingRequests.TryAdd(tempSub, tcs); + + await SendAsync(msg); + + try + { + var res = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + if (res.Type == null) + { + _logger.LogWarning("Trade Republic rejected the request for ID {Id} with message type '{Type}'.", tempSub, res.Type); + return null; + } + + if (!res.Type.Contains('A')) + { + return null; + } + + if (string.IsNullOrWhiteSpace(res.Data)) return null; + + if (typeof(TResponse) == typeof(string)) + { + return res.Data as TResponse; + } + else + { + return JsonSerializer.Deserialize(res.Data); + } + } + catch (TimeoutException) + { + _pendingRequests.TryRemove(tempSub, out _); + _logger.LogWarning("Timeout while waiting for response to ID {Id}.", tempSub); + return null; + } + catch (TaskCanceledException) + { + _logger.LogWarning( + "Trade Republic immediately rejected the request for ID {Id} (e.g., invalid ISIN or access denied).", + tempSub); + return null; + } + finally + { + if (IsConnected) + { + try + { + await SendAsync($"unsub {tempSub}"); + } + catch + { + //ignore + } + } + } + } +/// +/// Processes incoming WebSocket messages, extracting the JSON payload and resolving pending tasks. +/// +/// The raw text message received from the server. +protected override void OnMessageReceived(string message) +{ + if (string.IsNullOrWhiteSpace(message)) return; + + // 1. Handshake-Nachricht direkt abfangen + if (message == "connected") + { + if (_pendingRequests.TryRemove(-1, out var tcs)) + { + tcs.SetResult(new ReceivedMessage(-1, "connected", null)); + } + return; + } + + // 2. Erstes Leerzeichen finden, um die ID zu isolieren + var firstSpaceIndex = message.IndexOf(' '); + if (firstSpaceIndex <= 0) + { + _logger.LogWarning("Unknown message format received: {Message}", message); + SystemMessageReceived?.Invoke(message); + return; + } + + var idString = message[..firstSpaceIndex]; + if (!int.TryParse(idString, out var responseId)) + { + SystemMessageReceived?.Invoke(message); + return; + } + + // Der Rest nach der ID (z. B. "A {...}" oder "C") + var remainder = message[firstSpaceIndex..].Trim(); + + // 3. Nachrichtentyp ("A", "C", etc.) und JSON-Inhalt sauber trennen + var nextSpaceIndex = remainder.IndexOf(' '); + string msgType; + string? json = null; + + if (nextSpaceIndex == -1) + { + // Kein weiteres Leerzeichen vorhanden (wie bei "2 C") + msgType = remainder; + } + else + { + // Typ und JSON trennen (wie bei "2 A {...}") + msgType = remainder[..nextSpaceIndex].Trim(); + json = remainder[nextSpaceIndex..].Trim(); + } + + // 4. KORREKTUR: "C" signalisiert nur das Ende des Datenstroms auf dieser ID. + // Wir ignorieren es, da die Daten bereits im Typ "A" übertragen wurden. + if (msgType == "C") + { + _logger.LogDebug("Trade Republic closed subscription channel for ID {ResponseId}.", responseId); + return; + } + + // 5. Task auflösen, falls jemand auf diese ID wartet + if (_pendingRequests.TryRemove(responseId, out var pendingTcs)) + { + pendingTcs.SetResult(new ReceivedMessage(responseId, msgType, json)); + } + else + { + UnhandledMessageReceived?.Invoke(new ReceivedMessage(responseId, msgType, json)); + } +} + + /// + /// Determines whether the incoming message is a keep-alive echo response. + /// + /// The raw text message. + /// True if the message is an echo response; otherwise, false. + protected override bool IsKeepAliveMessage(string message) + { + return message.StartsWith("echo"); + } + + /// + /// Sends a periodic keep-alive echo to maintain the WebSocket connection. + /// + protected override Task SendLifeMessageAsync() + { + var echo = $"echo {DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + return SendAsync(echo); + } +} + +public record ReceivedMessage(int? Sub, string? Type, string? Data); \ No newline at end of file diff --git a/FinlyticAssets/Util/Volumes.cs b/FinlyticAssets/Util/Volumes.cs new file mode 100644 index 0000000..aa71727 --- /dev/null +++ b/FinlyticAssets/Util/Volumes.cs @@ -0,0 +1,9 @@ +namespace FinlyticAssets.Util; + +public class Volumes +{ + /// + /// Der relative Pfad für die schlanke Index-Datei (ISINs + Namen) zur Asset-Erkennung. + /// + public const string IndexRelativePath = "assets/index"; +} \ No newline at end of file diff --git a/FinlyticAssets/appsettings.json b/FinlyticAssets/appsettings.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/FinlyticAssets/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/FinlyticCore/FinlyticCore.csproj b/FinlyticCore/FinlyticCore.csproj new file mode 100644 index 0000000..858b750 --- /dev/null +++ b/FinlyticCore/FinlyticCore.csproj @@ -0,0 +1,14 @@ + + + + net10.0 + enable + enable + + + + + + + + diff --git a/FinlyticCore/Models/MqttConfiguration.cs b/FinlyticCore/Models/MqttConfiguration.cs new file mode 100644 index 0000000..272209e --- /dev/null +++ b/FinlyticCore/Models/MqttConfiguration.cs @@ -0,0 +1,32 @@ +namespace FinlyticCore.Models; + +/// +/// Represents the network and security settings used to connect to the central MQTT broker. +/// +public class MqttConfiguration +{ + /// + /// Gets or sets the host address or IP of the MQTT broker. + /// + public string Host { get; set; } = string.Empty; + + /// + /// Gets or sets the port number for the connection. Defaults to 1883. + /// + public int Port { get; set; } = 1883; + + /// + /// Gets or sets the unique client identifier used when registering with the broker. + /// + public string ClientId { get; set; } = string.Empty; + + /// + /// Gets or sets the username for authentication (optional). + /// + public string? Username { get; set; } + + /// + /// Gets or sets the password for authentication (optional). + /// + public string? Password { get; set; } +} \ No newline at end of file diff --git a/FinlyticCore/Util/AssetMapper.cs b/FinlyticCore/Util/AssetMapper.cs new file mode 100644 index 0000000..2a68d77 --- /dev/null +++ b/FinlyticCore/Util/AssetMapper.cs @@ -0,0 +1,65 @@ +using FinlyticCore.Dtos.Assets; +using FinlyticCore.Entities.Assets; + +namespace FinlyticCore.Util; + +public static class AssetMapper +{ + /// + /// Mappt eine AssetEntity (Datenbank) sicher auf ein zyklusfreies AssetDto (MQTT Payload). + /// + public static AssetDto ToDto(this AssetEntity entity) + { + // 1. Tags zyklusfrei mappen + var dtoTags = entity.Tags.Select(t => new TagDto + { + Id = t.Id, + Name = t.Name, + Type = t.Type + }).ToList(); + + // 2. Polymorphes Mapping basierend auf dem Laufzeittyp + return entity switch + { + StockEntity stock => new StockDto + { + Isin = stock.Isin, Name = stock.Name, Type = stock.Type, InstrumentCategory = stock.InstrumentCategory, HasCfd = stock.HasCfd, ImageId = stock.ImageId, UpdateAt = stock.UpdateAt, LastUpdatedAt = stock.LastUpdatedAt, Tags = dtoTags, + DerivativeProductCategories = stock.DerivativeProductCategories + }, + EtfEntity etf => new EtfDto + { + Isin = etf.Isin, Name = etf.Name, Type = etf.Type, InstrumentCategory = etf.InstrumentCategory, HasCfd = etf.HasCfd, ImageId = etf.ImageId, UpdateAt = etf.UpdateAt, LastUpdatedAt = etf.LastUpdatedAt, Tags = dtoTags, + DerivativeProductCategories = etf.DerivativeProductCategories, EtfDescription = etf.EtfDescription, MappedEtfIndexName = etf.MappedEtfIndexName, Subtitle = etf.Subtitle, SearchSubtitle = etf.SearchSubtitle + }, + CryptoEntity crypto => new CryptoDto + { + Isin = crypto.Isin, Name = crypto.Name, Type = crypto.Type, InstrumentCategory = crypto.InstrumentCategory, HasCfd = crypto.HasCfd, ImageId = crypto.ImageId, UpdateAt = crypto.UpdateAt, LastUpdatedAt = crypto.LastUpdatedAt, Tags = dtoTags, + Subtitle = crypto.Subtitle, SearchSubtitle = crypto.SearchSubtitle + }, + BondEntity bond => new BondDto + { + Isin = bond.Isin, Name = bond.Name, Type = bond.Type, InstrumentCategory = bond.InstrumentCategory, HasCfd = bond.HasCfd, ImageId = bond.ImageId, UpdateAt = bond.UpdateAt, LastUpdatedAt = bond.LastUpdatedAt, Tags = dtoTags, + BondIssuerName = bond.BondIssuerName, SearchSubtitle = bond.SearchSubtitle + }, + DerivativeEntity deriv => new DerivativeDto + { + Isin = deriv.Isin, Name = deriv.Name, Type = deriv.Type, InstrumentCategory = deriv.InstrumentCategory, HasCfd = deriv.HasCfd, ImageId = deriv.ImageId, UpdateAt = deriv.UpdateAt, LastUpdatedAt = deriv.LastUpdatedAt, Tags = dtoTags, + DerivativeProductCategories = deriv.DerivativeProductCategories, UnderlyingIsin = deriv.UnderlyingIsin + }, + SyntheticEntity synth => new SyntheticDto + { + Isin = synth.Isin, Name = synth.Name, Type = synth.Type, InstrumentCategory = synth.InstrumentCategory, HasCfd = synth.HasCfd, ImageId = synth.ImageId, UpdateAt = synth.UpdateAt, LastUpdatedAt = synth.LastUpdatedAt, Tags = dtoTags, + DerivativeProductCategories = synth.DerivativeProductCategories + }, + _ => throw new NotSupportedException($"Mapping for type {entity.GetType().Name} is not supported.") + }; + } + + /// + /// Mappt direkt eine ganze Liste von AssetEntities. + /// + public static List ToDtoList(this IEnumerable entities) + { + return entities.Select(e => e.ToDto()).ToList(); + } +} \ No newline at end of file diff --git a/FinlyticCore/Util/ManagedMqttClient.cs b/FinlyticCore/Util/ManagedMqttClient.cs new file mode 100644 index 0000000..1f71c95 --- /dev/null +++ b/FinlyticCore/Util/ManagedMqttClient.cs @@ -0,0 +1,307 @@ +using System; +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using FinlyticCore.Models; +using Microsoft.Extensions.Logging; +using MQTTnet; + +namespace FinlyticCore.Util; + +/// +/// An abstract, resilient MQTT client wrapper designed for microservice architectures. +/// Handles automatic reconnection, structured JSON publishing, thread-safe subscription management, and synchronous Request-Reply (RPC). +/// +public abstract class ManagedMqttClient : IDisposable +{ + private readonly ILogger _logger; + private readonly IMqttClient _mqttClient; + private CancellationTokenSource? _cts; + + // Tracks pending RPC requests waiting for a specific correlation ID reply + private readonly ConcurrentDictionary> _pendingRequests = new(); + + /// + /// Gets a value indicating whether the client is currently connected to the MQTT broker. + /// + public bool IsConnected => _mqttClient.IsConnected; + + protected ManagedMqttClient(ILogger logger) + { + _logger = logger; + _mqttClient = new MqttClientFactory().CreateMqttClient(); + + _mqttClient.ApplicationMessageReceivedAsync += HandleIncomingMessageAsync; + _mqttClient.DisconnectedAsync += HandleDisconnectAsync; + } + + /// + /// Establishes a connection to the MQTT broker and initializes the background auto-reconnection loop. + /// + /// The network and credential configuration options for the broker. + public async Task ConnectAsync(MqttConfiguration config) + { + if (IsConnected) + throw new InvalidOperationException("MQTT client is already connected."); + + _cts = new CancellationTokenSource(); + + var optionsBuilder = new MqttClientOptionsBuilder() + .WithTcpServer(config.Host, config.Port) + .WithClientId(config.ClientId) + .WithCleanSession(); + + if (!string.IsNullOrWhiteSpace(config.Username)) + { + optionsBuilder.WithCredentials(config.Username, config.Password); + } + + var options = optionsBuilder.Build(); + + _logger.LogInformation("Connecting to MQTT broker at {Host}:{Port}...", config.Host, config.Port); + + try + { + await _mqttClient.ConnectAsync(options, _cts.Token); + _logger.LogInformation("Successfully connected to MQTT broker."); + + await OnConnectedAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to establish initial connection to MQTT broker. Reconnection loop will handle recovery."); + } + } + + /// + /// Gracefully disconnects from the broker and stops all ongoing background loops. + /// + public async Task DisconnectAsync() + { + if (_cts != null) + { + await _cts.CancelAsync(); + } + + if (_mqttClient.IsConnected) + { + try + { + await _mqttClient.DisconnectAsync(new MqttClientDisconnectOptions + { + Reason = MqttClientDisconnectOptionsReason.NormalDisconnection + }); + _logger.LogInformation("MQTT connection gracefully closed."); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "An error occurred while disconnecting from the MQTT broker."); + } + } + } + + /// + /// Subscribes to a specific MQTT topic filter. + /// + /// The topic pattern or wildcard to subscribe to. + /// If set to true, the broker will not forward messages published by this client back to itself. + protected async Task SubscribeAsync(string topic, bool noLocal = false) + { + if (!IsConnected) + { + _logger.LogWarning("Subscription to topic '{Topic}' delayed: Client is currently offline.", topic); + return; + } + + var filterBuilder = new MqttTopicFilterBuilder().WithTopic(topic); + if (noLocal) + { + filterBuilder.WithNoLocal(); + } + + var subscribeOptions = new MqttClientFactory().CreateSubscribeOptionsBuilder() + .WithTopicFilter(filterBuilder.Build()) + .Build(); + + await _mqttClient.SubscribeAsync(subscribeOptions, CancellationToken.None); + _logger.LogDebug("Successfully subscribed to topic: {Topic} (NoLocal: {NoLocal})", topic, noLocal); + } + + /// + /// Publishes a raw string message payload to the specified topic. + /// + public async Task PublishAsync(string topic, string payload, bool retain = false) + { + if (!IsConnected) + throw new InvalidOperationException("Cannot publish message: MQTT client is offline."); + + var message = new MqttApplicationMessageBuilder() + .WithTopic(topic) + .WithPayload(payload) + .WithQualityOfServiceLevel(MQTTnet.Protocol.MqttQualityOfServiceLevel.AtLeastOnce) + .WithRetainFlag(retain) + .Build(); + + await _mqttClient.PublishAsync(message, CancellationToken.None); + } + + /// + /// Serializes a generic object into a structured JSON string and publishes it to the specified topic. + /// + public Task PublishAsync(string topic, T data, bool retain = false) + { + var jsonOptions = new JsonSerializerOptions + { + ReferenceHandler = ReferenceHandler.IgnoreCycles + }; + var json = JsonSerializer.Serialize(data, jsonOptions); + return PublishAsync(topic, json, retain); + } + + /// + /// Sends a generic request payload to an RPC channel and asynchronously blocks until a matching response arrives. + /// Uses the topic conventions: services/request/{channel}/{correlationId} and services/response/{channel}/{correlationId}. + /// + /// The expected strongly-typed object type of the reply. + /// The type of the payload being transmitted. + /// The target sub-channel or service name (e.g., "sentix", "assets"). + /// The object that will be serialized to JSON and sent. + /// Optional. Maximum time to wait before returning null. Defaults to 10 seconds. + public async Task SendRpcRequestAsync( + string channel, + TRequest requestData, + TimeSpan? timeout = null) + where TResponse : class + where TRequest : class + { + if (!IsConnected) + throw new InvalidOperationException("Cannot execute RPC request: MQTT client is offline."); + + // 1. Generate a unique Correlation ID for this specific transaction + string correlationId = Guid.NewGuid().ToString("N"); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _pendingRequests.TryAdd(correlationId, tcs); + + string requestTopic = $"services/request/{channel}/{correlationId}"; + + // 2. Serialize and dispatch via the existing JSON helper + await PublishAsync(requestTopic, requestData); + _logger.LogDebug("RPC request published to '{Topic}' [CorrelationId: {Id}]", requestTopic, correlationId); + + try + { + // 3. Block asynchronously until the response loop resolves the token + var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(10); + var rawJsonResult = await tcs.Task.WaitAsync(effectiveTimeout); + + if (typeof(TResponse) == typeof(string)) + { + return rawJsonResult as TResponse; + } + + return JsonSerializer.Deserialize(rawJsonResult); + } + catch (TimeoutException) + { + _logger.LogWarning("RPC request timed out on channel '{Channel}' [CorrelationId: {Id}]", channel, correlationId); + return null; + } + finally + { + // Always clean up the dictionary to prevent memory leaks + _pendingRequests.TryRemove(correlationId, out _); + } + } + + private async Task HandleIncomingMessageAsync(MqttApplicationMessageReceivedEventArgs e) + { + try + { + var topic = e.ApplicationMessage.Topic; + var payload = Encoding.UTF8.GetString(e.ApplicationMessage.Payload); + + // Intercept message if it belongs to the RPC response convention + if (topic.StartsWith("services/response/")) + { + var lastSlashIndex = topic.LastIndexOf('/'); + if (lastSlashIndex != -1) + { + string correlationId = topic[(lastSlashIndex + 1)..]; + + if (_pendingRequests.TryRemove(correlationId, out var tcs)) + { + tcs.SetResult(payload); + return; // Sinks the message, avoiding triggering OnMessageReceivedAsync for active RPC handles + } + } + } + + // Regular Pub/Sub message propagation + await OnMessageReceivedAsync(topic, payload); + } + catch (Exception ex) + { + OnError(ex); + } + } + + private async Task HandleDisconnectAsync(MqttClientDisconnectedEventArgs e) + { + // Prevent trigger during deliberate connection shutdowns + if (_cts == null || _cts.IsCancellationRequested) + return; + + _logger.LogWarning("Lost connection to MQTT broker (Reason: {Reason}). Initiating auto-reconnect loop in 5 seconds...", e.Reason); + + try + { + await Task.Delay(TimeSpan.FromSeconds(5), _cts.Token); + + await _mqttClient.ReconnectAsync(_cts.Token); + + if (_mqttClient.IsConnected) + { + _logger.LogInformation("MQTT client reconnected successfully."); + await OnConnectedAsync(); + } + } + catch (OperationCanceledException) { /* Expected swallow on application shutdown */ } + catch (Exception ex) + { + _logger.LogError(ex, "Reconnection attempt to the MQTT broker failed."); + } + } + + /// + /// Fired automatically whenever a connection or reconnection is successfully established. + /// Ideal place to trigger operations. + /// + protected abstract Task OnConnectedAsync(); + + /// + /// Fired whenever a new message lands on a registered subscription channel. + /// + /// The specific topic where the message was broadcasted. + /// The deserialized UTF-8 payload string. + protected abstract Task OnMessageReceivedAsync(string topic, string payload); + + /// + /// Virtual fallback method to catch and handle processing level exceptions inside the incoming pipeline. + /// + protected virtual void OnError(Exception ex) + { + _logger.LogError(ex, "An unhandled exception occurred within the ManagedMqttClient messaging pipeline."); + } + + public void Dispose() + { + DisconnectAsync().GetAwaiter().GetResult(); + _cts?.Dispose(); + _mqttClient.Dispose(); + GC.SuppressFinalize(this); + } +} \ No newline at end of file diff --git a/FinlyticCore/Util/ManagedWebSocket.cs b/FinlyticCore/Util/ManagedWebSocket.cs new file mode 100644 index 0000000..19c7537 --- /dev/null +++ b/FinlyticCore/Util/ManagedWebSocket.cs @@ -0,0 +1,234 @@ +namespace FinlyticAssets.Util; + +using System; +using System.IO; +using System.Net.WebSockets; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +public abstract class ManagedWebSocket : IDisposable +{ + private ClientWebSocket? _webSocket; + private CancellationTokenSource? _cts; + private Task? _receiveTask; + private Task? _keepAliveTask; + + /// + /// Gets or sets the interval for keep-alive messages. + /// If set to null, the periodic timer is not started and no life messages are sent. + /// + public TimeSpan? KeepAliveInterval { get; set; } + + /// + /// Gets a value indicating whether the WebSocket is currently open. + /// + public bool IsConnected => _webSocket?.State == WebSocketState.Open; + + /// + /// Gets the current state of the WebSocket connection. + /// + public WebSocketState State => _webSocket?.State ?? WebSocketState.None; + + /// + /// Connects to the specified WebSocket URI and starts the background tasks for receiving and keep-alive. + /// + /// The URI of the WebSocket server to connect to. + /// Optional. Sets or overrides the keep-alive interval for this connection. + public async Task ConnectAsync(string uri, TimeSpan? keepAliveInterval = null) + { + if (IsConnected) + throw new InvalidOperationException("WebSocket is already connected."); + + if (keepAliveInterval.HasValue) + { + KeepAliveInterval = keepAliveInterval; + } + + _webSocket = new ClientWebSocket(); + _cts = new CancellationTokenSource(); + + await _webSocket.ConnectAsync(new Uri(uri), _cts.Token); + + _receiveTask = ReceiveLoopAsync(_cts.Token); + + if (KeepAliveInterval.HasValue && KeepAliveInterval.Value > TimeSpan.Zero) + { + _keepAliveTask = KeepAliveLoopAsync(_cts.Token); + } + } + + /// + /// Closes the WebSocket connection gracefully and cancels all running background tasks. + /// + public async Task DisconnectAsync() + { + if (_webSocket != null) + { + if (_cts != null) + { + await _cts.CancelAsync(); + } + + var tasksToWait = new List(); + if (_receiveTask != null) tasksToWait.Add(_receiveTask); + if (_keepAliveTask != null) tasksToWait.Add(_keepAliveTask); + + if (tasksToWait.Count > 0) + { + try + { + await Task.WhenAll(tasksToWait); + } + catch (OperationCanceledException) { /* Ignore */ } + catch { /* Ignore */ } + } + + if (_webSocket.State == WebSocketState.Open) + { + try + { + await _webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Connection closed by client", CancellationToken.None); + } + catch { /* Ignore exceptions during closure */ } + } + + _webSocket.Dispose(); + _webSocket = null; + + _receiveTask = null; + _keepAliveTask = null; + } + } + /// + /// Sends a raw string message over the WebSocket connection. + /// + /// The text message to send. + public async Task SendAsync(string message) + { + if (!IsConnected || _webSocket == null || _cts == null) + throw new InvalidOperationException("WebSocket is not connected."); + + var buffer = Encoding.UTF8.GetBytes(message); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, _cts.Token); + } + + /// + /// Serializes the given generic object to JSON and sends it over the WebSocket connection. + /// + /// The type of the object to send. + /// The object to serialize and send. + public Task SendAsync(T data) + { + var json = JsonSerializer.Serialize(data); + + return SendAsync(json); + } + + private async Task ReceiveLoopAsync(CancellationToken cancellationToken) + { + var buffer = new byte[8192]; + + try + { + while (_webSocket?.State == WebSocketState.Open && !cancellationToken.IsCancellationRequested) + { + using var ms = new MemoryStream(); + WebSocketReceiveResult result; + + do + { + result = await _webSocket.ReceiveAsync(new ArraySegment(buffer), cancellationToken); + ms.Write(buffer, 0, result.Count); + } + while (!result.EndOfMessage); + + if (result.MessageType == WebSocketMessageType.Close) + { + await DisconnectAsync(); + break; + } + + if (result.MessageType == WebSocketMessageType.Text) + { + var message = Encoding.UTF8.GetString(ms.ToArray()); + + if (!IsKeepAliveMessage(message)) + { + OnMessageReceived(message); + } + } + } + } + catch (OperationCanceledException) { /* Expected behavior when cancellation is requested */ } + catch (Exception ex) + { + OnError(ex); + } + } + + private async Task KeepAliveLoopAsync(CancellationToken cancellationToken) + { + if (!KeepAliveInterval.HasValue) return; + + using var timer = new PeriodicTimer(KeepAliveInterval.Value); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + if (IsConnected) + { + await SendLifeMessageAsync(); + } + } + } + catch (OperationCanceledException) { /* Expected behavior when cancellation is requested */ } + } + + /// + /// Triggered when a new text message is received that is not filtered as a keep-alive message. + /// + /// The received message content. + protected abstract void OnMessageReceived(string message); + + /// + /// Triggered periodically based on the KeepAliveInterval to send a life message or ping to the server. + /// Override this method in the derived class if you want to send actual keep-alive payloads. + /// + protected virtual Task SendLifeMessageAsync() + { + // Default implementation does nothing. + return Task.CompletedTask; + } + + /// + /// Determines whether the incoming message is a keep-alive response (e.g., a "pong"). + /// If true is returned, the message is filtered out and OnMessageReceived is not triggered. + /// + /// The incoming text message. + /// True if the message should be filtered; otherwise, false. + protected virtual bool IsKeepAliveMessage(string message) + { + // Default implementation does not filter any messages. + return false; + } + + /// + /// Optionally overridable method to handle exceptions that occur within the receive loop. + /// + /// The exception that was caught. + protected virtual void OnError(Exception ex) + { + // Default implementation is empty. Can be overridden in the derived class. + } + + /// + /// Disposes the WebSocket connection and releases all associated resources. + /// + public void Dispose() + { + DisconnectAsync().Wait(); + _cts?.Dispose(); + } +} \ No newline at end of file diff --git a/FinlyticNews/Dockerfile b/FinlyticNews/Dockerfile new file mode 100644 index 0000000..1786e80 --- /dev/null +++ b/FinlyticNews/Dockerfile @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/dotnet/runtime:10.0 AS base +USER $APP_UID +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["FinlyticNews/FinlyticNews.csproj", "FinlyticNews/"] +RUN dotnet restore "FinlyticNews/FinlyticNews.csproj" +COPY . . +WORKDIR "/src/FinlyticNews" +RUN dotnet build "./FinlyticNews.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./FinlyticNews.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "FinlyticNews.dll"] diff --git a/FinlyticNews/FinlyticNews.csproj b/FinlyticNews/FinlyticNews.csproj new file mode 100644 index 0000000..edbe331 --- /dev/null +++ b/FinlyticNews/FinlyticNews.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + dotnet-FinlyticNews-59a2d28c-0353-4726-b60f-9ac53f5100a0 + Linux + + + + + + + + + .dockerignore + + + + + + + + + diff --git a/FinlyticNews/Program.cs b/FinlyticNews/Program.cs new file mode 100644 index 0000000..ed83a68 --- /dev/null +++ b/FinlyticNews/Program.cs @@ -0,0 +1,5 @@ + +var builder = Host.CreateApplicationBuilder(args); + +var host = builder.Build(); +host.Run(); \ No newline at end of file diff --git a/FinlyticNews/Properties/launchSettings.json b/FinlyticNews/Properties/launchSettings.json new file mode 100644 index 0000000..4c45b1a --- /dev/null +++ b/FinlyticNews/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "FinlyticNews": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/FinlyticNews/appsettings.json b/FinlyticNews/appsettings.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/FinlyticNews/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/Project.md b/Project.md new file mode 100644 index 0000000..0a3802a --- /dev/null +++ b/Project.md @@ -0,0 +1,66 @@ +# Finlytic Assets Service + +Finlytic Assets is a scalable C# microservice designed for asset discovery, metadata ingestion, and quick querying. It interfaces directly with the Trade Republic API via WebSockets and exposes a high-performance RPC interface over MQTT to other services in the Finlytic ecosystem. + +--- + +## Architecture Overview + +```mermaid +graph TD + TR[Trade Republic WebSocket API] <-->|WS Protocol| TRS[TradeRepublicService] + TRS <-->|Ingest| ADS[AssetsFullScanService] + ADS <-->|Save / Update| DB[PostgreSQL Database] + DB -->|Trigger Index Update| AIS[AssetsIndexService] + AIS -->|Write Cache| Index[assets/index/index.json] + + MS[Other Finlytic Services] <-->|MQTT Request-Reply| MqttClient[AssetsMqttClient] + MqttClient <-->|Query Cache| DB +``` + +--- + +## Core Features & Workflows + +### 1. Automated WebSocket scraping +- **Continuous Scan**: The background worker (`AssetsFullScanService`) iterates through all support asset types: + - **Stocks** (`StockEntity`) + - **Crypto** (`CryptoEntity`) + - **Derivatives** (`DerivativeEntity`) + - **Bonds** (`BondEntity`) + - **Funds/ETFs** (`EtfEntity`) +- **Resilient Reconnection**: Uses a robust custom socket wrapper (`TradeRepublicClient`) that closes automatically after 5 minutes of inactivity to mimic human interaction profiles and reconnects dynamically when a request is made. + +### 2. Stealth Scheduling with Randomized Jitter +- **Dynamic Delays**: Scanner wait times are read dynamically from database configuration (`Settings`). +- **Jitter Offset**: High/low jitter values (randomized offsets between batches and asset types) are injected to avoid predictable traffic patterns and prevent Trade Republic rate limiting. + +### 3. Active vs. Dead Asset Tracking +- **Recency Filter**: Assets are continuously updated with their latest metadata. If an asset has not been updated within **14 days**, it is considered inactive, de-listed, or "dead". +- **Implicit Filtration**: Queries by other services only return active/valid records unless explicitly requested otherwise. + +### 4. MQTT RPC Interface +- **Get Asset by ISIN**: + - Subscribes to: `services/request/assets_Get/#` + - Returns a list of all active assets matching the specified ISIN across different instrument types (e.g., matching both a stock and its derivative tracker). +- **Omnibox Search**: + - Subscribes to: `services/request/assets_Search/#` + - Evaluates search keywords against local records (matching `ISIN`, `Name`, or tags like `Region`, `Country`, `Sector`, etc.). + - **JIT Fallback**: If a search query is formatted as an unknown valid ISIN, it performs a JIT-lookup directly against the Trade Republic API and registers the discovered asset before replying. + +### 5. Local Indexing File (`index.json`) +- **Automatic Regeneration**: On database modifications, `AssetsIndexService` is triggered to regenerate `assets/index/index.json`. +- **Pre-Filtering**: This lean index (containing only `ISIN` and `Name` properties) is written to a shared volume to allow other services to perform instant pre-filtering without querying the database or sending network requests. + +--- + +## Technical Components + +| Component | Class / Interface | Responsibility | +| :--- | :--- | :--- | +| **Ingestion Worker** | `AssetsFullScanService` | Runs the main cron loop, scraping and paginating all Trade Republic asset types. | +| **Database Store** | [IAssetsDbService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/AssetsDbService.cs) | Handles CRUD operations, stateful merges, JIT lookups, and active record filtering. | +| **WebSocket Handler** | [ITradeRepublicService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/TradeRepublicService.cs) | Wraps the connection and request dispatching for the Trade Republic API. | +| **Local Indexer** | [IAssetsIndexService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/AssetsIndexService.cs) | Serializes the active assets list to `index.json` for external service indexing. | +| **Messaging Broker** | `AssetsMqttClient` | Listens for incoming MQTT requests, queries the database, and responds to RPC channels. | +| **Config Service** | [ISettingsDbService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/SettingsDbService.cs) | Manages scanner configurations (scan intervals, active pointers, offsets). | diff --git a/Rules.md b/Rules.md new file mode 100644 index 0000000..71d02dd --- /dev/null +++ b/Rules.md @@ -0,0 +1,20 @@ +# Development Rules & Guidelines + +To ensure scalability, readability, and consistency across all microservices, the following architectural and coding rules must be adhered to: + +## 1. Service Interfaces and Implementations +- **Single File Co-location**: Every service interface (e.g., `IAssetsDbService`) and its corresponding implementation class (e.g., `AssetsDbService`) **must** reside in the same file. +- **Naming Convention**: The file must be named after the implementation class (e.g., `AssetsDbService.cs`). + +## 2. Mandatory Method Documentation +- **English Language**: All documentation must be written in English. +- **XML Documentation**: Every method (public, private, internal, or protected) must have proper XML documentation. +- **Required Fields**: + - ``: Clarifying the purpose and behavior of the method. + - ``: Explaining each parameter (if applicable). + - ``: Specifying what the method returns (if applicable). +- **Implementation Inheritdoc**: For class methods implementing interfaces or overriding base methods, use `/// ` to inherit documentation unless customization is needed. Note that any custom helper or private methods in the implementation must still have their own explicit XML comments. + +## 3. Data Class & Model Architecture +- **Core Placement**: All data classes, DTOs, and shared enums (e.g., `AssetType`) that are or could be used by multiple services must be defined in the `FinlyticCore` project. +- **Service-Specific Exception**: Data classes and models may only reside in a specific service project (e.g., `FinlyticAssets`) if they strictly concern the internal operations of that service (e.g., raw API request/response structures for an integration that only that service manages). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..108540a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,35 @@ +services: + finlyticassets: + image: finlyticassets + build: + context: . + dockerfile: FinlyticAssets/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_assets;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + #- MQTT__Username=admin + #- MQTT__Password=${MQTT_PASSWORD} + - MQTT__ClientId=finlytic_assets + volumes: + - C:\Users\larsh\Documents\docker\finlytic\assets\index:/app/assets/index + + finlyticnews: + image: finlyticnews + build: + context: . + dockerfile: FinlyticNews/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_news;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + #- MQTT__Username=admin + #- MQTT__Password=${MQTT_PASSWORD} + - MQTT__ClientId=finlytic_news +networks: + postgres-network: + external: true