From 1447f0aa4c52fbee3ac8e200052b83e5ce4992ef Mon Sep 17 00:00:00 2001 From: Kleidukos Date: Fri, 14 Aug 2026 23:55:15 +0200 Subject: [PATCH] feat(fundamentals): refactor entities, add Yahoo modules scraper, 52W metrics and migrations --- .../Database/FundamentalsDbContext.cs | 103 +- .../Entities/AssetDataEntity.cs | 14 +- .../Entities/AssetEventEntity.cs | 19 +- .../Entities/FundamentalDataEntity.cs | 65 +- .../Entities/KeyExecutiveEntity.cs | 15 +- FinlyticFundamentals/Entities/TickerEntity.cs | 8 +- .../FinlyticFundamentals.csproj | 4 + .../20260814211010_Init.Designer.cs | 394 ++++++ .../Migrations/20260814211010_Init.cs | 206 +++ ...4749_AddMoreFundamentalMetrics.Designer.cs | 424 ++++++ ...0260814214749_AddMoreFundamentalMetrics.cs | 118 ++ .../FundamentalsDbContextModelSnapshot.cs | 421 ++++++ FinlyticFundamentals/Program.cs | 29 +- .../Services/FundamentalsDbService.cs | 1257 ++++++++--------- .../Services/HtmlFallbackScraper.cs | 305 ---- .../Services/YahooFinanceScraper.cs | 616 +++----- .../Util/FundamentalsMqttClient.cs | 113 +- FinlyticFundamentals/Util/SettingKeys.cs | 16 +- 18 files changed, 2565 insertions(+), 1562 deletions(-) create mode 100644 FinlyticFundamentals/Migrations/20260814211010_Init.Designer.cs create mode 100644 FinlyticFundamentals/Migrations/20260814211010_Init.cs create mode 100644 FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.Designer.cs create mode 100644 FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.cs create mode 100644 FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs delete mode 100644 FinlyticFundamentals/Services/HtmlFallbackScraper.cs diff --git a/FinlyticFundamentals/Database/FundamentalsDbContext.cs b/FinlyticFundamentals/Database/FundamentalsDbContext.cs index da65942..9537f59 100644 --- a/FinlyticFundamentals/Database/FundamentalsDbContext.cs +++ b/FinlyticFundamentals/Database/FundamentalsDbContext.cs @@ -1,3 +1,4 @@ +using FinlyticCore.Entities.Settings; using FinlyticFundamentals.Entities; using Microsoft.EntityFrameworkCore; @@ -9,74 +10,84 @@ public class FundamentalsDbContext : DbContext { } - public DbSet AssetFundamentals => Set(); - public DbSet CompanyExecutives => Set(); - public DbSet FinancialStatements => Set(); - public DbSet ForwardEstimates => Set(); - public DbSet TickerFundamentals => Set(); - public DbSet Settings => Set(); + public DbSet DynamicSettings => Set(); + public DbSet AssetData => Set(); + public DbSet FundamentalData => Set(); + public DbSet KeyExecutives => Set(); + public DbSet AssetEvents => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); - // AssetFundamentals Configurations - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Id); + entity.HasIndex(e => e.Key); + }); + + modelBuilder.Entity(entity => { entity.HasKey(e => e.Isin); - entity.HasIndex(e => e.PrimaryTicker).IsUnique(); - // Setup One-to-Many Relationships with cascades - entity.HasMany(e => e.Executives) - .WithOne(e => e.AssetFundamentals) - .HasForeignKey(e => e.Isin) + entity.OwnsOne(e => e.PrimaryTicker, t => + { + t.Property(p => p.Ticker).HasColumnName("PrimaryTicker").HasDefaultValue(string.Empty); + t.Property(p => p.Exchange).HasColumnName("PrimaryTickerExchange").HasDefaultValue(string.Empty); + }); + + entity.OwnsMany(e => e.AvailableTickers, t => + { + t.ToTable("Tickers"); + t.WithOwner().HasForeignKey("AssetDataIsin"); + t.Property("Id"); + t.HasKey("Id"); + t.Property(p => p.Ticker).HasColumnName("Ticker"); + t.Property(p => p.Exchange).HasColumnName("Exchange"); + t.HasIndex(p => p.Ticker); + }); + + entity.HasMany(e => e.FundamentalData) + .WithOne(e => e.AssetData) + .HasForeignKey(e => e.AssetDataIsin) .OnDelete(DeleteBehavior.Cascade); - entity.HasMany(e => e.FinancialStatements) - .WithOne(e => e.AssetFundamentals) - .HasForeignKey(e => e.Isin) + entity.HasMany(e => e.KeyExecutives) + .WithOne(e => e.AssetData) + .HasForeignKey(e => e.AssetDataIsin) .OnDelete(DeleteBehavior.Cascade); - entity.HasMany(e => e.Estimates) - .WithOne(e => e.AssetFundamentals) - .HasForeignKey(e => e.Isin) - .OnDelete(DeleteBehavior.Cascade); - - entity.HasMany(e => e.TickerFundamentals) - .WithOne(e => e.AssetFundamentals) - .HasForeignKey(e => e.Isin) + entity.HasMany(e => e.AssetEvents) + .WithOne(e => e.AssetData) + .HasForeignKey(e => e.AssetDataIsin) .OnDelete(DeleteBehavior.Cascade); }); - // CompanyExecutive Configurations - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Isin); + + entity.OwnsOne(e => e.Ticker, t => + { + t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty); + t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty); + }); + }); + + modelBuilder.Entity(entity => { entity.HasKey(e => e.Id); - entity.HasIndex(e => e.Isin); }); - // FinancialStatement Configurations - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => { entity.HasKey(e => e.Id); - entity.HasIndex(e => e.Isin); - // Compound Index to prevent duplicate statement entries - entity.HasIndex(e => new { e.Isin, e.PeriodType, e.EndDate }).IsUnique(); - }); - // ForwardEstimate Configurations - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.Id); - entity.HasIndex(e => e.Isin); - entity.HasIndex(e => new { e.Isin, e.Period }).IsUnique(); - }); - - // TickerFundamentals Configurations - modelBuilder.Entity(entity => - { - entity.HasKey(e => e.Ticker); - entity.HasIndex(e => e.Isin); + entity.OwnsOne(e => e.Ticker, t => + { + t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty); + t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty); + }); }); } } diff --git a/FinlyticFundamentals/Entities/AssetDataEntity.cs b/FinlyticFundamentals/Entities/AssetDataEntity.cs index 8de1c75..7c9f422 100644 --- a/FinlyticFundamentals/Entities/AssetDataEntity.cs +++ b/FinlyticFundamentals/Entities/AssetDataEntity.cs @@ -1,15 +1,19 @@ +using System.Collections.Generic; using System.ComponentModel.DataAnnotations; namespace FinlyticFundamentals.Entities; public class AssetDataEntity { + [Key] public string Isin { get; set; } = string.Empty; - [Key] public string Isin { get; set; } = ""; + public string Name { get; set; } = string.Empty; + public string Description { get; set; } = string.Empty; - public string Description { get; set; } = ""; + public TickerEntity PrimaryTicker { get; set; } = new(); + public List AvailableTickers { get; set; } = new(); - public ICollection KeyExecutives { get; set; } - - public ICollection AssetEvents { get; set; } + public ICollection KeyExecutives { get; set; } = new List(); + public ICollection AssetEvents { get; set; } = new List(); + public ICollection FundamentalData { get; set; } = new List(); } \ No newline at end of file diff --git a/FinlyticFundamentals/Entities/AssetEventEntity.cs b/FinlyticFundamentals/Entities/AssetEventEntity.cs index 2e0c528..9eebb6e 100644 --- a/FinlyticFundamentals/Entities/AssetEventEntity.cs +++ b/FinlyticFundamentals/Entities/AssetEventEntity.cs @@ -1,6 +1,23 @@ -namespace FinlyticFundamentals.Entities; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace FinlyticFundamentals.Entities; public class AssetEventEntity { + [Key] + public Guid Id { get; set; } = Guid.NewGuid(); + + public TickerEntity Ticker { get; set; } = new(); + + public string Type { get; set; } = string.Empty; + + public DateTime Date { get; set; } + + [Required] + public string AssetDataIsin { get; set; } = string.Empty; + + [ForeignKey(nameof(AssetDataIsin))] + public AssetDataEntity AssetData { get; set; } = null!; } \ No newline at end of file diff --git a/FinlyticFundamentals/Entities/FundamentalDataEntity.cs b/FinlyticFundamentals/Entities/FundamentalDataEntity.cs index 8da51fc..4c2dab9 100644 --- a/FinlyticFundamentals/Entities/FundamentalDataEntity.cs +++ b/FinlyticFundamentals/Entities/FundamentalDataEntity.cs @@ -1,6 +1,69 @@ -namespace FinlyticFundamentals.Entities; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace FinlyticFundamentals.Entities; public class FundamentalDataEntity { + [Key] + public string Isin { get; set; } = string.Empty; + + public TickerEntity Ticker { get; set; } = new(); + + // --- Valuation & Multiples --- + public decimal? MarketCap { get; set; } + public decimal? EnterpriseValue { get; set; } + public decimal? TrailingPe { get; set; } + public decimal? ForwardPe { get; set; } + public decimal? PegRatio { get; set; } + public decimal? PriceToSales { get; set; } + public decimal? PriceToBook { get; set; } + public decimal? EvToEbitda { get; set; } + + // --- Income Statement (TTM) --- + public decimal? TotalRevenue { get; set; } + public decimal? RevenueGrowthYoY { get; set; } + public decimal? GrossProfit { get; set; } + public decimal? OperatingIncome { get; set; } // EBIT + public decimal? Ebitda { get; set; } + public decimal? NetIncome { get; set; } + public decimal? DilutedEps { get; set; } + + // --- Balance Sheet & Cash Flow (MRQ / TTM) --- + public decimal? TotalCash { get; set; } + public decimal? TotalDebt { get; set; } + public decimal? DebtToEquity { get; set; } + public decimal? CurrentRatio { get; set; } + public decimal? OperatingCashFlow { get; set; } + public decimal? FreeCashFlow { get; set; } + + // --- Dividenden & Profitabilität --- + public decimal? ReturnOnEquity { get; set; } + public decimal? ReturnOnAssets { get; set; } + public decimal? ForwardDividendYield { get; set; } + public decimal? PayoutRatio { get; set; } + + // --- 52-Wochen-Spannbreite --- + public decimal? FiftyTwoWeekHigh { get; set; } + public decimal? FiftyTwoWeekLow { get; set; } + + // --- Analysten-Ratings & Kursziele --- + public string? ConsensusRating { get; set; } + public decimal? PriceTargetLow { get; set; } + public decimal? PriceTargetMean { get; set; } + public decimal? PriceTargetHigh { get; set; } + + // --- Aktionärsstruktur & Short-Interesse --- + public decimal? PercentHeldByInstitutions { get; set; } + public decimal? PercentHeldByInsiders { get; set; } + public decimal? ShortPercentOfFloat { get; set; } + public decimal? ShortRatio { get; set; } + + public DateTime LastUpdatedUtc { get; set; } = DateTime.UtcNow; + [Required] + public string AssetDataIsin { get; set; } = string.Empty; + + [ForeignKey(nameof(AssetDataIsin))] + public AssetDataEntity AssetData { get; set; } = null!; } \ No newline at end of file diff --git a/FinlyticFundamentals/Entities/KeyExecutiveEntity.cs b/FinlyticFundamentals/Entities/KeyExecutiveEntity.cs index eef9d48..26a1ed9 100644 --- a/FinlyticFundamentals/Entities/KeyExecutiveEntity.cs +++ b/FinlyticFundamentals/Entities/KeyExecutiveEntity.cs @@ -1,6 +1,17 @@ -namespace FinlyticFundamentals.Entities; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace FinlyticFundamentals.Entities; public class KeyExecutiveEntity { - + [Key] public Guid Id { get; set; } = Guid.NewGuid(); + + public string Name { get; set; } = string.Empty; + public string Title { get; set; } = string.Empty; + public string Payment { get; set; } = string.Empty; + + [Required] public string AssetDataIsin { get; set; } = string.Empty; + + [ForeignKey(nameof(AssetDataIsin))] public AssetDataEntity AssetData { get; set; } = null!; } \ No newline at end of file diff --git a/FinlyticFundamentals/Entities/TickerEntity.cs b/FinlyticFundamentals/Entities/TickerEntity.cs index 001d0da..b6be7b7 100644 --- a/FinlyticFundamentals/Entities/TickerEntity.cs +++ b/FinlyticFundamentals/Entities/TickerEntity.cs @@ -1,6 +1,10 @@ -namespace FinlyticFundamentals.Entities; +using Microsoft.EntityFrameworkCore; +namespace FinlyticFundamentals.Entities; + +[Owned] public class TickerEntity { - + public string Ticker { get; set; } = string.Empty; + public string Exchange { get; set; } = string.Empty; } \ No newline at end of file diff --git a/FinlyticFundamentals/FinlyticFundamentals.csproj b/FinlyticFundamentals/FinlyticFundamentals.csproj index 62a1143..08396be 100644 --- a/FinlyticFundamentals/FinlyticFundamentals.csproj +++ b/FinlyticFundamentals/FinlyticFundamentals.csproj @@ -29,4 +29,8 @@ + + + + diff --git a/FinlyticFundamentals/Migrations/20260814211010_Init.Designer.cs b/FinlyticFundamentals/Migrations/20260814211010_Init.Designer.cs new file mode 100644 index 0000000..299db49 --- /dev/null +++ b/FinlyticFundamentals/Migrations/20260814211010_Init.Designer.cs @@ -0,0 +1,394 @@ +// +using System; +using FinlyticFundamentals.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + [DbContext(typeof(FundamentalsDbContext))] + [Migration("20260814211010_Init")] + partial class Init + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("ServiceIdentifier") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ValueJson") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Key"); + + b.ToTable("DynamicSettings"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Isin"); + + b.ToTable("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Date") + .HasColumnType("timestamp with time zone"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("AssetEvents"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("CurrentRatio") + .HasColumnType("numeric"); + + b.Property("DebtToEquity") + .HasColumnType("numeric"); + + b.Property("DilutedEps") + .HasColumnType("numeric"); + + b.Property("Ebitda") + .HasColumnType("numeric"); + + b.Property("EnterpriseValue") + .HasColumnType("numeric"); + + b.Property("EvToEbitda") + .HasColumnType("numeric"); + + b.Property("ForwardDividendYield") + .HasColumnType("numeric"); + + b.Property("ForwardPe") + .HasColumnType("numeric"); + + b.Property("FreeCashFlow") + .HasColumnType("numeric"); + + b.Property("GrossProfit") + .HasColumnType("numeric"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("MarketCap") + .HasColumnType("numeric"); + + b.Property("NetIncome") + .HasColumnType("numeric"); + + b.Property("OperatingCashFlow") + .HasColumnType("numeric"); + + b.Property("OperatingIncome") + .HasColumnType("numeric"); + + b.Property("PayoutRatio") + .HasColumnType("numeric"); + + b.Property("PegRatio") + .HasColumnType("numeric"); + + b.Property("PriceToBook") + .HasColumnType("numeric"); + + b.Property("PriceToSales") + .HasColumnType("numeric"); + + b.Property("ReturnOnAssets") + .HasColumnType("numeric"); + + b.Property("ReturnOnEquity") + .HasColumnType("numeric"); + + b.Property("RevenueGrowthYoY") + .HasColumnType("numeric"); + + b.Property("TotalCash") + .HasColumnType("numeric"); + + b.Property("TotalDebt") + .HasColumnType("numeric"); + + b.Property("TotalRevenue") + .HasColumnType("numeric"); + + b.Property("TrailingPe") + .HasColumnType("numeric"); + + b.HasKey("Isin"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("FundamentalData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Payment") + .IsRequired() + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("KeyExecutives"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 => + { + b1.Property("AssetDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTicker"); + + b1.HasKey("AssetDataEntityIsin"); + + b1.ToTable("AssetData"); + + b1.WithOwner() + .HasForeignKey("AssetDataEntityIsin"); + }); + + b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 => + { + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b1.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Exchange"); + + b1.Property("Ticker") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Ticker"); + + b1.HasKey("Id"); + + b1.HasIndex("AssetDataIsin"); + + b1.HasIndex("Ticker"); + + b1.ToTable("Tickers", (string)null); + + b1.WithOwner() + .HasForeignKey("AssetDataIsin"); + }); + + b.Navigation("AvailableTickers"); + + b.Navigation("PrimaryTicker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("AssetEvents") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("AssetEventEntityId") + .HasColumnType("uuid"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("AssetEventEntityId"); + + b1.ToTable("AssetEvents"); + + b1.WithOwner() + .HasForeignKey("AssetEventEntityId"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("FundamentalData") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("FundamentalDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("FundamentalDataEntityIsin"); + + b1.ToTable("FundamentalData"); + + b1.WithOwner() + .HasForeignKey("FundamentalDataEntityIsin"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("KeyExecutives") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Navigation("AssetEvents"); + + b.Navigation("FundamentalData"); + + b.Navigation("KeyExecutives"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticFundamentals/Migrations/20260814211010_Init.cs b/FinlyticFundamentals/Migrations/20260814211010_Init.cs new file mode 100644 index 0000000..d3220ad --- /dev/null +++ b/FinlyticFundamentals/Migrations/20260814211010_Init.cs @@ -0,0 +1,206 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + /// + public partial class Init : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AssetData", + columns: table => new + { + Isin = table.Column(type: "text", nullable: false), + Name = table.Column(type: "text", nullable: false), + Description = table.Column(type: "text", nullable: false), + PrimaryTicker = table.Column(type: "text", nullable: false, defaultValue: ""), + PrimaryTickerExchange = table.Column(type: "text", nullable: false, defaultValue: "") + }, + constraints: table => + { + table.PrimaryKey("PK_AssetData", x => x.Isin); + }); + + migrationBuilder.CreateTable( + name: "DynamicSettings", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Key = table.Column(type: "character varying(150)", maxLength: 150, nullable: false), + ValueJson = table.Column(type: "text", nullable: false), + ServiceIdentifier = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + LastUpdatedUtc = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_DynamicSettings", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "AssetEvents", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Ticker = table.Column(type: "text", nullable: false, defaultValue: ""), + TickerExchange = table.Column(type: "text", nullable: false, defaultValue: ""), + Type = table.Column(type: "text", nullable: false), + Date = table.Column(type: "timestamp with time zone", nullable: false), + AssetDataIsin = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AssetEvents", x => x.Id); + table.ForeignKey( + name: "FK_AssetEvents_AssetData_AssetDataIsin", + column: x => x.AssetDataIsin, + principalTable: "AssetData", + principalColumn: "Isin", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "FundamentalData", + columns: table => new + { + Isin = table.Column(type: "text", nullable: false), + Ticker = table.Column(type: "text", nullable: false, defaultValue: ""), + TickerExchange = table.Column(type: "text", nullable: false, defaultValue: ""), + MarketCap = table.Column(type: "numeric", nullable: true), + EnterpriseValue = table.Column(type: "numeric", nullable: true), + TrailingPe = table.Column(type: "numeric", nullable: true), + ForwardPe = table.Column(type: "numeric", nullable: true), + PegRatio = table.Column(type: "numeric", nullable: true), + PriceToSales = table.Column(type: "numeric", nullable: true), + PriceToBook = table.Column(type: "numeric", nullable: true), + EvToEbitda = table.Column(type: "numeric", nullable: true), + TotalRevenue = table.Column(type: "numeric", nullable: true), + RevenueGrowthYoY = table.Column(type: "numeric", nullable: true), + GrossProfit = table.Column(type: "numeric", nullable: true), + OperatingIncome = table.Column(type: "numeric", nullable: true), + Ebitda = table.Column(type: "numeric", nullable: true), + NetIncome = table.Column(type: "numeric", nullable: true), + DilutedEps = table.Column(type: "numeric", nullable: true), + TotalCash = table.Column(type: "numeric", nullable: true), + TotalDebt = table.Column(type: "numeric", nullable: true), + DebtToEquity = table.Column(type: "numeric", nullable: true), + CurrentRatio = table.Column(type: "numeric", nullable: true), + OperatingCashFlow = table.Column(type: "numeric", nullable: true), + FreeCashFlow = table.Column(type: "numeric", nullable: true), + ReturnOnEquity = table.Column(type: "numeric", nullable: true), + ReturnOnAssets = table.Column(type: "numeric", nullable: true), + ForwardDividendYield = table.Column(type: "numeric", nullable: true), + PayoutRatio = table.Column(type: "numeric", nullable: true), + LastUpdatedUtc = table.Column(type: "timestamp with time zone", nullable: false), + AssetDataIsin = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_FundamentalData", x => x.Isin); + table.ForeignKey( + name: "FK_FundamentalData_AssetData_AssetDataIsin", + column: x => x.AssetDataIsin, + principalTable: "AssetData", + principalColumn: "Isin", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "KeyExecutives", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Name = table.Column(type: "text", nullable: false), + Title = table.Column(type: "text", nullable: false), + Payment = table.Column(type: "text", nullable: false), + AssetDataIsin = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_KeyExecutives", x => x.Id); + table.ForeignKey( + name: "FK_KeyExecutives_AssetData_AssetDataIsin", + column: x => x.AssetDataIsin, + principalTable: "AssetData", + principalColumn: "Isin", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "Tickers", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Ticker = table.Column(type: "text", nullable: false), + Exchange = table.Column(type: "text", nullable: false), + AssetDataIsin = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Tickers", x => x.Id); + table.ForeignKey( + name: "FK_Tickers_AssetData_AssetDataIsin", + column: x => x.AssetDataIsin, + principalTable: "AssetData", + principalColumn: "Isin", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_AssetEvents_AssetDataIsin", + table: "AssetEvents", + column: "AssetDataIsin"); + + migrationBuilder.CreateIndex( + name: "IX_DynamicSettings_Key", + table: "DynamicSettings", + column: "Key"); + + migrationBuilder.CreateIndex( + name: "IX_FundamentalData_AssetDataIsin", + table: "FundamentalData", + column: "AssetDataIsin"); + + migrationBuilder.CreateIndex( + name: "IX_KeyExecutives_AssetDataIsin", + table: "KeyExecutives", + column: "AssetDataIsin"); + + migrationBuilder.CreateIndex( + name: "IX_Tickers_AssetDataIsin", + table: "Tickers", + column: "AssetDataIsin"); + + migrationBuilder.CreateIndex( + name: "IX_Tickers_Ticker", + table: "Tickers", + column: "Ticker"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AssetEvents"); + + migrationBuilder.DropTable( + name: "DynamicSettings"); + + migrationBuilder.DropTable( + name: "FundamentalData"); + + migrationBuilder.DropTable( + name: "KeyExecutives"); + + migrationBuilder.DropTable( + name: "Tickers"); + + migrationBuilder.DropTable( + name: "AssetData"); + } + } +} diff --git a/FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.Designer.cs b/FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.Designer.cs new file mode 100644 index 0000000..01151f3 --- /dev/null +++ b/FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.Designer.cs @@ -0,0 +1,424 @@ +// +using System; +using FinlyticFundamentals.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + [DbContext(typeof(FundamentalsDbContext))] + [Migration("20260814214749_AddMoreFundamentalMetrics")] + partial class AddMoreFundamentalMetrics + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("ServiceIdentifier") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ValueJson") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Key"); + + b.ToTable("DynamicSettings"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Isin"); + + b.ToTable("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Date") + .HasColumnType("timestamp with time zone"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("AssetEvents"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConsensusRating") + .HasColumnType("text"); + + b.Property("CurrentRatio") + .HasColumnType("numeric"); + + b.Property("DebtToEquity") + .HasColumnType("numeric"); + + b.Property("DilutedEps") + .HasColumnType("numeric"); + + b.Property("Ebitda") + .HasColumnType("numeric"); + + b.Property("EnterpriseValue") + .HasColumnType("numeric"); + + b.Property("EvToEbitda") + .HasColumnType("numeric"); + + b.Property("FiftyTwoWeekHigh") + .HasColumnType("numeric"); + + b.Property("FiftyTwoWeekLow") + .HasColumnType("numeric"); + + b.Property("ForwardDividendYield") + .HasColumnType("numeric"); + + b.Property("ForwardPe") + .HasColumnType("numeric"); + + b.Property("FreeCashFlow") + .HasColumnType("numeric"); + + b.Property("GrossProfit") + .HasColumnType("numeric"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("MarketCap") + .HasColumnType("numeric"); + + b.Property("NetIncome") + .HasColumnType("numeric"); + + b.Property("OperatingCashFlow") + .HasColumnType("numeric"); + + b.Property("OperatingIncome") + .HasColumnType("numeric"); + + b.Property("PayoutRatio") + .HasColumnType("numeric"); + + b.Property("PegRatio") + .HasColumnType("numeric"); + + b.Property("PercentHeldByInsiders") + .HasColumnType("numeric"); + + b.Property("PercentHeldByInstitutions") + .HasColumnType("numeric"); + + b.Property("PriceTargetHigh") + .HasColumnType("numeric"); + + b.Property("PriceTargetLow") + .HasColumnType("numeric"); + + b.Property("PriceTargetMean") + .HasColumnType("numeric"); + + b.Property("PriceToBook") + .HasColumnType("numeric"); + + b.Property("PriceToSales") + .HasColumnType("numeric"); + + b.Property("ReturnOnAssets") + .HasColumnType("numeric"); + + b.Property("ReturnOnEquity") + .HasColumnType("numeric"); + + b.Property("RevenueGrowthYoY") + .HasColumnType("numeric"); + + b.Property("ShortPercentOfFloat") + .HasColumnType("numeric"); + + b.Property("ShortRatio") + .HasColumnType("numeric"); + + b.Property("TotalCash") + .HasColumnType("numeric"); + + b.Property("TotalDebt") + .HasColumnType("numeric"); + + b.Property("TotalRevenue") + .HasColumnType("numeric"); + + b.Property("TrailingPe") + .HasColumnType("numeric"); + + b.HasKey("Isin"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("FundamentalData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Payment") + .IsRequired() + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("KeyExecutives"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 => + { + b1.Property("AssetDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTicker"); + + b1.HasKey("AssetDataEntityIsin"); + + b1.ToTable("AssetData"); + + b1.WithOwner() + .HasForeignKey("AssetDataEntityIsin"); + }); + + b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 => + { + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b1.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Exchange"); + + b1.Property("Ticker") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Ticker"); + + b1.HasKey("Id"); + + b1.HasIndex("AssetDataIsin"); + + b1.HasIndex("Ticker"); + + b1.ToTable("Tickers", (string)null); + + b1.WithOwner() + .HasForeignKey("AssetDataIsin"); + }); + + b.Navigation("AvailableTickers"); + + b.Navigation("PrimaryTicker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("AssetEvents") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("AssetEventEntityId") + .HasColumnType("uuid"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("AssetEventEntityId"); + + b1.ToTable("AssetEvents"); + + b1.WithOwner() + .HasForeignKey("AssetEventEntityId"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("FundamentalData") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("FundamentalDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("FundamentalDataEntityIsin"); + + b1.ToTable("FundamentalData"); + + b1.WithOwner() + .HasForeignKey("FundamentalDataEntityIsin"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("KeyExecutives") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Navigation("AssetEvents"); + + b.Navigation("FundamentalData"); + + b.Navigation("KeyExecutives"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.cs b/FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.cs new file mode 100644 index 0000000..ebb4a83 --- /dev/null +++ b/FinlyticFundamentals/Migrations/20260814214749_AddMoreFundamentalMetrics.cs @@ -0,0 +1,118 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + /// + public partial class AddMoreFundamentalMetrics : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "ConsensusRating", + table: "FundamentalData", + type: "text", + nullable: true); + + migrationBuilder.AddColumn( + name: "FiftyTwoWeekHigh", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "FiftyTwoWeekLow", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "PercentHeldByInsiders", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "PercentHeldByInstitutions", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "PriceTargetHigh", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "PriceTargetLow", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "PriceTargetMean", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "ShortPercentOfFloat", + table: "FundamentalData", + type: "numeric", + nullable: true); + + migrationBuilder.AddColumn( + name: "ShortRatio", + table: "FundamentalData", + type: "numeric", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "ConsensusRating", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "FiftyTwoWeekHigh", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "FiftyTwoWeekLow", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "PercentHeldByInsiders", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "PercentHeldByInstitutions", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "PriceTargetHigh", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "PriceTargetLow", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "PriceTargetMean", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "ShortPercentOfFloat", + table: "FundamentalData"); + + migrationBuilder.DropColumn( + name: "ShortRatio", + table: "FundamentalData"); + } + } +} diff --git a/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs b/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs new file mode 100644 index 0000000..de312d0 --- /dev/null +++ b/FinlyticFundamentals/Migrations/FundamentalsDbContextModelSnapshot.cs @@ -0,0 +1,421 @@ +// +using System; +using FinlyticFundamentals.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FinlyticFundamentals.Migrations +{ + [DbContext(typeof(FundamentalsDbContext))] + partial class FundamentalsDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("ServiceIdentifier") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ValueJson") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Key"); + + b.ToTable("DynamicSettings"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Isin"); + + b.ToTable("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Date") + .HasColumnType("timestamp with time zone"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("AssetEvents"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.Property("Isin") + .HasColumnType("text"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConsensusRating") + .HasColumnType("text"); + + b.Property("CurrentRatio") + .HasColumnType("numeric"); + + b.Property("DebtToEquity") + .HasColumnType("numeric"); + + b.Property("DilutedEps") + .HasColumnType("numeric"); + + b.Property("Ebitda") + .HasColumnType("numeric"); + + b.Property("EnterpriseValue") + .HasColumnType("numeric"); + + b.Property("EvToEbitda") + .HasColumnType("numeric"); + + b.Property("FiftyTwoWeekHigh") + .HasColumnType("numeric"); + + b.Property("FiftyTwoWeekLow") + .HasColumnType("numeric"); + + b.Property("ForwardDividendYield") + .HasColumnType("numeric"); + + b.Property("ForwardPe") + .HasColumnType("numeric"); + + b.Property("FreeCashFlow") + .HasColumnType("numeric"); + + b.Property("GrossProfit") + .HasColumnType("numeric"); + + b.Property("LastUpdatedUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("MarketCap") + .HasColumnType("numeric"); + + b.Property("NetIncome") + .HasColumnType("numeric"); + + b.Property("OperatingCashFlow") + .HasColumnType("numeric"); + + b.Property("OperatingIncome") + .HasColumnType("numeric"); + + b.Property("PayoutRatio") + .HasColumnType("numeric"); + + b.Property("PegRatio") + .HasColumnType("numeric"); + + b.Property("PercentHeldByInsiders") + .HasColumnType("numeric"); + + b.Property("PercentHeldByInstitutions") + .HasColumnType("numeric"); + + b.Property("PriceTargetHigh") + .HasColumnType("numeric"); + + b.Property("PriceTargetLow") + .HasColumnType("numeric"); + + b.Property("PriceTargetMean") + .HasColumnType("numeric"); + + b.Property("PriceToBook") + .HasColumnType("numeric"); + + b.Property("PriceToSales") + .HasColumnType("numeric"); + + b.Property("ReturnOnAssets") + .HasColumnType("numeric"); + + b.Property("ReturnOnEquity") + .HasColumnType("numeric"); + + b.Property("RevenueGrowthYoY") + .HasColumnType("numeric"); + + b.Property("ShortPercentOfFloat") + .HasColumnType("numeric"); + + b.Property("ShortRatio") + .HasColumnType("numeric"); + + b.Property("TotalCash") + .HasColumnType("numeric"); + + b.Property("TotalDebt") + .HasColumnType("numeric"); + + b.Property("TotalRevenue") + .HasColumnType("numeric"); + + b.Property("TrailingPe") + .HasColumnType("numeric"); + + b.HasKey("Isin"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("FundamentalData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Payment") + .IsRequired() + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AssetDataIsin"); + + b.ToTable("KeyExecutives"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 => + { + b1.Property("AssetDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("PrimaryTicker"); + + b1.HasKey("AssetDataEntityIsin"); + + b1.ToTable("AssetData"); + + b1.WithOwner() + .HasForeignKey("AssetDataEntityIsin"); + }); + + b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 => + { + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b1.Property("AssetDataIsin") + .IsRequired() + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Exchange"); + + b1.Property("Ticker") + .IsRequired() + .HasColumnType("text") + .HasColumnName("Ticker"); + + b1.HasKey("Id"); + + b1.HasIndex("AssetDataIsin"); + + b1.HasIndex("Ticker"); + + b1.ToTable("Tickers", (string)null); + + b1.WithOwner() + .HasForeignKey("AssetDataIsin"); + }); + + b.Navigation("AvailableTickers"); + + b.Navigation("PrimaryTicker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("AssetEvents") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("AssetEventEntityId") + .HasColumnType("uuid"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("AssetEventEntityId"); + + b1.ToTable("AssetEvents"); + + b1.WithOwner() + .HasForeignKey("AssetEventEntityId"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("FundamentalData") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 => + { + b1.Property("FundamentalDataEntityIsin") + .HasColumnType("text"); + + b1.Property("Exchange") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("TickerExchange"); + + b1.Property("Ticker") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("") + .HasColumnName("Ticker"); + + b1.HasKey("FundamentalDataEntityIsin"); + + b1.ToTable("FundamentalData"); + + b1.WithOwner() + .HasForeignKey("FundamentalDataEntityIsin"); + }); + + b.Navigation("AssetData"); + + b.Navigation("Ticker") + .IsRequired(); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b => + { + b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData") + .WithMany("KeyExecutives") + .HasForeignKey("AssetDataIsin") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AssetData"); + }); + + modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b => + { + b.Navigation("AssetEvents"); + + b.Navigation("FundamentalData"); + + b.Navigation("KeyExecutives"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/FinlyticFundamentals/Program.cs b/FinlyticFundamentals/Program.cs index 6e916cb..4d4ecb0 100644 --- a/FinlyticFundamentals/Program.cs +++ b/FinlyticFundamentals/Program.cs @@ -1,18 +1,20 @@ -using System; +using FinlyticCore.Clients; +using FinlyticCore.Services; +using FinlyticCore.Services.PlaywrightScrapper; +using FinlyticCore.Services.TradeRepublic; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using FinlyticFundamentals.Database; using FinlyticFundamentals.Services; using FinlyticFundamentals.Util; +using Microsoft.EntityFrameworkCore.Diagnostics; + var builder = Host.CreateApplicationBuilder(args); // Register DB Context builder.Services.AddDbContext(options => - options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection"))); + options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")) + .ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))); // Register HTTP Clients builder.Services.AddHttpClient() @@ -23,12 +25,19 @@ builder.Services.AddHttpClient() AllowAutoRedirect = true }); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddTransient>(); // Register Application Services +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddScoped(); +builder.Services.AddTransient(); +builder.Services.AddScoped(); + +builder.Services.AddScoped(typeof(ISettingsService<>), typeof(SettingsService<>)); +builder.Services.AddScoped(typeof(IFinlyticLogger<,>), typeof(FinlyticLogger<,>)); // Register MQTT Client (as a Hosted Service) builder.Services.AddHostedService(); @@ -44,8 +53,6 @@ using (var scope = host.Services.CreateScope()) await context.Database.MigrateAsync(); Console.WriteLine("Database migrations successfully executed for FinlyticFundamentals."); - var settingsService = scope.ServiceProvider.GetRequiredService(); - await settingsService.GetSettingsAsync(); } catch (Exception ex) { diff --git a/FinlyticFundamentals/Services/FundamentalsDbService.cs b/FinlyticFundamentals/Services/FundamentalsDbService.cs index eab1416..bdcbadd 100644 --- a/FinlyticFundamentals/Services/FundamentalsDbService.cs +++ b/FinlyticFundamentals/Services/FundamentalsDbService.cs @@ -5,67 +5,52 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.Fundamentals; -using FinlyticCore.Services.Yahoo; +using FinlyticCore.Dtos.TradeRepublic; +using FinlyticCore.Dtos.Yahoo; +using FinlyticCore.Models.Settings; +using FinlyticCore.Services; +using FinlyticCore.Services.TradeRepublic; using FinlyticFundamentals.Database; using FinlyticFundamentals.Entities; +using FinlyticFundamentals.Util; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; namespace FinlyticFundamentals.Services; public interface IFundamentalsDbService { - /// - /// Gets the fundamental data for a given ISIN. - /// If a specific ticker is provided, the resolution pipeline prioritizes/fetches only that ticker. - /// - /// The ISIN identifier of the asset. - /// Optional specific ticker symbol (e.g., "APC.DE"). If omitted, tickers are resolved automatically. - /// If true, forces a full static scrape for profile, financials, and executives. - /// Cancellation token. - /// The mapped or null if unavailable. Task GetFundamentalsAsync( string isin, string? ticker = null, bool forceRefresh = false, CancellationToken cancellationToken = default); - /// - /// Gets all upcoming and historic corporate events (e.g., earnings releases, ex-dividend dates). - /// - /// Cancellation token. - /// A list of corporate events sorted chronologically. Task> GetAllEventsAsync(CancellationToken cancellationToken = default); - /// - /// Gets corporate events for a specific month. - /// - Task> GetEventsByMonthAsync(int year, int month, CancellationToken cancellationToken = default); + Task> GetEventsByMonthAsync(int year, int month, + CancellationToken cancellationToken = default); } public class FundamentalsDbService : IFundamentalsDbService { private static readonly ConcurrentDictionary IsinLocks = new(); - + private readonly IServiceScopeFactory _scopeFactory; private readonly IYahooFinanceScraper _scraper; - private readonly IHtmlFallbackScraper _fallbackScraper; - private readonly YahooFinanceClient _yahooClient; - private readonly ILogger _logger; + private readonly ITradeRepublicService _tradeRepublicService; + private readonly IFinlyticLogger _finlyticLogger; public FundamentalsDbService( IServiceScopeFactory scopeFactory, IYahooFinanceScraper scraper, - IHtmlFallbackScraper fallbackScraper, - YahooFinanceClient yahooClient, - ILogger logger) + ITradeRepublicService tradeRepublicService, + IFinlyticLogger finlyticLogger) { _scopeFactory = scopeFactory; _scraper = scraper; - _fallbackScraper = fallbackScraper; - _yahooClient = yahooClient; - _logger = logger; + _tradeRepublicService = tradeRepublicService; + _finlyticLogger = finlyticLogger; } /// @@ -79,39 +64,398 @@ public class FundamentalsDbService : IFundamentalsDbService var cleanIsin = isin.Trim().ToUpperInvariant(); var requestedTicker = ticker?.Trim().ToUpperInvariant(); - using var scope = _scopeFactory.CreateScope(); - var context = scope.ServiceProvider.GetRequiredService(); - var isinLock = IsinLocks.GetOrAdd(cleanIsin, _ => new SemaphoreSlim(1, 1)); await isinLock.WaitAsync(cancellationToken); try { - // 1. Aus DB laden - var entity = await LoadEntityGraphAsync(context, cleanIsin, cancellationToken); + using var scope = _scopeFactory.CreateScope(); + var context = scope.ServiceProvider.GetRequiredService(); + var settingsService = scope.ServiceProvider.GetRequiredService>(); - // Statische Daten älter als 30 Tage oder forced? - bool needsStaticScrape = entity == null || forceRefresh || (DateTime.UtcNow - entity.LastStaticUpdatedAt).TotalDays > 30; + // 1. Dynamic Settings lesen + bool allowForceRefresh = + await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken); + bool enableHtmlFallback = + await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken); + int validityDays = + await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken); - if (needsStaticScrape) + bool effectiveForceRefresh = forceRefresh && allowForceRefresh; + + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtmlFallback: {Html}", + cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback); + + // 2. Entitäten aus DB laden + var assetData = await context.AssetData + .Include(a => a.AvailableTickers) + .Include(a => a.KeyExecutives) + .Include(a => a.AssetEvents) + .FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken); + + var fundamentalData = await context.FundamentalData + .FirstOrDefaultAsync(f => f.Isin == cleanIsin, cancellationToken); + + // 3. Prüfen, was aktualisiert werden muss + bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name); + bool executivesMissing = assetData == null || assetData.KeyExecutives == null || + assetData.KeyExecutives.Count == 0; + bool fundamentalsExpired = fundamentalData == null || + (DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays; + + // Wenn ein expliziter Ticker übergeben wurde und sich vom gespeicherten unterscheidet, + // müssen Asset-Daten und Fundamentals mit dem neuen Ticker neu abgerufen werden. + bool tickerChanged = !string.IsNullOrWhiteSpace(requestedTicker) + && assetData?.PrimaryTicker != null + && !string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker, + StringComparison.OrdinalIgnoreCase); + + bool shouldUpdateAssetData = assetDataMissing || effectiveForceRefresh || tickerChanged; + bool shouldUpdateExecutives = executivesMissing || effectiveForceRefresh; + bool shouldUpdateFundamentals = fundamentalsExpired || effectiveForceRefresh || tickerChanged; + + if (shouldUpdateAssetData || shouldUpdateExecutives || shouldUpdateFundamentals) { - entity = await ExecuteFullScrapeAndPersistAsync(context, cleanIsin, requestedTicker, entity, cancellationToken); - } - else - { - // Statik ist frisch -> Prüfen ob requested Ticker existiert oder neu nachgeladen werden muss - entity = await EnsureTickerDataUpToDateAsync(context, cleanIsin, requestedTicker, entity!, cancellationToken); + // --- STEP 1: Trade Republic Details --- + TradeRepublicStockDetailsResponse? trDetails = null; + try + { + trDetails = await _tradeRepublicService.GetStockDetailsAsync(cleanIsin, cancellationToken); + } + catch (Exception ex) + { + await _finlyticLogger.LogWarningAsync(SettingKeys.FundamentalsChannel, ex, + "[DEBUG-TR-ERROR] Could not fetch Trade Republic details for {Isin}", cleanIsin); + } + + // --- STEP 2: Ticker auflösen (Null-safe) --- + TickerInfoDto primaryTicker; + + if (!string.IsNullOrWhiteSpace(requestedTicker)) + { + var match = assetData?.AvailableTickers? + .FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)); + + if (match != null) + { + primaryTicker = new TickerInfoDto + { + Ticker = match.Ticker, + Exchange = !string.IsNullOrWhiteSpace(match.Exchange) + ? match.Exchange + : GetExchangeDisplayName(match.Ticker) + }; + } + else if (assetData?.PrimaryTicker != null && + string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase)) + { + primaryTicker = new TickerInfoDto + { + Ticker = assetData.PrimaryTicker.Ticker, + Exchange = !string.IsNullOrWhiteSpace(assetData.PrimaryTicker.Exchange) + ? assetData.PrimaryTicker.Exchange + : GetExchangeDisplayName(assetData.PrimaryTicker.Ticker) + }; + } + else + { + primaryTicker = new TickerInfoDto + { + Ticker = requestedTicker, + Exchange = GetExchangeDisplayName(requestedTicker) + }; + } + } + else if (assetData?.PrimaryTicker != null && !string.IsNullOrWhiteSpace(assetData.PrimaryTicker.Ticker)) + { + primaryTicker = new TickerInfoDto + { + Ticker = assetData.PrimaryTicker.Ticker, + Exchange = !string.IsNullOrWhiteSpace(assetData.PrimaryTicker.Exchange) + ? assetData.PrimaryTicker.Exchange + : GetExchangeDisplayName(assetData.PrimaryTicker.Ticker) + }; + } + else + { + var resolved = await _scraper.ResolveTickerFromIsinAsync(cleanIsin, cancellationToken); + primaryTicker = resolved != null && !string.IsNullOrWhiteSpace(resolved.Ticker) + ? resolved + : new TickerInfoDto + { + Ticker = cleanIsin, + Exchange = "Unknown" + }; + } + + if (string.IsNullOrWhiteSpace(primaryTicker.Exchange)) + { + primaryTicker = new TickerInfoDto + { + Ticker = primaryTicker.Ticker, + Exchange = GetExchangeDisplayName(primaryTicker.Ticker) + }; + } + + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-TICKER-RESOLVED] Ticker aufgelöst zu: '{Ticker}' (Exchange: '{Exchange}') für ISIN {Isin}", + primaryTicker.Ticker, primaryTicker.Exchange ?? "Unknown", cleanIsin); + + // --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper --- + YahooQuoteSummaryModulesDto? modulesDto = null; + if (!string.IsNullOrWhiteSpace(primaryTicker.Ticker) && primaryTicker.Ticker != cleanIsin) + { + modulesDto = await _scraper.GetQuoteSummaryModulesAsync( + primaryTicker.Ticker, + forceHtmlScrape: false, + cancellationToken: cancellationToken); + } + else + { + await _finlyticLogger.LogWarningAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", primaryTicker.Ticker); + } + + // --- Update AssetDataEntity --- + if (shouldUpdateAssetData) + { + if (assetData == null) + { + assetData = new AssetDataEntity + { + Isin = cleanIsin, + PrimaryTicker = new TickerEntity + { + Ticker = primaryTicker.Ticker, + Exchange = primaryTicker.Exchange ?? "Unknown" + }, + KeyExecutives = new List(), + AssetEvents = new List() + }; + context.AssetData.Add(assetData); + } + + string trName = trDetails?.Company?.Name ?? string.Empty; + string trDescription = trDetails?.Company?.Description ?? string.Empty; + + string fallbackName = modulesDto?.QuoteType?.ShortName + ?? modulesDto?.QuoteType?.LongName + ?? primaryTicker.Ticker; + + assetData.Name = !string.IsNullOrWhiteSpace(trName) ? trName : fallbackName; + assetData.Description = !string.IsNullOrWhiteSpace(trDescription) + ? trDescription + : (modulesDto?.AssetProfile?.LongBusinessSummary ?? string.Empty); + + assetData.PrimaryTicker = new TickerEntity + { + Ticker = primaryTicker.Ticker, + Exchange = primaryTicker.Exchange ?? "Unknown" + }; + + var tickers = await _scraper.ResolveAllTickersFromIsinAsync(cleanIsin, cancellationToken); + if (!tickers.Any(t => string.Equals(t.Ticker, primaryTicker.Ticker, StringComparison.OrdinalIgnoreCase))) + { + tickers.Insert(0, primaryTicker); + } + + assetData.AvailableTickers.Clear(); + foreach (var a in tickers) + { + assetData.AvailableTickers.Add(new TickerEntity + { + Ticker = a.Ticker, + Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker) + }); + } + + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}'", + assetData.Name, assetData.PrimaryTicker.Ticker); + } + + // --- Process Trade Republic Corporate Events --- + if (trDetails != null && (shouldUpdateAssetData || effectiveForceRefresh) && assetData != null) + { + assetData.AssetEvents ??= new List(); + + var trEventList = new List(); + if (trDetails.Events != null) trEventList.AddRange(trDetails.Events); + if (trDetails.PastEvents != null) trEventList.AddRange(trDetails.PastEvents); + + foreach (var trEvt in trEventList) + { + if (!trEvt.Timestamp.HasValue) continue; + var evtDate = DateTimeOffset.FromUnixTimeMilliseconds(trEvt.Timestamp.Value).UtcDateTime; + var evtType = trEvt.Type ?? trEvt.Title ?? "EVENT"; + + bool isDuplicate = assetData.AssetEvents.Any(e => + e.Date.Date == evtDate.Date && + (string.Equals(e.Type, evtType, StringComparison.OrdinalIgnoreCase) || + (trEvt.Title != null && + string.Equals(e.Type, trEvt.Title, StringComparison.OrdinalIgnoreCase)))); + + if (!isDuplicate) + { + assetData.AssetEvents.Add(new AssetEventEntity + { + AssetDataIsin = cleanIsin, + Ticker = new TickerEntity + { + Ticker = primaryTicker.Ticker, + Exchange = primaryTicker.Exchange ?? "Unknown" + }, + Type = evtType, + Date = evtDate + }); + } + } + } + + // --- Process Modules DTO (Executives & Fundamental Data) --- + if (modulesDto != null) + { + // Update KeyExecutives + if (shouldUpdateExecutives && assetData != null) + { + // 1. Alte Executives direkt in der DB löschen (bypasses Change Tracker) + await context.KeyExecutives + .Where(e => e.AssetDataIsin == cleanIsin) + .ExecuteDeleteAsync(cancellationToken); + + // 2. ALLE tracked KeyExecutiveEntity-Einträge aus dem Change Tracker entfernen + // (nicht nur die in der Navigation-Collection — der Tracker kann mehr halten) + foreach (var entry in context.ChangeTracker.Entries() + .Where(e => e.Entity.AssetDataIsin == cleanIsin) + .ToList()) + { + entry.State = EntityState.Detached; + } + + // 3. Navigation-Collection zurücksetzen + assetData.KeyExecutives = new List(); + + // 4. Neue Executives aufbauen und direkt über den DbSet hinzufügen + if (modulesDto.AssetProfile?.CompanyOfficers != null) + { + foreach (var officer in modulesDto.AssetProfile.CompanyOfficers) + { + if (!string.IsNullOrWhiteSpace(officer.Name)) + { + var newExec = new KeyExecutiveEntity + { + AssetDataIsin = cleanIsin, + Name = officer.Name, + Title = officer.Title ?? string.Empty, + Payment = officer.TotalPay?.Fmt ?? + (officer.TotalPay?.Raw?.ToString() ?? string.Empty) + }; + context.KeyExecutives.Add(newExec); + assetData.KeyExecutives.Add(newExec); + } + } + } + + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.", + assetData.KeyExecutives.Count); + } + + // Update FundamentalDataEntity + if (shouldUpdateFundamentals) + { + if (fundamentalData == null) + { + fundamentalData = new FundamentalDataEntity + { + Isin = cleanIsin, + AssetDataIsin = cleanIsin + }; + context.FundamentalData.Add(fundamentalData); + } + + fundamentalData.Ticker = new TickerEntity + { + Ticker = primaryTicker.Ticker, + Exchange = primaryTicker.Exchange ?? "Unknown" + }; + fundamentalData.MarketCap = (decimal?)modulesDto.SummaryDetail?.MarketCap?.Raw; + fundamentalData.EnterpriseValue = + (decimal?)modulesDto.DefaultKeyStatistics?.EnterpriseValue?.Raw; + fundamentalData.TrailingPe = (decimal?)modulesDto.SummaryDetail?.TrailingPE?.Raw; + fundamentalData.ForwardPe = (decimal?)modulesDto.DefaultKeyStatistics?.ForwardPE?.Raw ?? + (decimal?)modulesDto.SummaryDetail?.ForwardPE?.Raw; + fundamentalData.PegRatio = (decimal?)modulesDto.DefaultKeyStatistics?.PegRatio?.Raw; + fundamentalData.PriceToSales = + (decimal?)modulesDto.SummaryDetail?.PriceToSalesTrailing12Months?.Raw; + fundamentalData.PriceToBook = (decimal?)modulesDto.DefaultKeyStatistics?.PriceToBook?.Raw; + fundamentalData.EvToEbitda = (decimal?)modulesDto.DefaultKeyStatistics?.EnterpriseToEbitda?.Raw; + + fundamentalData.TotalRevenue = (decimal?)modulesDto.FinancialData?.TotalRevenue?.Raw; + fundamentalData.RevenueGrowthYoY = (decimal?)modulesDto.FinancialData?.RevenueGrowth?.Raw; + fundamentalData.GrossProfit = (decimal?)modulesDto.FinancialData?.GrossMargins?.Raw ?? (decimal?)modulesDto.FinancialData?.GrossProfits?.Raw; + fundamentalData.OperatingIncome = (decimal?)modulesDto.FinancialData?.OperatingMargins?.Raw; + fundamentalData.Ebitda = (decimal?)modulesDto.FinancialData?.Ebitda?.Raw; + fundamentalData.NetIncome = (decimal?)modulesDto.FinancialData?.ProfitMargins?.Raw; + fundamentalData.DilutedEps = (decimal?)modulesDto.DefaultKeyStatistics?.TrailingEps?.Raw; + + fundamentalData.TotalCash = (decimal?)modulesDto.FinancialData?.TotalCash?.Raw; + fundamentalData.TotalDebt = (decimal?)modulesDto.FinancialData?.TotalDebt?.Raw; + fundamentalData.DebtToEquity = (decimal?)modulesDto.FinancialData?.DebtToEquity?.Raw; + fundamentalData.CurrentRatio = (decimal?)modulesDto.FinancialData?.CurrentRatio?.Raw; + fundamentalData.OperatingCashFlow = (decimal?)modulesDto.FinancialData?.OperatingCashflow?.Raw; + fundamentalData.FreeCashFlow = (decimal?)modulesDto.FinancialData?.FreeCashflow?.Raw; + + fundamentalData.ReturnOnEquity = (decimal?)modulesDto.FinancialData?.ReturnOnEquity?.Raw; + fundamentalData.ReturnOnAssets = (decimal?)modulesDto.FinancialData?.ReturnOnAssets?.Raw; + fundamentalData.ForwardDividendYield = (decimal?)modulesDto.SummaryDetail?.DividendYield?.Raw; + fundamentalData.PayoutRatio = (decimal?)modulesDto.SummaryDetail?.PayoutRatio?.Raw; + + fundamentalData.FiftyTwoWeekHigh = (decimal?)modulesDto.SummaryDetail?.FiftyTwoWeekHigh?.Raw; + fundamentalData.FiftyTwoWeekLow = (decimal?)modulesDto.SummaryDetail?.FiftyTwoWeekLow?.Raw; + + fundamentalData.ConsensusRating = modulesDto.FinancialData?.RecommendationKey; + fundamentalData.PriceTargetLow = (decimal?)modulesDto.FinancialData?.TargetLowPrice?.Raw; + fundamentalData.PriceTargetMean = (decimal?)modulesDto.FinancialData?.TargetMeanPrice?.Raw; + fundamentalData.PriceTargetHigh = (decimal?)modulesDto.FinancialData?.TargetHighPrice?.Raw; + + fundamentalData.PercentHeldByInstitutions = (decimal?)modulesDto.DefaultKeyStatistics?.HeldPercentInstitutions?.Raw; + fundamentalData.PercentHeldByInsiders = (decimal?)modulesDto.DefaultKeyStatistics?.HeldPercentInsiders?.Raw; + fundamentalData.ShortPercentOfFloat = (decimal?)modulesDto.DefaultKeyStatistics?.ShortPercentOfFloat?.Raw; + fundamentalData.ShortRatio = (decimal?)modulesDto.DefaultKeyStatistics?.ShortRatio?.Raw; + + fundamentalData.LastUpdatedUtc = DateTime.UtcNow; + + await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-FUNDAMENTALS-SAVED] FundamentalData gesetzt -> MarketCap: {MC} | PE: {PE}", + fundamentalData.MarketCap ?? (object)"null", fundamentalData.TrailingPe ?? (object)"null"); + } + } + + try + { + await context.SaveChangesAsync(cancellationToken); + } + catch (Microsoft.EntityFrameworkCore.DbUpdateConcurrencyException ex) + { + foreach (var entry in ex.Entries) + { + await _finlyticLogger.LogErrorAsync(SettingKeys.FundamentalsChannel, + "[DEBUG-CONCURRENCY-FAIL] Failed to save entity: {EntityType}, State: {State}", + entry.Entity.GetType().Name, entry.State.ToString()); + } + + throw; + } } - return entity != null ? MapToDto(entity, requestedTicker) : null; - } - catch (Exception ex) - { - _logger.LogError(ex, "[{Channel}] Failed to process fundamentals for ISIN {Isin}", "FundamentalsChannel", cleanIsin); - - // Fallback auf Datenbankstand, falls vorhanden - var fallback = await LoadEntityGraphAsync(context, cleanIsin, cancellationToken); - return fallback != null ? MapToDto(fallback, requestedTicker) : null; + if (assetData == null) return null; + + var executivesList = assetData.KeyExecutives?.ToList() ?? new List(); + var eventsList = assetData.AssetEvents?.ToList() ?? new List(); + + return MapToDto(assetData, fundamentalData, executivesList, eventsList); } finally { @@ -119,612 +463,31 @@ public class FundamentalsDbService : IFundamentalsDbService } } - #region Internal Logic Pipelines - - /// - /// Stellt sicher, dass der angeforderte Ticker existiert und dessen Live-Preise frisch sind (TTL: 15 Minuten). - /// - private async Task EnsureTickerDataUpToDateAsync( - FundamentalsDbContext context, - string isin, - string? requestedTicker, - AssetFundamentalsEntity entity, - CancellationToken cancellationToken) - { - var targetTickerSymbol = requestedTicker - ?? (entity.TickerFundamentals.FirstOrDefault(t => t.Ticker == entity.PrimaryTicker)?.Ticker - ?? entity.TickerFundamentals.FirstOrDefault()?.Ticker); - - // Fall A: Ticker noch gar nicht in DB -> Einzel-Scrape für diesen Ticker durchführen - if (!string.IsNullOrEmpty(targetTickerSymbol) && !entity.TickerFundamentals.Any(t => t.Ticker.Equals(targetTickerSymbol, StringComparison.OrdinalIgnoreCase))) - { - _logger.LogInformation("[{Channel}] Targeted ticker '{Ticker}' missing in DB for ISIN {Isin}. Fetching on-demand...", "FundamentalsChannel", targetTickerSymbol, isin); - var scraped = await _scraper.ScrapeFundamentalsAsync(isin, targetTickerSymbol, cancellationToken); - if (scraped?.TickerData != null) - { - entity.TickerFundamentals.Add(scraped.TickerData); - await context.SaveChangesAsync(cancellationToken); - } - return entity; - } - - // Fall B: Ticker existiert -> Prüfen ob Live-Kurs älter als 15 Minuten ist - var targetTickerEntity = entity.TickerFundamentals.FirstOrDefault(t => t.Ticker.Equals(targetTickerSymbol, StringComparison.OrdinalIgnoreCase)); - if (targetTickerEntity != null && (DateTime.UtcNow - targetTickerEntity.LastUpdatedAt).TotalMinutes > 15) - { - _logger.LogInformation("[{Channel}] Quote expired for ticker '{Ticker}'. Refreshing live price...", "FundamentalsChannel", targetTickerSymbol); - var quotesResponse = await _yahooClient.GetQuotesAsync(new[] { targetTickerEntity.Ticker }, cancellationToken); - var liveQuote = quotesResponse?.QuoteResponse?.Result?.FirstOrDefault(); - - if (liveQuote != null) - { - targetTickerEntity.CurrentPrice = (decimal?)liveQuote.RegularMarketPrice ?? targetTickerEntity.CurrentPrice; - targetTickerEntity.DayChangeAbsolute = (decimal?)liveQuote.RegularMarketChange ?? targetTickerEntity.DayChangeAbsolute; - targetTickerEntity.DayChangePercent = (decimal?)liveQuote.RegularMarketChangePercent ?? targetTickerEntity.DayChangePercent; - targetTickerEntity.FiftyTwoWeekHigh = (decimal?)liveQuote.FiftyTwoWeekHigh ?? targetTickerEntity.FiftyTwoWeekHigh; - targetTickerEntity.FiftyTwoWeekLow = (decimal?)liveQuote.FiftyTwoWeekLow ?? targetTickerEntity.FiftyTwoWeekLow; - targetTickerEntity.MarketCapitalization = (decimal?)liveQuote.MarketCap ?? targetTickerEntity.MarketCapitalization; - targetTickerEntity.LastUpdatedAt = DateTime.UtcNow; - - entity.LastUpdatedAt = DateTime.UtcNow; - await context.SaveChangesAsync(cancellationToken); - } - } - - return entity; - } - - /// - /// Führt ein vollständiges Scraping der Bilanzen und Ticker durch und speichert das Ergebnis ab. - /// - private async Task ExecuteFullScrapeAndPersistAsync( - FundamentalsDbContext context, - string isin, - string? requestedTicker, - AssetFundamentalsEntity? existingEntity, - CancellationToken cancellationToken) - { - _logger.LogInformation("[{Channel}] Initiating full static scrape for ISIN {Isin}...", "FundamentalsChannel", isin); - - List tickers = new(); - - if (!string.IsNullOrWhiteSpace(requestedTicker)) - { - tickers.Add(requestedTicker); - } - else - { - tickers = await _scraper.ResolveAllTickersFromIsinAsync(isin, cancellationToken); - if (existingEntity?.TickerFundamentals != null) - { - foreach (var tf in existingEntity.TickerFundamentals) - { - if (!tickers.Contains(tf.Ticker, StringComparer.OrdinalIgnoreCase)) - tickers.Add(tf.Ticker); - } - } - } - - if (tickers.Count == 0) - { - _logger.LogWarning("[YahooFallbackScraper] No tickers resolved for ISIN {Isin}. Fallback scraper cannot be invoked without a ticker.", isin); - return existingEntity; - } - - var primaryTicker = tickers[0]; - _logger.LogInformation("[YahooFallbackScraper] Primary ticker resolved: '{Ticker}' for ISIN {Isin}", primaryTicker, isin); - var scraped = await _scraper.ScrapeFundamentalsAsync(isin, primaryTicker, cancellationToken); - - bool needsFallback = IsDataIncomplete(scraped, _logger); - _logger.LogInformation("[YahooFallbackScraper] Primary scrape completeness check for '{Ticker}': scrapedIsNull={ScrapedIsNull}, needsFallback={NeedsFallback}", - primaryTicker, scraped == null, needsFallback); - - if (needsFallback) - { - _logger.LogInformation("[YahooFallbackScraper] Executing Playwright Fallback Scraper for ticker '{Ticker}' (ISIN: {Isin})...", primaryTicker, isin); - var fallbackData = await _fallbackScraper.ScrapeFallbackAsync(isin, primaryTicker, cancellationToken); - if (fallbackData != null) - { - _logger.LogInformation("[YahooFallbackScraper] Fallback scraper returned data for {Ticker}. MarketCap={MarketCap}, EV={EV}, Sector='{Sector}'", - primaryTicker, fallbackData.TickerData?.MarketCapitalization, fallbackData.TickerData?.EnterpriseValue, fallbackData.Fundamentals?.Sector); - - if (scraped == null) - { - _logger.LogInformation("[YahooFallbackScraper] Primary scraped data was null. Using entirely Playwright fallback data for {Ticker}...", primaryTicker); - scraped = fallbackData; - } - else - { - _logger.LogInformation("[YahooFallbackScraper] Merging Playwright fallback data into primary scraped data for {Ticker}...", primaryTicker); - - // Merge fallback into scraped - MergeFundamentals(scraped, fallbackData); - } - } - else - { - _logger.LogWarning("[YahooFallbackScraper] Fallback scraper returned NULL for {Ticker}!", primaryTicker); - } - } - // ------------------------------ - - if (scraped == null) return existingEntity; - - var tickerEntities = new List { scraped.TickerData }; - - // Sekundär-Ticker parallel laden (nur wenn kein spezifischer Ticker verlangt war) - if (string.IsNullOrWhiteSpace(requestedTicker) && tickers.Count > 1) - { - var altTasks = tickers.Skip(1).Take(4).Select(async alt => - { - try { return await _scraper.ScrapeFundamentalsAsync(isin, alt, cancellationToken); } - catch { return null; } - }); - - var altResults = await Task.WhenAll(altTasks); - foreach (var alt in altResults) - { - if (alt?.TickerData != null) tickerEntities.Add(alt.TickerData); - } - } - - // DB Upsert - try - { - await SaveOrUpdateFundamentalsAsync(context, isin, primaryTicker, scraped, tickerEntities, cancellationToken); - } - catch (DbUpdateException ex) when (ex.InnerException is Npgsql.NpgsqlException npgEx && npgEx.SqlState == "23505") - { - context.ChangeTracker.Clear(); - await SaveOrUpdateFundamentalsAsync(context, isin, primaryTicker, scraped, tickerEntities, cancellationToken); - } - - return await LoadEntityGraphAsync(context, isin, cancellationToken); - } - - private static void MergeFundamentals(ScrapedFundamentalsData target, ScrapedFundamentalsData source) - { - var t = target.TickerData; - var s = source.TickerData; - - // Kennzahlen & Ratios - if (t.MarketCapitalization == 0 && s.MarketCapitalization > 0) t.MarketCapitalization = s.MarketCapitalization; - if ((t.EnterpriseValue == 0) && s.EnterpriseValue > 0) t.EnterpriseValue = s.EnterpriseValue; - - t.PeRatioTrailing ??= s.PeRatioTrailing; - t.PeRatioForward ??= s.PeRatioForward; - t.PegRatio ??= s.PegRatio; - t.PbRatio ??= s.PbRatio; - t.PsRatio ??= s.PsRatio; - t.EvToEbitda ??= s.EvToEbitda; - t.EvToRevenue ??= s.EvToRevenue; - - // Margen - t.GrossMargin ??= s.GrossMargin; - t.OperatingMargin ??= s.OperatingMargin; - t.NetProfitMargin ??= s.NetProfitMargin; - t.ReturnOnEquity ??= s.ReturnOnEquity; - t.ReturnOnAssets ??= s.ReturnOnAssets; - - // Preise & Dividenden - if (t.FiftyTwoWeekHigh == 0 && s.FiftyTwoWeekHigh > 0) t.FiftyTwoWeekHigh = s.FiftyTwoWeekHigh; - if (t.FiftyTwoWeekLow == 0 && s.FiftyTwoWeekLow > 0) t.FiftyTwoWeekLow = s.FiftyTwoWeekLow; - if ((!t.DividendYield.HasValue || t.DividendYield == 0) && s.DividendYield > 0) t.DividendYield = s.DividendYield; - - // Stammdaten - if (string.IsNullOrWhiteSpace(target.Fundamentals.Sector)) target.Fundamentals.Sector = source.Fundamentals.Sector; - if (string.IsNullOrWhiteSpace(target.Fundamentals.Industry)) target.Fundamentals.Industry = source.Fundamentals.Industry; - if (!target.Fundamentals.Employees.HasValue) target.Fundamentals.Employees = source.Fundamentals.Employees; - if (string.IsNullOrWhiteSpace(target.Fundamentals.BusinessSummary)) target.Fundamentals.BusinessSummary = source.Fundamentals.BusinessSummary; - } - - private static bool IsDataIncomplete(ScrapedFundamentalsData? data, ILogger logger) - { - if (data == null || data.TickerData == null) - { - logger.LogWarning("[YahooFallbackScraper] IsDataIncomplete -> TRUE (scraped data or TickerData is NULL)"); - return true; - } - - var td = data.TickerData; - var f = data.Fundamentals; - - int missingCriticalFields = 0; - - // 1. Absolute Must-Haves (sofortiger Fallback wenn 0) - if (td.MarketCapitalization == 0) - { - logger.LogWarning("[YahooFallbackScraper] IsDataIncomplete -> TRUE (MarketCapitalization is 0)"); - return true; - } - if (td.FiftyTwoWeekHigh == 0 || td.FiftyTwoWeekLow == 0) - { - logger.LogWarning("[YahooFallbackScraper] IsDataIncomplete -> TRUE (52WeekHigh={High} or 52WeekLow={Low} is 0)", td.FiftyTwoWeekHigh, td.FiftyTwoWeekLow); - return true; - } - - // 2. Bewertung & Ratios (Zähle fehlende Metriken) - // KGV: Trailing ODER Forward muss vorhanden sein, sonst zählt die KGV-Bewertung als fehlend - if ((!td.PeRatioTrailing.HasValue || td.PeRatioTrailing == 0) && (!td.PeRatioForward.HasValue || td.PeRatioForward == 0)) - missingCriticalFields++; - - if (!td.PbRatio.HasValue || td.PbRatio == 0) missingCriticalFields++; - if (!td.PsRatio.HasValue || td.PsRatio == 0) missingCriticalFields++; - if (td.EnterpriseValue == 0) missingCriticalFields++; - - // 3. Margen & Profitabilität - if (!td.GrossMargin.HasValue) missingCriticalFields++; - if (!td.OperatingMargin.HasValue) missingCriticalFields++; - if (!td.NetProfitMargin.HasValue) missingCriticalFields++; - - // 4. Stammdaten - if (string.IsNullOrWhiteSpace(f.Sector)) missingCriticalFields++; - if (string.IsNullOrWhiteSpace(f.Industry)) missingCriticalFields++; - - // Wenn 2 oder mehr der wichtigen Kennzahlen fehlen, gilt die Quelle als unvollständig - bool isIncomplete = missingCriticalFields >= 2; - logger.LogInformation("[YahooFallbackScraper] IsDataIncomplete total missingCriticalFields={Count} (threshold >= 2 -> isIncomplete={Result})", missingCriticalFields, isIncomplete); - - return isIncomplete; - } - - #endregion - - #region Data Access & Mapping Helpers - - private static Task LoadEntityGraphAsync(FundamentalsDbContext context, string isin, CancellationToken ct) - { - return context.AssetFundamentals - .AsNoTracking() - .Include(f => f.Executives) - .Include(f => f.FinancialStatements) - .Include(f => f.Estimates) - .Include(f => f.TickerFundamentals) - .FirstOrDefaultAsync(f => f.Isin == isin, ct); - } - - private async Task SaveOrUpdateFundamentalsAsync( - FundamentalsDbContext context, - string isin, - string primaryTicker, - ScrapedFundamentalsData scraped, - List tickerEntities, - CancellationToken cancellationToken) - { - var entity = await context.AssetFundamentals.FirstOrDefaultAsync(f => f.Isin == isin, cancellationToken); - - if (entity == null) - { - entity = scraped.Fundamentals; - entity.Isin = isin; - entity.PrimaryTicker = primaryTicker; - entity.Executives = scraped.Executives; - entity.FinancialStatements = scraped.Statements; - entity.Estimates = scraped.Estimates; - entity.TickerFundamentals = new List(); - - foreach (var ex in entity.Executives) { ex.Isin = isin; if (ex.Id == Guid.Empty) ex.Id = Guid.NewGuid(); } - foreach (var stmt in entity.FinancialStatements) { stmt.Isin = isin; if (stmt.Id == Guid.Empty) stmt.Id = Guid.NewGuid(); } - - context.AssetFundamentals.Add(entity); - } - else - { - entity.PrimaryTicker = primaryTicker; - entity.CompanyName = !string.IsNullOrWhiteSpace(scraped.Fundamentals.CompanyName) ? scraped.Fundamentals.CompanyName : entity.CompanyName; - entity.BusinessSummary = !string.IsNullOrWhiteSpace(scraped.Fundamentals.BusinessSummary) ? scraped.Fundamentals.BusinessSummary : entity.BusinessSummary; - entity.Sector = !string.IsNullOrWhiteSpace(scraped.Fundamentals.Sector) ? scraped.Fundamentals.Sector : entity.Sector; - entity.Industry = !string.IsNullOrWhiteSpace(scraped.Fundamentals.Industry) ? scraped.Fundamentals.Industry : entity.Industry; - entity.Country = !string.IsNullOrWhiteSpace(scraped.Fundamentals.Country) ? scraped.Fundamentals.Country : entity.Country; - entity.Employees = scraped.Fundamentals.Employees ?? entity.Employees; - - entity.PercentHeldByInstitutions = scraped.Fundamentals.PercentHeldByInstitutions ?? entity.PercentHeldByInstitutions; - entity.PercentHeldByInsiders = scraped.Fundamentals.PercentHeldByInsiders ?? entity.PercentHeldByInsiders; - entity.ShortRatio = scraped.Fundamentals.ShortRatio ?? entity.ShortRatio; - entity.ShortPercentOfFloat = scraped.Fundamentals.ShortPercentOfFloat ?? entity.ShortPercentOfFloat; - - if (!string.IsNullOrWhiteSpace(scraped.Fundamentals.ConsensusRating) && !scraped.Fundamentals.ConsensusRating.Equals("none", StringComparison.OrdinalIgnoreCase)) - entity.ConsensusRating = scraped.Fundamentals.ConsensusRating; - - entity.PriceTargetLow = scraped.Fundamentals.PriceTargetLow ?? entity.PriceTargetLow; - entity.PriceTargetHigh = scraped.Fundamentals.PriceTargetHigh ?? entity.PriceTargetHigh; - entity.PriceTargetMedian = scraped.Fundamentals.PriceTargetMedian ?? entity.PriceTargetMedian; - entity.PriceTargetMean = scraped.Fundamentals.PriceTargetMean ?? entity.PriceTargetMean; - - entity.ExDividendDate = scraped.Fundamentals.ExDividendDate ?? entity.ExDividendDate; - entity.NextEarningsDate = scraped.Fundamentals.NextEarningsDate ?? entity.NextEarningsDate; - entity.LastStaticUpdatedAt = DateTime.UtcNow; - entity.LastUpdatedAt = DateTime.UtcNow; - - // Executives & Statements aktualisieren - if (scraped.Executives.Count > 0) - { - await context.CompanyExecutives.Where(e => e.Isin == isin).ExecuteDeleteAsync(cancellationToken); - foreach (var exec in scraped.Executives) - { - exec.Isin = isin; - if (exec.Id == Guid.Empty) exec.Id = Guid.NewGuid(); - context.CompanyExecutives.Add(exec); - } - } - - if (scraped.Statements.Count > 0) - { - var existingStmts = await context.FinancialStatements.Where(s => s.Isin == isin).ToListAsync(cancellationToken); - foreach (var stmt in scraped.Statements) - { - var existingStmt = existingStmts.FirstOrDefault(s => s.PeriodType == stmt.PeriodType && s.EndDate.Date == stmt.EndDate.Date); - if (existingStmt == null) - { - stmt.Isin = isin; - if (stmt.Id == Guid.Empty) stmt.Id = Guid.NewGuid(); - context.FinancialStatements.Add(stmt); - } - else - { - existingStmt.TotalRevenue = stmt.TotalRevenue ?? existingStmt.TotalRevenue; - existingStmt.CostOfRevenue = stmt.CostOfRevenue ?? existingStmt.CostOfRevenue; - existingStmt.GrossProfit = stmt.GrossProfit ?? existingStmt.GrossProfit; - existingStmt.OperatingExpenses = stmt.OperatingExpenses ?? existingStmt.OperatingExpenses; - existingStmt.OperatingIncome = stmt.OperatingIncome ?? existingStmt.OperatingIncome; - existingStmt.Ebitda = stmt.Ebitda ?? existingStmt.Ebitda; - existingStmt.NetIncome = stmt.NetIncome ?? existingStmt.NetIncome; - existingStmt.CashAndCashEquivalents = stmt.CashAndCashEquivalents ?? existingStmt.CashAndCashEquivalents; - existingStmt.TotalCurrentAssets = stmt.TotalCurrentAssets ?? existingStmt.TotalCurrentAssets; - existingStmt.CurrentLiabilities = stmt.CurrentLiabilities ?? existingStmt.CurrentLiabilities; - existingStmt.LongTermDebt = stmt.LongTermDebt ?? existingStmt.LongTermDebt; - existingStmt.TotalLiabilities = stmt.TotalLiabilities ?? existingStmt.TotalLiabilities; - existingStmt.TotalStockholdersEquity = stmt.TotalStockholdersEquity ?? existingStmt.TotalStockholdersEquity; - existingStmt.OperatingCashFlow = stmt.OperatingCashFlow ?? existingStmt.OperatingCashFlow; - existingStmt.InvestingCashFlow = stmt.InvestingCashFlow ?? existingStmt.InvestingCashFlow; - existingStmt.CapitalExpenditures = stmt.CapitalExpenditures ?? existingStmt.CapitalExpenditures; - existingStmt.FinancingCashFlow = stmt.FinancingCashFlow ?? existingStmt.FinancingCashFlow; - existingStmt.FreeCashFlow = stmt.FreeCashFlow ?? existingStmt.FreeCashFlow; - } - } - } - } - - // Ticker-Fundamentaldaten aktualisieren - foreach (var t in tickerEntities) - { - t.Isin = isin; - var existingTicker = await context.TickerFundamentals.FirstOrDefaultAsync(tf => tf.Ticker == t.Ticker, cancellationToken); - - if (existingTicker == null) - { - context.TickerFundamentals.Add(t); - } - else - { - existingTicker.Exchange = !string.IsNullOrEmpty(t.Exchange) ? t.Exchange : existingTicker.Exchange; - existingTicker.TradingCurrency = !string.IsNullOrEmpty(t.TradingCurrency) ? t.TradingCurrency : existingTicker.TradingCurrency; - existingTicker.CurrentPrice = t.CurrentPrice > 0 ? t.CurrentPrice : existingTicker.CurrentPrice; - existingTicker.DayChangeAbsolute = t.DayChangeAbsolute != 0 ? t.DayChangeAbsolute : existingTicker.DayChangeAbsolute; - existingTicker.DayChangePercent = t.DayChangePercent != 0 ? t.DayChangePercent : existingTicker.DayChangePercent; - existingTicker.FiftyTwoWeekHigh = t.FiftyTwoWeekHigh > 0 ? t.FiftyTwoWeekHigh : existingTicker.FiftyTwoWeekHigh; - existingTicker.FiftyTwoWeekLow = t.FiftyTwoWeekLow > 0 ? t.FiftyTwoWeekLow : existingTicker.FiftyTwoWeekLow; - existingTicker.MarketCapitalization = t.MarketCapitalization > 0 ? t.MarketCapitalization : existingTicker.MarketCapitalization; - existingTicker.EnterpriseValue = t.EnterpriseValue > 0 ? t.EnterpriseValue : existingTicker.EnterpriseValue; - existingTicker.PeRatioTrailing = t.PeRatioTrailing ?? existingTicker.PeRatioTrailing; - existingTicker.PeRatioForward = t.PeRatioForward ?? existingTicker.PeRatioForward; - existingTicker.PegRatio = t.PegRatio ?? existingTicker.PegRatio; - existingTicker.PbRatio = t.PbRatio ?? existingTicker.PbRatio; - existingTicker.PsRatio = t.PsRatio ?? existingTicker.PsRatio; - existingTicker.EvToEbitda = t.EvToEbitda ?? existingTicker.EvToEbitda; - existingTicker.EvToRevenue = t.EvToRevenue ?? existingTicker.EvToRevenue; - existingTicker.GrossMargin = t.GrossMargin ?? existingTicker.GrossMargin; - existingTicker.OperatingMargin = t.OperatingMargin ?? existingTicker.OperatingMargin; - existingTicker.NetProfitMargin = t.NetProfitMargin ?? existingTicker.NetProfitMargin; - existingTicker.ReturnOnEquity = t.ReturnOnEquity ?? existingTicker.ReturnOnEquity; - existingTicker.ReturnOnAssets = t.ReturnOnAssets ?? existingTicker.ReturnOnAssets; - existingTicker.DebtToEquity = t.DebtToEquity ?? existingTicker.DebtToEquity; - existingTicker.CurrentRatio = t.CurrentRatio ?? existingTicker.CurrentRatio; - existingTicker.QuickRatio = t.QuickRatio ?? existingTicker.QuickRatio; - existingTicker.DividendYield = t.DividendYield ?? existingTicker.DividendYield; - existingTicker.PayoutRatio = t.PayoutRatio ?? existingTicker.PayoutRatio; - existingTicker.ExDividendDate = t.ExDividendDate ?? existingTicker.ExDividendDate; - existingTicker.LastUpdatedAt = DateTime.UtcNow; - } - } - - await context.SaveChangesAsync(cancellationToken); - } - - private static AssetFundamentalsDto MapToDto(AssetFundamentalsEntity entity, string? requestedTicker) - { - var targetTicker = entity.TickerFundamentals?.FirstOrDefault(t => t.Ticker.Equals(requestedTicker, StringComparison.OrdinalIgnoreCase)) - ?? entity.TickerFundamentals?.FirstOrDefault(t => t.Ticker.Equals(entity.PrimaryTicker, StringComparison.OrdinalIgnoreCase)) - ?? entity.TickerFundamentals?.FirstOrDefault(); - - var selectedTickerSymbol = targetTicker?.Ticker ?? requestedTicker ?? entity.PrimaryTicker; - - return new AssetFundamentalsDto - { - Isin = entity.Isin, - PrimaryTicker = entity.PrimaryTicker, - Ticker = selectedTickerSymbol, - CompanyName = entity.CompanyName, - Exchange = targetTicker?.Exchange, - TradingCurrency = targetTicker?.TradingCurrency, - BusinessSummary = entity.BusinessSummary, - Sector = entity.Sector, - Industry = entity.Industry, - Country = entity.Country, - Employees = entity.Employees, - - CurrentPrice = targetTicker?.CurrentPrice ?? 0, - DayChangeAbsolute = targetTicker?.DayChangeAbsolute ?? 0, - DayChangePercent = targetTicker?.DayChangePercent ?? 0, - FiftyTwoWeekHigh = targetTicker?.FiftyTwoWeekHigh ?? 0, - FiftyTwoWeekLow = targetTicker?.FiftyTwoWeekLow ?? 0, - MarketCapitalization = targetTicker?.MarketCapitalization ?? 0, - EnterpriseValue = targetTicker?.EnterpriseValue ?? 0, - PeRatioTrailing = targetTicker?.PeRatioTrailing, - PeRatioForward = targetTicker?.PeRatioForward, - PegRatio = targetTicker?.PegRatio, - PbRatio = targetTicker?.PbRatio, - PsRatio = targetTicker?.PsRatio, - EvToEbitda = targetTicker?.EvToEbitda, - EvToRevenue = targetTicker?.EvToRevenue, - - GrossMargin = targetTicker?.GrossMargin, - OperatingMargin = targetTicker?.OperatingMargin, - NetProfitMargin = targetTicker?.NetProfitMargin, - ReturnOnEquity = targetTicker?.ReturnOnEquity, - ReturnOnAssets = targetTicker?.ReturnOnAssets, - ReturnOnInvestedCapital = targetTicker?.ReturnOnInvestedCapital, - DebtToEquity = targetTicker?.DebtToEquity, - CurrentRatio = targetTicker?.CurrentRatio, - QuickRatio = targetTicker?.QuickRatio, - InterestCoverage = targetTicker?.InterestCoverage, - - DividendYield = targetTicker?.DividendYield, - PayoutRatio = targetTicker?.PayoutRatio, - ExDividendDate = entity.ExDividendDate ?? targetTicker?.ExDividendDate, - NextEarningsDate = entity.NextEarningsDate, - - PercentHeldByInstitutions = entity.PercentHeldByInstitutions, - PercentHeldByInsiders = entity.PercentHeldByInsiders, - ShortRatio = entity.ShortRatio, - ShortPercentOfFloat = entity.ShortPercentOfFloat, - ConsensusRating = entity.ConsensusRating, - PriceTargetLow = entity.PriceTargetLow, - PriceTargetHigh = entity.PriceTargetHigh, - PriceTargetMedian = entity.PriceTargetMedian, - PriceTargetMean = entity.PriceTargetMean, - LastUpdatedAt = entity.LastUpdatedAt, - - Executives = entity.Executives.Select(e => new CompanyExecutiveDto - { - Name = e.Name, - Title = e.Title, - Age = e.Age, - Compensation = e.Compensation - }).ToList(), - FinancialStatements = entity.FinancialStatements.Select(s => new FinancialStatementDto - { - PeriodType = s.PeriodType, - EndDate = s.EndDate, - TotalRevenue = s.TotalRevenue, - CostOfRevenue = s.CostOfRevenue, - GrossProfit = s.GrossProfit, - OperatingExpenses = s.OperatingExpenses, - OperatingIncome = s.OperatingIncome, - Ebitda = s.Ebitda, - NetIncome = s.NetIncome, - EpsBasic = s.EpsBasic, - EpsDiluted = s.EpsDiluted, - CashAndCashEquivalents = s.CashAndCashEquivalents, - AccountsReceivable = s.AccountsReceivable, - Inventory = s.Inventory, - TotalCurrentAssets = s.TotalCurrentAssets, - TotalNonCurrentAssets = s.TotalNonCurrentAssets, - CurrentLiabilities = s.CurrentLiabilities, - LongTermDebt = s.LongTermDebt, - TotalLiabilities = s.TotalLiabilities, - TotalStockholdersEquity = s.TotalStockholdersEquity, - OperatingCashFlow = s.OperatingCashFlow, - InvestingCashFlow = s.InvestingCashFlow, - CapitalExpenditures = s.CapitalExpenditures, - FinancingCashFlow = s.FinancingCashFlow, - FreeCashFlow = s.FreeCashFlow - }).OrderByDescending(s => s.EndDate).ToList(), - Estimates = entity.Estimates.Select(e => new ForwardEstimateDto - { - Period = e.Period, - ExpectedRevenue = e.ExpectedRevenue, - ExpectedEps = e.ExpectedEps, - ExpectedGrowthRate = e.ExpectedGrowthRate - }).ToList(), - AvailableTickers = entity.TickerFundamentals.Select(t => new TickerDto - { - Ticker = t.Ticker, - Exchange = t.Exchange, - TradingCurrency = t.TradingCurrency, - CurrentPrice = t.CurrentPrice, - DayChangeAbsolute = t.DayChangeAbsolute, - DayChangePercent = t.DayChangePercent, - FiftyTwoWeekHigh = t.FiftyTwoWeekHigh, - FiftyTwoWeekLow = t.FiftyTwoWeekLow, - MarketCapitalization = t.MarketCapitalization, - EnterpriseValue = t.EnterpriseValue, - PeRatioTrailing = t.PeRatioTrailing, - PeRatioForward = t.PeRatioForward, - PegRatio = t.PegRatio, - PbRatio = t.PbRatio, - PsRatio = t.PsRatio, - EvToEbitda = t.EvToEbitda, - EvToRevenue = t.EvToRevenue, - GrossMargin = t.GrossMargin, - OperatingMargin = t.OperatingMargin, - NetProfitMargin = t.NetProfitMargin, - ReturnOnEquity = t.ReturnOnEquity, - ReturnOnAssets = t.ReturnOnAssets, - ReturnOnInvestedCapital = t.ReturnOnInvestedCapital, - DebtToEquity = t.DebtToEquity, - CurrentRatio = t.CurrentRatio, - QuickRatio = t.QuickRatio, - InterestCoverage = t.InterestCoverage, - DividendYield = t.DividendYield, - PayoutRatio = t.PayoutRatio, - ExDividendDate = t.ExDividendDate ?? entity.ExDividendDate - }).ToList() - }; - } - /// public async Task> GetAllEventsAsync(CancellationToken cancellationToken = default) { using var scope = _scopeFactory.CreateScope(); var context = scope.ServiceProvider.GetRequiredService(); - var now = DateTime.UtcNow; - var startOfToday = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0, DateTimeKind.Utc); - var endOfYear = new DateTime(now.Year, 12, 31, 23, 59, 59, DateTimeKind.Utc); - - var entities = await context.AssetFundamentals + var events = await context.AssetEvents + .Include(e => e.AssetData) .AsNoTracking() - .Where(f => (f.NextEarningsDate.HasValue && f.NextEarningsDate.Value >= startOfToday && f.NextEarningsDate.Value <= endOfYear) - || (f.ExDividendDate.HasValue && f.ExDividendDate.Value >= startOfToday && f.ExDividendDate.Value <= endOfYear)) .ToListAsync(cancellationToken); - var events = new List(); - - foreach (var entity in entities) + return events.Select(e => new CorporateEventDto { - var companyName = string.IsNullOrWhiteSpace(entity.CompanyName) ? entity.PrimaryTicker : entity.CompanyName; - - if (entity.NextEarningsDate.HasValue) - { - events.Add(new CorporateEventDto - { - Isin = entity.Isin, - Ticker = entity.PrimaryTicker, - CompanyName = companyName, - EventType = "Quartalsergebnis", - Date = entity.NextEarningsDate.Value - }); - } - - if (entity.ExDividendDate.HasValue) - { - events.Add(new CorporateEventDto - { - Isin = entity.Isin, - Ticker = entity.PrimaryTicker, - CompanyName = companyName, - EventType = "Ex-Dividendentag", - Date = entity.ExDividendDate.Value - }); - } - } - - return events.OrderBy(e => e.Date).ToList(); + Id = e.Id, + Ticker = e.Ticker != null + ? new TickerInfoDto { Ticker = e.Ticker.Ticker, Exchange = !string.IsNullOrWhiteSpace(e.Ticker.Exchange) ? e.Ticker.Exchange : GetExchangeDisplayName(e.Ticker.Ticker) } + : new TickerInfoDto { Ticker = "Unknown", Exchange = "Unknown" }, + Type = e.Type, + Date = e.Date + }).OrderBy(e => e.Date).ToList(); } /// - public async Task> GetEventsByMonthAsync(int year, int month, CancellationToken cancellationToken = default) + public async Task> GetEventsByMonthAsync(int year, int month, + CancellationToken cancellationToken = default) { using var scope = _scopeFactory.CreateScope(); var context = scope.ServiceProvider.GetRequiredService(); @@ -732,51 +495,171 @@ public class FundamentalsDbService : IFundamentalsDbService var startOfMonth = new DateTime(year, month, 1, 0, 0, 0, DateTimeKind.Utc); var startOfNextMonth = startOfMonth.AddMonths(1); - _logger.LogInformation("[FundamentalsDbService] Querying events between {Start} and {End}", startOfMonth, startOfNextMonth); - - var entities = await context.AssetFundamentals + var events = await context.AssetEvents + .Include(e => e.AssetData) .AsNoTracking() - .Where(f => (f.NextEarningsDate != null && f.NextEarningsDate >= startOfMonth && f.NextEarningsDate < startOfNextMonth) - || (f.ExDividendDate != null && f.ExDividendDate >= startOfMonth && f.ExDividendDate < startOfNextMonth)) + .Where(e => e.Date >= startOfMonth && e.Date < startOfNextMonth) .ToListAsync(cancellationToken); - - _logger.LogInformation("[FundamentalsDbService] Found {Count} entities.", entities.Count); - var events = new List(); - - foreach (var entity in entities) + return events.Select(e => new CorporateEventDto { - var companyName = string.IsNullOrWhiteSpace(entity.CompanyName) ? entity.PrimaryTicker : entity.CompanyName; - - if (entity.NextEarningsDate != null && entity.NextEarningsDate >= startOfMonth && entity.NextEarningsDate < startOfNextMonth) - { - events.Add(new CorporateEventDto - { - Isin = entity.Isin, - Ticker = entity.PrimaryTicker, - CompanyName = companyName, - EventType = "Quartalsergebnis", - Date = entity.NextEarningsDate.Value - }); - } - - if (entity.ExDividendDate != null && entity.ExDividendDate >= startOfMonth && entity.ExDividendDate < startOfNextMonth) - { - events.Add(new CorporateEventDto - { - Isin = entity.Isin, - Ticker = entity.PrimaryTicker, - CompanyName = companyName, - EventType = "Ex-Dividendentag", - Date = entity.ExDividendDate.Value - }); - } - } - - _logger.LogInformation("[FundamentalsDbService] Returning {Count} total events.", events.Count); - - return events.OrderBy(e => e.Date).ToList(); + Id = e.Id, + Ticker = e.Ticker != null + ? new TickerInfoDto { Ticker = e.Ticker.Ticker, Exchange = !string.IsNullOrWhiteSpace(e.Ticker.Exchange) ? e.Ticker.Exchange : GetExchangeDisplayName(e.Ticker.Ticker) } + : new TickerInfoDto { Ticker = "Unknown", Exchange = "Unknown" }, + Type = e.Type, + Date = e.Date + }).OrderBy(e => e.Date).ToList(); } - #endregion + private static AssetFundamentalsDto MapToDto( + AssetDataEntity assetData, + FundamentalDataEntity? fundData, + List executives, + List events) + { + var tickerEntities = assetData.AvailableTickers != null && assetData.AvailableTickers.Count > 0 + ? assetData.AvailableTickers + : (assetData.PrimaryTicker != null ? new List { assetData.PrimaryTicker } : new List()); + + var tickerDtos = tickerEntities + .Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker)) + .Select(a => new TickerInfoDto + { + Ticker = a.Ticker, + Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker) + }) + .ToList(); + + var primaryTickerDto = assetData.PrimaryTicker != null && !string.IsNullOrWhiteSpace(assetData.PrimaryTicker.Ticker) + ? new TickerInfoDto + { + Ticker = assetData.PrimaryTicker.Ticker, + Exchange = !string.IsNullOrWhiteSpace(assetData.PrimaryTicker.Exchange) + ? assetData.PrimaryTicker.Exchange + : GetExchangeDisplayName(assetData.PrimaryTicker.Ticker) + } + : (tickerDtos.FirstOrDefault() ?? new TickerInfoDto { Ticker = assetData.Isin, Exchange = "Unknown" }); + + if (!tickerDtos.Any(t => string.Equals(t.Ticker, primaryTickerDto.Ticker, StringComparison.OrdinalIgnoreCase))) + { + tickerDtos.Insert(0, primaryTickerDto); + } + + return new AssetFundamentalsDto + { + Asset = new AssetHeaderDto + { + Isin = assetData.Isin, + Name = assetData.Name, + Description = assetData.Description, + PrimaryTicker = primaryTickerDto, + AvailableTickers = tickerDtos + }, + Fundamentals = fundData != null + ? new FundamentalDataDto + { + Ticker = fundData.Ticker != null && !string.IsNullOrWhiteSpace(fundData.Ticker.Ticker) + ? new TickerInfoDto + { + Ticker = fundData.Ticker.Ticker, + Exchange = !string.IsNullOrWhiteSpace(fundData.Ticker.Exchange) + ? fundData.Ticker.Exchange + : GetExchangeDisplayName(fundData.Ticker.Ticker) + } + : primaryTickerDto, + MarketCap = fundData.MarketCap, + EnterpriseValue = fundData.EnterpriseValue, + TrailingPe = fundData.TrailingPe, + ForwardPe = fundData.ForwardPe, + PegRatio = fundData.PegRatio, + PriceToSales = fundData.PriceToSales, + PriceToBook = fundData.PriceToBook, + EvToEbitda = fundData.EvToEbitda, + TotalRevenue = fundData.TotalRevenue, + RevenueGrowthYoY = fundData.RevenueGrowthYoY, + GrossProfit = fundData.GrossProfit, + OperatingIncome = fundData.OperatingIncome, + Ebitda = fundData.Ebitda, + NetIncome = fundData.NetIncome, + DilutedEps = fundData.DilutedEps, + TotalCash = fundData.TotalCash, + TotalDebt = fundData.TotalDebt, + DebtToEquity = fundData.DebtToEquity, + CurrentRatio = fundData.CurrentRatio, + OperatingCashFlow = fundData.OperatingCashFlow, + FreeCashFlow = fundData.FreeCashFlow, + ReturnOnEquity = fundData.ReturnOnEquity, + ReturnOnAssets = fundData.ReturnOnAssets, + ForwardDividendYield = fundData.ForwardDividendYield, + PayoutRatio = fundData.PayoutRatio, + FiftyTwoWeekHigh = fundData.FiftyTwoWeekHigh, + FiftyTwoWeekLow = fundData.FiftyTwoWeekLow, + ConsensusRating = fundData.ConsensusRating, + PriceTargetLow = fundData.PriceTargetLow, + PriceTargetMean = fundData.PriceTargetMean, + PriceTargetHigh = fundData.PriceTargetHigh, + PercentHeldByInstitutions = fundData.PercentHeldByInstitutions, + PercentHeldByInsiders = fundData.PercentHeldByInsiders, + ShortPercentOfFloat = fundData.ShortPercentOfFloat, + ShortRatio = fundData.ShortRatio, + LastUpdatedUtc = fundData.LastUpdatedUtc + } + : null, + Executives = executives.Select(e => new KeyExecutiveDto + { + Id = e.Id, + Name = e.Name, + Title = e.Title, + Payment = e.Payment + }).ToList(), + Events = events.Select(e => new CorporateEventDto + { + Id = e.Id, + Ticker = e.Ticker != null && !string.IsNullOrWhiteSpace(e.Ticker.Ticker) + ? new TickerInfoDto + { + Ticker = e.Ticker.Ticker, + Exchange = !string.IsNullOrWhiteSpace(e.Ticker.Exchange) + ? e.Ticker.Exchange + : GetExchangeDisplayName(e.Ticker.Ticker) + } + : primaryTickerDto, + Type = e.Type, + Date = e.Date + }).ToList(), + LastUpdatedAt = fundData?.LastUpdatedUtc ?? DateTime.UtcNow + }; + } + + /// + /// Leitet den Anzeigenamen der Börse aus dem Ticker-Suffix ab. + /// + private static string GetExchangeDisplayName(string symbol) + { + if (string.IsNullOrWhiteSpace(symbol)) return "Unknown"; + + if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return "Xetra"; + if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return "Frankfurt"; + if (symbol.EndsWith(".STU", StringComparison.OrdinalIgnoreCase) || symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return "Stuttgart"; + if (symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) return "Hamburg"; + if (symbol.EndsWith(".MU", StringComparison.OrdinalIgnoreCase)) return "München"; + if (symbol.EndsWith(".DU", StringComparison.OrdinalIgnoreCase)) return "Düsseldorf"; + if (symbol.EndsWith(".BE", StringComparison.OrdinalIgnoreCase)) return "Berlin"; + if (symbol.EndsWith(".L", StringComparison.OrdinalIgnoreCase)) return "London"; + if (symbol.EndsWith(".PA", StringComparison.OrdinalIgnoreCase)) return "Paris"; + if (symbol.EndsWith(".AS", StringComparison.OrdinalIgnoreCase)) return "Amsterdam"; + if (symbol.EndsWith(".MI", StringComparison.OrdinalIgnoreCase)) return "Mailand"; + if (symbol.EndsWith(".MC", StringComparison.OrdinalIgnoreCase)) return "Madrid"; + if (symbol.EndsWith(".SW", StringComparison.OrdinalIgnoreCase)) return "Zürich"; + if (symbol.EndsWith(".TO", StringComparison.OrdinalIgnoreCase)) return "Toronto"; + if (symbol.EndsWith(".AX", StringComparison.OrdinalIgnoreCase)) return "Sydney"; + if (symbol.EndsWith(".T", StringComparison.OrdinalIgnoreCase)) return "Tokyo"; + if (symbol.EndsWith(".HK", StringComparison.OrdinalIgnoreCase)) return "Hong Kong"; + + // Kein Suffix -> US-Börse (NASDAQ / NYSE) + if (!symbol.Contains('.')) return "US"; + + return "Other"; + } } \ No newline at end of file diff --git a/FinlyticFundamentals/Services/HtmlFallbackScraper.cs b/FinlyticFundamentals/Services/HtmlFallbackScraper.cs deleted file mode 100644 index 27e27b3..0000000 --- a/FinlyticFundamentals/Services/HtmlFallbackScraper.cs +++ /dev/null @@ -1,305 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using FinlyticFundamentals.Entities; -using Microsoft.Extensions.Logging; -using Microsoft.Playwright; - -namespace FinlyticFundamentals.Services; - -public interface IHtmlFallbackScraper -{ - Task ScrapeFallbackAsync(string isin, string ticker, CancellationToken cancellationToken = default); -} - -/// -/// Fallback scraper using Playwright (headless Chromium) to scrape Yahoo Finance pages -/// for alternative ticker symbols (e.g., APC.SG) whose data is not available via the API. -/// Targets stable data-testid selectors from the rendered Yahoo Finance SPA. -/// -public class HtmlFallbackScraper : IHtmlFallbackScraper, IAsyncDisposable -{ - private readonly ILogger _logger; - - private IPlaywright? _playwright; - private IBrowser? _browser; - private readonly SemaphoreSlim _browserLock = new(1, 1); - - public HtmlFallbackScraper(ILogger logger) - { - _logger = logger; - } - - public async Task ScrapeFallbackAsync( - string isin, - string ticker, - CancellationToken cancellationToken = default) - { - _logger.LogInformation( - "[YahooFallbackScraper] Executing Playwright Fallback Scrape for ticker '{Ticker}' (ISIN: {Isin})...", - ticker, isin); - - try - { - var browser = await GetOrInitBrowserAsync(cancellationToken); - - var fundamentals = new AssetFundamentalsEntity - { - Isin = isin, - PrimaryTicker = ticker, - LastUpdatedAt = DateTime.UtcNow, - LastStaticUpdatedAt = DateTime.UtcNow - }; - - var tickerData = new TickerFundamentalsEntity - { - Ticker = ticker, - Isin = isin, - LastUpdatedAt = DateTime.UtcNow - }; - - fundamentals.CompanyName = ticker; - // Share context for both pages so we only have to accept cookies once - await using (var ctx = await browser.NewContextAsync(BuildContextOptions())) - { - // ── 1. Key Statistics Page ───────────────────────────────────── - var statsUrl = $"https://finance.yahoo.com/quote/{Uri.EscapeDataString(ticker)}/key-statistics/"; - _logger.LogInformation("[YahooFallbackScraper] Navigating to stats page: {Url}", statsUrl); - - var statsPage = await ctx.NewPageAsync(); - try - { - await statsPage.GotoAsync(statsUrl, new PageGotoOptions - { - WaitUntil = WaitUntilState.DOMContentLoaded, - Timeout = 45_000 - }); - - await HandleConsentAsync(statsPage); - - // Wait for the statistics section to be visible - await statsPage.WaitForSelectorAsync( - "section[data-testid='qsp-statistics'], section[data-testid='stats-highlight']", - new PageWaitForSelectorOptions { Timeout = 20_000 }); - - // --- Valuation Measures Table --- - var valuationRows = await statsPage.QuerySelectorAllAsync( - "section[data-testid='qsp-statistics'] table tbody tr"); - - foreach (var row in valuationRows) - { - var cells = await row.QuerySelectorAllAsync("td"); - if (cells.Count < 2) continue; - - var label = (await cells[0].InnerTextAsync()).Trim(); - var value = (await cells[1].InnerTextAsync()).Trim(); - - if (string.IsNullOrWhiteSpace(value) || value == "N/A" || value == "--") continue; - - switch (NormalizeLabel(label)) - { - case "market cap": tickerData.MarketCapitalization = ParseSuffixNumber(value) ?? 0m; break; - case "enterprise value": tickerData.EnterpriseValue = ParseSuffixNumber(value) ?? 0m; break; - case "trailing p/e": tickerData.PeRatioTrailing = ParseDecimal(value); break; - case "forward p/e": tickerData.PeRatioForward = ParseDecimal(value); break; - case "peg ratio (5yr expected)": tickerData.PegRatio = ParseDecimal(value); break; - case "price/sales": tickerData.PsRatio = ParseDecimal(value); break; - case "price/book": tickerData.PbRatio = ParseDecimal(value); break; - case "enterprise value/revenue": tickerData.EvToRevenue = ParseDecimal(value); break; - case "enterprise value/ebitda": tickerData.EvToEbitda = ParseDecimal(value); break; - } - } - - // --- Financial Highlight Cards --- - var highlightRows = await statsPage.QuerySelectorAllAsync( - "div[data-testid='stats-highlight'] section[data-testid='card-container'] table tr"); - - foreach (var row in highlightRows) - { - var cells = await row.QuerySelectorAllAsync("td"); - if (cells.Count < 2) continue; - - var label = (await cells[0].InnerTextAsync()).Trim(); - var value = (await cells[1].InnerTextAsync()).Trim(); - - if (string.IsNullOrWhiteSpace(value) || value == "N/A" || value == "--") continue; - - switch (NormalizeLabel(label)) - { - case "profit margin": tickerData.NetProfitMargin = ParsePercent(value); break; - case "operating margin": tickerData.OperatingMargin = ParsePercent(value); break; - case "return on assets": tickerData.ReturnOnAssets = ParsePercent(value); break; - case "return on equity": tickerData.ReturnOnEquity = ParsePercent(value); break; - case "current ratio": tickerData.CurrentRatio = ParseDecimal(value); break; - case "quick ratio": tickerData.QuickRatio = ParseDecimal(value); break; - case "total debt/equity": tickerData.DebtToEquity = ParseDecimal(value); break; - case "52 week high": tickerData.FiftyTwoWeekHigh = ParseDecimal(value) ?? 0m; break; - case "52 week low": tickerData.FiftyTwoWeekLow = ParseDecimal(value) ?? 0m; break; - case "forward annual dividend yield": - case "trailing annual dividend yield": - tickerData.DividendYield ??= ParsePercent(value); break; - case "payout ratio": tickerData.PayoutRatio = ParsePercent(value); break; - } - } - - _logger.LogInformation( - "[YahooFallbackScraper] Stats parsed for '{Ticker}': MarketCap={MarketCap}, EV={EV}, TrailingPE={PE}, ForwardPE={FPE}", - ticker, tickerData.MarketCapitalization, tickerData.EnterpriseValue, - tickerData.PeRatioTrailing, tickerData.PeRatioForward); - } - catch (TimeoutException tex) - { - _logger.LogWarning(tex, - "[YahooFallbackScraper] Timeout waiting for stats page selectors for ticker '{Ticker}'. Page may not have loaded.", ticker); - } - finally - { - await statsPage.CloseAsync(); - } - - } - - // ── 2. Guard: no meaningful data ────────────────────────────── - if (tickerData.MarketCapitalization == 0) - { - _logger.LogWarning( - "[YahooFallbackScraper] Playwright scrape for '{Ticker}' produced no meaningful data (MarketCap=0). Returning NULL.", - ticker); - return null; - } - - _logger.LogInformation( - "[YahooFallbackScraper] Playwright Fallback Scrape complete for '{Ticker}'. MarketCap={MarketCap}", - ticker, tickerData.MarketCapitalization); - - return new ScrapedFundamentalsData( - fundamentals, - tickerData, - new List(), - new List(), - new List() - ); - } - catch (Exception ex) - { - _logger.LogError(ex, - "[YahooFallbackScraper] Error during Playwright Fallback Scrape for ticker '{Ticker}' (ISIN: {Isin})", - ticker, isin); - return null; - } - } - - // ── Browser Lifecycle ────────────────────────────────────────────────── - - private async Task GetOrInitBrowserAsync(CancellationToken cancellationToken = default) - { - if (_browser != null) return _browser; - - await _browserLock.WaitAsync(cancellationToken); - try - { - if (_browser != null) return _browser; - - _playwright = await Playwright.CreateAsync(); - _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions - { - Headless = true, - Args = new[] { "--no-sandbox", "--disable-dev-shm-usage" } - }); - - - _logger.LogInformation("[YahooFallbackScraper] Playwright Chromium browser initialized."); - return _browser; - } - finally - { - _browserLock.Release(); - } - } - - private static BrowserNewContextOptions BuildContextOptions() => new() - { - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - ViewportSize = new ViewportSize { Width = 1280, Height = 900 }, - Locale = "en-US", - ExtraHTTPHeaders = new Dictionary - { - ["Accept-Language"] = "en-US,en;q=0.9" - } - }; - - public async ValueTask DisposeAsync() - { - if (_browser != null) await _browser.CloseAsync(); - _playwright?.Dispose(); - } - - // ── Parse Helpers ───────────────────────────────────────────────────── - - private async Task HandleConsentAsync(IPage page) - { - try - { - if (page.Url.Contains("consent.yahoo.com")) - { - _logger.LogInformation("[YahooFallbackScraper] Redirected to consent page. Attempting to accept cookies..."); - var agreeBtn = page.Locator("button[name='agree'], button.accept-all, button[value='agree']"); - if (await agreeBtn.CountAsync() > 0) - { - await agreeBtn.First.ClickAsync(); - await page.WaitForNavigationAsync(new PageWaitForNavigationOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 20_000 }); - _logger.LogInformation("[YahooFallbackScraper] Cookie consent accepted. Navigated back to: {Url}", page.Url); - } - else - { - _logger.LogWarning("[YahooFallbackScraper] On consent page but could not find the 'agree' button."); - } - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "[YahooFallbackScraper] Error while handling cookie consent."); - } - } - - /// Lowercases, strips trailing digits, common Yahoo qualifiers like (ttm), (mrq), and extra spaces. - private static string NormalizeLabel(string label) - { - label = label.ToLowerInvariant(); - label = Regex.Replace(label, @"\s*\d+\s*$", ""); // trailing superscripts - label = label.Replace("(ttm)", "").Replace("(mrq)", "").Replace("(fye)", ""); // remove date qualifiers - label = Regex.Replace(label, @"\s+", " "); // collapse spaces - return label.Trim(); - } - - private static decimal? ParseDecimal(string? input) - { - if (string.IsNullOrWhiteSpace(input) || input == "N/A" || input == "--" || input == "-") return null; - input = Regex.Replace(input, @"[^\d.-]", ""); - return decimal.TryParse(input, NumberStyles.Any, CultureInfo.InvariantCulture, out var val) ? val : null; - } - - private static decimal? ParsePercent(string? input) - { - var val = ParseDecimal(input); - if (!val.HasValue) return null; - return val.Value > 1m ? val.Value / 100m : val.Value; - } - - private static decimal? ParseSuffixNumber(string? input) - { - if (string.IsNullOrWhiteSpace(input) || input == "N/A" || input == "--" || input == "-") return null; - input = input.Trim(); - decimal multiplier = input.EndsWith("T", StringComparison.OrdinalIgnoreCase) ? 1_000_000_000_000m - : input.EndsWith("B", StringComparison.OrdinalIgnoreCase) ? 1_000_000_000m - : input.EndsWith("M", StringComparison.OrdinalIgnoreCase) ? 1_000_000m - : input.EndsWith("K", StringComparison.OrdinalIgnoreCase) ? 1_000m - : 1m; - var numPart = Regex.Replace(input, @"[^\d.-]", ""); - var val = ParseDecimal(numPart); - return val.HasValue ? val.Value * multiplier : null; - } -} \ No newline at end of file diff --git a/FinlyticFundamentals/Services/YahooFinanceScraper.cs b/FinlyticFundamentals/Services/YahooFinanceScraper.cs index 7500214..912defe 100644 --- a/FinlyticFundamentals/Services/YahooFinanceScraper.cs +++ b/FinlyticFundamentals/Services/YahooFinanceScraper.cs @@ -2,12 +2,15 @@ using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using FinlyticCore.Clients; +using FinlyticCore.Dtos.Fundamentals; using FinlyticCore.Dtos.Yahoo; +using FinlyticCore.Services; using FinlyticCore.Services.Yahoo; -using FinlyticFundamentals.Entities; +using FinlyticFundamentals.Database; +using FinlyticFundamentals.Util; using Microsoft.Extensions.Logging; namespace FinlyticFundamentals.Services; @@ -15,477 +18,242 @@ namespace FinlyticFundamentals.Services; public interface IYahooFinanceScraper { /// - /// Resolves ticker from ISIN. + /// Ermittelt den primären Börsenticker zu einer ISIN anhand von Börsenplatz-Prioritäten. /// - Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default); + Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// - /// Resolves all tickers from ISIN. + /// Ermittelt alle gefundenen Börsenticker zu einer ISIN, sortiert nach Priorität. /// - Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default); + Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default); /// - /// Scrapes fundamentals. + /// Ruft Fundamental- und Unternehmensdaten primär über die Yahoo Finance API ab + /// und fällt automatisch auf den Playwright HTML Scraper zurück, falls keine Daten vorhanden sind. /// - Task ScrapeFundamentalsAsync(string isin, string ticker, + /// Das Tickersymbol (z. B. "MSFT") oder die ISIN. + /// Erzwingt sofortiges HTML-Scraping ohne API-Vorprüfung. + /// Abbruch-Token. + /// Das aggregierte oder null. + Task GetQuoteSummaryModulesAsync( + string symbolOrIsin, + bool forceHtmlScrape = false, CancellationToken cancellationToken = default); } -public record ScrapedFundamentalsData( - AssetFundamentalsEntity Fundamentals, - TickerFundamentalsEntity TickerData, - List Executives, - List Statements, - List Estimates -); - public class YahooFinanceScraper : IYahooFinanceScraper { - private readonly HttpClient _httpClient; - private readonly YahooFinanceClient _yahooClient; - private readonly ILogger _logger; + private readonly YahooFinanceClient _yahooApiClient; + private readonly IYahooFinanceHtmlClient _htmlScraperClient; + private readonly IFinlyticLogger _finlyticLogger; - public YahooFinanceScraper(HttpClient httpClient, YahooFinanceClient yahooClient, - ILogger logger) + public YahooFinanceScraper( + YahooFinanceClient yahooApiClient, + IYahooFinanceHtmlClient htmlScraperClient, + IFinlyticLogger finlyticLogger) { - _httpClient = httpClient; - _yahooClient = yahooClient; - _logger = logger; + _yahooApiClient = yahooApiClient; + _htmlScraperClient = htmlScraperClient; + _finlyticLogger = finlyticLogger; } /// - public async Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default) + public async Task ResolveTickerFromIsinAsync(string isin, CancellationToken cancellationToken = default) { var tickers = await ResolveAllTickersFromIsinAsync(isin, cancellationToken); return tickers.FirstOrDefault(); } /// - public async Task> ResolveAllTickersFromIsinAsync(string isin, - CancellationToken cancellationToken = default) + public async Task> ResolveAllTickersFromIsinAsync(string isin, CancellationToken cancellationToken = default) { - if (string.IsNullOrWhiteSpace(isin)) return new(); + if (string.IsNullOrWhiteSpace(isin)) return new List(); - var symbols = new List<(string symbol, int priority)>(); - - var primary = await _yahooClient.SearchAsync(isin, quotesCount: 20, cancellationToken: cancellationToken); - var quotes = primary?.Quotes ?? new(); - - foreach (var q in quotes.Where(q => !string.IsNullOrEmpty(q.Symbol))) - { - symbols.Add((q.Symbol, GetExchangePriority(q.Symbol, isin))); - } - - if (quotes.Count == 0) return []; - - // 2. Namenssuche für deutsche/andere Handelsplätze - var companyName = quotes[0].LongName!; - - var secondary = - await _yahooClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken); - - foreach (var q in secondary?.Quotes ?? new()) - { - if (!string.IsNullOrEmpty(q.Symbol) && - !symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase))) - { - symbols.Add((q.Symbol, GetExchangePriority(q.Symbol, isin))); - } - } - - - // 3. Sortieren und zurückgeben - return symbols - .OrderBy(s => s.priority) - .Select(s => s.symbol) - .Distinct(StringComparer.OrdinalIgnoreCase) - .Take(20) - .ToList(); - } - - private int GetExchangePriority(string symbol, string isin) - { - if (!string.IsNullOrEmpty(isin) && isin.StartsWith("US", StringComparison.OrdinalIgnoreCase)) - { - if (!symbol.Contains('.')) return 1; - if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; - if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase) || - symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 3; - return 4; - } - - if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) - { - return 1; // XETRA - } - else if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) - { - return 2; // Frankfurt - } - else if (symbol.EndsWith(".TG", StringComparison.OrdinalIgnoreCase)) - { - return 3; // Gettex - } - else if (symbol.EndsWith(".MU", StringComparison.OrdinalIgnoreCase) || - symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase) || - symbol.EndsWith(".BE", StringComparison.OrdinalIgnoreCase) || - symbol.EndsWith(".DU", StringComparison.OrdinalIgnoreCase) || - symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) - { - return 4; // Other German regional exchanges - } - else if (symbol.Contains('.') && !symbol.EndsWith(".OB", StringComparison.OrdinalIgnoreCase) && - !symbol.EndsWith(".PK", StringComparison.OrdinalIgnoreCase)) - { - return 5; // Domestic/home non-US exchanges - } - else - { - return 6; // Other - } - } - - /// - public async Task ScrapeFundamentalsAsync(string isin, string ticker, - CancellationToken cancellationToken = default) - { - _logger.LogInformation( - "[{Channel}] Fetching fundamental data for Ticker {Ticker} (ISIN: {Isin}) using YahooFinanceClient...", - "FundamentalsChannel", ticker, isin); + var cleanIsin = isin.Trim().ToUpperInvariant(); + var symbols = new List<(string symbol, string exchange, int priority)>(); try { - var summaryResponse = await _yahooClient.GetFullQuoteSummaryAsync(ticker, cancellationToken); - if (summaryResponse?.QuoteSummary?.Result == null || summaryResponse.QuoteSummary.Result.Count == 0) + // 1. Suche via ISIN + var primary = await _yahooApiClient.SearchAsync(cleanIsin, quotesCount: 20, cancellationToken: cancellationToken); + var quotes = primary?.Quotes ?? new List(); + + foreach (var q in quotes.Where(q => !string.IsNullOrEmpty(q.Symbol))) { - _logger.LogWarning("[{Channel}] YahooFinanceClient returned no result for ticker {Ticker}", - "FundamentalsChannel", ticker); - return null; + symbols.Add((q.Symbol, q.Exchange ?? string.Empty, GetExchangePriority(q.Symbol, cleanIsin))); } - var root = summaryResponse.QuoteSummary.Result[0]; - - var assetProfile = root.AssetProfile; - var financialData = root.FinancialData; - var defaultKeyStatistics = root.DefaultKeyStatistics; - var summaryDetail = root.SummaryDetail; - var calendarEvents = root.CalendarEvents; - - // Instantiate entities - var fundamentals = new AssetFundamentalsEntity + // 2. Falls Ticker gefunden, aber mit Unternehmensname noch mehr Exchangeticker auffindbar sind + if (quotes.Count > 0) { - Isin = isin, - PrimaryTicker = ticker, - LastUpdatedAt = DateTime.UtcNow, - LastStaticUpdatedAt = DateTime.UtcNow - }; - - var tickerData = new TickerFundamentalsEntity - { - Ticker = ticker, - Isin = isin, - LastUpdatedAt = DateTime.UtcNow - }; - - // 1. Static Profile Data - if (assetProfile != null) - { - fundamentals.BusinessSummary = assetProfile.LongBusinessSummary; - fundamentals.Sector = assetProfile.Sector; - fundamentals.Industry = assetProfile.Industry; - fundamentals.Country = assetProfile.Country; - fundamentals.Employees = assetProfile.FullTimeEmployees; - } - - // Company Name - fundamentals.CompanyName = ticker; - - // 2. Exchange & Trading Currency for Ticker - if (financialData != null && !string.IsNullOrWhiteSpace(financialData.FinancialCurrency)) - { - tickerData.TradingCurrency = financialData.FinancialCurrency; - } - - if (summaryDetail != null && !string.IsNullOrWhiteSpace(summaryDetail.Currency)) - { - tickerData.TradingCurrency = summaryDetail.Currency; - } - - // 3. Dynamic Price & Valuation Data - if (financialData != null) - { - tickerData.CurrentPrice = financialData.CurrentPrice?.DecimalValue ?? 0; - tickerData.GrossMargin = financialData.GrossMargins?.DecimalValue; - tickerData.OperatingMargin = financialData.OperatingMargins?.DecimalValue; - tickerData.NetProfitMargin = financialData.ProfitMargins?.DecimalValue; - tickerData.ReturnOnEquity = financialData.ReturnOnEquity?.DecimalValue; - tickerData.ReturnOnAssets = financialData.ReturnOnAssets?.DecimalValue; - tickerData.CurrentRatio = financialData.CurrentRatio?.DecimalValue; - tickerData.QuickRatio = financialData.QuickRatio?.DecimalValue; - tickerData.DebtToEquity = financialData.DebtToEquity?.DecimalValue; - - // Targets on Company Level - fundamentals.PriceTargetLow = financialData.TargetLowPrice?.DecimalValue; - fundamentals.PriceTargetHigh = financialData.TargetHighPrice?.DecimalValue; - fundamentals.PriceTargetMedian = financialData.TargetMedianPrice?.DecimalValue; - fundamentals.PriceTargetMean = financialData.TargetMeanPrice?.DecimalValue; - } - - if (summaryDetail != null) - { - if (tickerData.CurrentPrice == 0) + var companyName = quotes[0].LongName ?? quotes[0].ShortName; + if (!string.IsNullOrWhiteSpace(companyName)) { - tickerData.CurrentPrice = summaryDetail.Open?.DecimalValue ?? - summaryDetail.PreviousClose?.DecimalValue ?? 0; - } - - tickerData.FiftyTwoWeekHigh = summaryDetail.FiftyTwoWeekHigh?.DecimalValue ?? 0; - tickerData.FiftyTwoWeekLow = summaryDetail.FiftyTwoWeekLow?.DecimalValue ?? 0; - } - - var mCap = defaultKeyStatistics?.SharesOutstanding?.DecimalValue; - mCap ??= summaryDetail?.MarketCap?.DecimalValue; - tickerData.MarketCapitalization = mCap ?? 0; - - var ev = defaultKeyStatistics?.EnterpriseValue?.DecimalValue; - tickerData.EnterpriseValue = ev ?? 0; - - tickerData.PeRatioTrailing = defaultKeyStatistics?.TrailingEps?.DecimalValue ?? - summaryDetail?.TrailingPE?.DecimalValue; - tickerData.PeRatioForward = - defaultKeyStatistics?.ForwardPE?.DecimalValue ?? summaryDetail?.ForwardPE?.DecimalValue; - - if (defaultKeyStatistics != null) - { - tickerData.PegRatio = defaultKeyStatistics.PegRatio?.DecimalValue; - tickerData.PbRatio = defaultKeyStatistics.PriceToBook?.DecimalValue; - fundamentals.ShortRatio = defaultKeyStatistics.ShortRatio?.DecimalValue; - fundamentals.ShortPercentOfFloat = defaultKeyStatistics.ShortPercentOfFloat?.DecimalValue; - fundamentals.PercentHeldByInstitutions = defaultKeyStatistics.HeldPercentInstitutions?.DecimalValue; - fundamentals.PercentHeldByInsiders = defaultKeyStatistics.HeldPercentInsiders?.DecimalValue; - } - - tickerData.PsRatio = defaultKeyStatistics?.PriceToSalesTrailing12Months?.DecimalValue ?? - summaryDetail?.PriceToSalesTrailing12Months?.DecimalValue; - tickerData.EvToEbitda = defaultKeyStatistics?.EnterpriseToEbitda?.DecimalValue; - tickerData.EvToRevenue = defaultKeyStatistics?.EnterpriseToRevenue?.DecimalValue; - - tickerData.DividendYield = summaryDetail?.DividendYield?.DecimalValue; - tickerData.PayoutRatio = summaryDetail?.PayoutRatio?.DecimalValue; - - if (financialData != null) - { - if (!string.IsNullOrWhiteSpace(financialData.RecommendationKey) && - !financialData.RecommendationKey.Equals("none", StringComparison.OrdinalIgnoreCase)) - { - fundamentals.ConsensusRating = financialData.RecommendationKey; - } - else if (financialData.RecommendationMean != null && financialData.RecommendationMean.Raw.HasValue) - { - double mean = financialData.RecommendationMean.Raw.Value; - fundamentals.ConsensusRating = mean <= 1.8 - ? "strong_buy" - : (mean <= 2.5 ? "buy" : (mean <= 3.5 ? "hold" : (mean <= 4.2 ? "sell" : "strong_sell"))); - } - } - - // 4. Calendar Events Data - if (calendarEvents != null) - { - if (calendarEvents.ExDividendDate?.Raw.HasValue == true) - { - long seconds = (long)calendarEvents.ExDividendDate.Raw.Value; - if (seconds > 0) - fundamentals.ExDividendDate = DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime; - } - - if (calendarEvents.Earnings?.EarningsDate != null && calendarEvents.Earnings.EarningsDate.Count > 0) - { - var firstDate = calendarEvents.Earnings.EarningsDate[0]; - if (firstDate.Raw.HasValue && firstDate.Raw.Value > 0) + var secondary = await _yahooApiClient.SearchAsync(companyName, quotesCount: 20, cancellationToken: cancellationToken); + foreach (var q in secondary?.Quotes ?? new List()) { - fundamentals.NextEarningsDate = - DateTimeOffset.FromUnixTimeSeconds((long)firstDate.Raw.Value).UtcDateTime; + if (!string.IsNullOrEmpty(q.Symbol) && + !symbols.Any(s => s.symbol.Equals(q.Symbol, StringComparison.OrdinalIgnoreCase))) + { + symbols.Add((q.Symbol, q.Exchange ?? string.Empty, GetExchangePriority(q.Symbol, cleanIsin))); + } } } } - - if (!fundamentals.ExDividendDate.HasValue && summaryDetail?.ExDividendDate?.Raw.HasValue == true) - { - long seconds = (long)summaryDetail.ExDividendDate.Raw.Value; - if (seconds > 0) fundamentals.ExDividendDate = DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime; - } - - tickerData.ExDividendDate = fundamentals.ExDividendDate; - - // 5. Executives List - var executives = new List(); - if (assetProfile?.CompanyOfficers != null) - { - foreach (var officer in assetProfile.CompanyOfficers) - { - var exec = new CompanyExecutiveEntity - { - Isin = isin, - Name = !string.IsNullOrWhiteSpace(officer.Name) ? officer.Name : "Unknown", - Title = !string.IsNullOrWhiteSpace(officer.Title) ? officer.Title : "Officer", - Age = officer.Age, - Compensation = officer.TotalPay?.DecimalValue - }; - executives.Add(exec); - } - } - - // 6. Financial Statements - var statements = new List(); - - // A. Annual Statements - if (root.IncomeStatementHistory?.IncomeStatementHistory != null) - { - foreach (var item in root.IncomeStatementHistory.IncomeStatementHistory) - { - MapIncomeStatement(item, isin, "Annual", statements); - } - } - - if (root.BalanceSheetHistory?.BalanceSheetStatements != null) - { - foreach (var item in root.BalanceSheetHistory.BalanceSheetStatements) - { - MapBalanceSheet(item, isin, "Annual", statements); - } - } - - if (root.CashflowStatementHistory?.CashflowStatements != null) - { - foreach (var item in root.CashflowStatementHistory.CashflowStatements) - { - MapCashflowStatement(item, isin, "Annual", statements); - } - } - - // B. Quarterly Statements - if (root.IncomeStatementHistoryQuarterly?.IncomeStatementHistory != null) - { - foreach (var item in root.IncomeStatementHistoryQuarterly.IncomeStatementHistory) - { - MapIncomeStatement(item, isin, "Quarterly", statements); - } - } - - if (root.BalanceSheetHistoryQuarterly?.BalanceSheetStatements != null) - { - foreach (var item in root.BalanceSheetHistoryQuarterly.BalanceSheetStatements) - { - MapBalanceSheet(item, isin, "Quarterly", statements); - } - } - - if (root.CashflowStatementHistoryQuarterly?.CashflowStatements != null) - { - foreach (var item in root.CashflowStatementHistoryQuarterly.CashflowStatements) - { - MapCashflowStatement(item, isin, "Quarterly", statements); - } - } - - // 7. Forward Estimates - var estimates = new List(); - - return new ScrapedFundamentalsData(fundamentals, tickerData, executives, statements, estimates); } catch (Exception ex) { - _logger.LogError(ex, "[{Channel}] Failed to scrape fundamentals for ISIN {Isin} (Ticker: {Ticker})", - "FundamentalsChannel", isin, ticker); - return null; + await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex, + "[YahooFinanceScraper] Fehler beim Auflösen des Tickers für ISIN '{Isin}'", cleanIsin); } + + return symbols + .OrderBy(s => s.priority) + .Select(s => new TickerInfoDto(){Ticker = s.symbol, Exchange = s.exchange}) + .ToList(); } - private static void MapIncomeStatement(YahooIncomeStatementDto item, string isin, string periodType, - List statements) + /// + public async Task GetQuoteSummaryModulesAsync( + string symbolOrIsin, + bool forceHtmlScrape = false, + CancellationToken cancellationToken = default) { - if (item.EndDate?.Raw.HasValue != true) return; - var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date; + if (string.IsNullOrWhiteSpace(symbolOrIsin)) return null; - var statement = GetOrCreateStatement(statements, isin, periodType, endDate); + var symbol = symbolOrIsin.Trim().ToUpperInvariant(); - if (item.TotalRevenue?.Raw.HasValue == true) statement.TotalRevenue = item.TotalRevenue.DecimalValue; - if (item.CostOfRevenue?.Raw.HasValue == true) statement.CostOfRevenue = item.CostOfRevenue.DecimalValue; - if (item.GrossProfit?.Raw.HasValue == true) statement.GrossProfit = item.GrossProfit.DecimalValue; - else if (statement.TotalRevenue.HasValue && statement.CostOfRevenue.HasValue) - statement.GrossProfit = statement.TotalRevenue - statement.CostOfRevenue; - - if (item.TotalOperatingExpenses?.Raw.HasValue == true) - statement.OperatingExpenses = item.TotalOperatingExpenses.DecimalValue; - if (item.OperatingIncome?.Raw.HasValue == true) statement.OperatingIncome = item.OperatingIncome.DecimalValue; - else if (statement.GrossProfit.HasValue && statement.OperatingExpenses.HasValue) - statement.OperatingIncome = statement.GrossProfit - statement.OperatingExpenses; - - if (item.Ebit?.Raw.HasValue == true) statement.Ebitda = item.Ebit.DecimalValue; - if (item.NetIncome?.Raw.HasValue == true) statement.NetIncome = item.NetIncome.DecimalValue; - } - - private static void MapBalanceSheet(YahooBalanceSheetStatementDto item, string isin, string periodType, - List statements) - { - if (item.EndDate?.Raw.HasValue != true) return; - var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date; - - var statement = GetOrCreateStatement(statements, isin, periodType, endDate); - - if (item.Cash?.Raw.HasValue == true) statement.CashAndCashEquivalents = item.Cash.DecimalValue; - if (item.NetReceivables?.Raw.HasValue == true) statement.AccountsReceivable = item.NetReceivables.DecimalValue; - if (item.Inventory?.Raw.HasValue == true) statement.Inventory = item.Inventory.DecimalValue; - if (item.TotalCurrentAssets?.Raw.HasValue == true) - statement.TotalCurrentAssets = item.TotalCurrentAssets.DecimalValue; - if (item.TotalCurrentLiabilities?.Raw.HasValue == true) - statement.CurrentLiabilities = item.TotalCurrentLiabilities.DecimalValue; - if (item.LongTermDebt?.Raw.HasValue == true) statement.LongTermDebt = item.LongTermDebt.DecimalValue; - if (item.TotalLiab?.Raw.HasValue == true) statement.TotalLiabilities = item.TotalLiab.DecimalValue; - if (item.TotalStockholderEquity?.Raw.HasValue == true) - statement.TotalStockholdersEquity = item.TotalStockholderEquity.DecimalValue; - } - - private static void MapCashflowStatement(YahooCashflowStatementDto item, string isin, string periodType, - List statements) - { - if (item.EndDate?.Raw.HasValue != true) return; - var endDate = DateTimeOffset.FromUnixTimeSeconds((long)item.EndDate.Raw.Value).UtcDateTime.Date; - - var statement = GetOrCreateStatement(statements, isin, periodType, endDate); - - if (item.TotalCashFromOperatingActivities?.Raw.HasValue == true) - statement.OperatingCashFlow = item.TotalCashFromOperatingActivities.DecimalValue; - if (item.TotalCashflowsFromInvestingActivities?.Raw.HasValue == true) - statement.InvestingCashFlow = item.TotalCashflowsFromInvestingActivities.DecimalValue; - if (item.CapitalExpenditures?.Raw.HasValue == true) - statement.CapitalExpenditures = item.CapitalExpenditures.DecimalValue; - if (item.TotalCashFromFinancingActivities?.Raw.HasValue == true) - statement.FinancingCashFlow = item.TotalCashFromFinancingActivities.DecimalValue; - - if (statement.OperatingCashFlow.HasValue) + // Falls eine ISIN übergeben wurde, zuerst Ticker auflösen + if (IsIsin(symbol)) { - var capex = statement.CapitalExpenditures ?? 0m; - statement.FreeCashFlow = statement.OperatingCashFlow.Value - Math.Abs(capex); - } - } - - private static FinancialStatementEntity GetOrCreateStatement(List statements, string isin, - string periodType, DateTime endDate) - { - var existing = statements.FirstOrDefault(s => s.PeriodType == periodType && s.EndDate.Date == endDate.Date); - if (existing == null) - { - existing = new FinancialStatementEntity + var resolvedTicker = await ResolveTickerFromIsinAsync(symbol, cancellationToken); + if (resolvedTicker != null) { - Isin = isin, - PeriodType = periodType, - EndDate = endDate.Date - }; - statements.Add(existing); + symbol = resolvedTicker.Ticker; + } } - return existing; + YahooQuoteSummaryModulesDto? apiModules = null; + + // ------------------------------------------------------------- + // 1. PRIMÄRE DATENQUELLE: Yahoo Finance API (Cookie/Crumb) + // ------------------------------------------------------------- + if (!forceHtmlScrape) + { + try + { + await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel, + "[YahooFinanceScraper] Starte primären API-Abruf für '{Symbol}'...", symbol); + + var apiResponse = await _yahooApiClient.GetFullQuoteSummaryAsync(symbol, cancellationToken); + apiModules = apiResponse?.QuoteSummary?.Result?.FirstOrDefault(); + + if (apiModules != null && HasSufficientData(apiModules)) + { + await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel, + "[YahooFinanceScraper] Erfolgreich Daten über API bezogen für '{Symbol}'.", symbol); + return apiModules; + } + + await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, + "[YahooFinanceScraper] API lieferte unvollständige Daten für '{Symbol}'. Initiiere Fallback...", symbol); + } + catch (Exception ex) + { + await _finlyticLogger.LogWarningAsync(SettingKeys.YahooClientChannel, ex, + "[YahooFinanceScraper] API-Abruf fehlgeschlagen für '{Symbol}'. Wechsle zu Scraper...", symbol); + } + } + + // ------------------------------------------------------------- + // 2. FALLBACK DATENQUELLE: Playwright HTML Scraper + // ------------------------------------------------------------- + YahooQuoteSummaryModulesDto? htmlModules = null; + try + { + await _finlyticLogger.LogInfoAsync(SettingKeys.YahooClientChannel, + "[YahooFinanceScraper] Starte HTML-Scraper Fallback für '{Symbol}'...", symbol); + + htmlModules = await _htmlScraperClient.ScrapeQuoteSummaryModulesAsync(symbol, cancellationToken); + } + catch (Exception ex) + { + await _finlyticLogger.LogErrorAsync(SettingKeys.YahooClientChannel, ex, + "[YahooFinanceScraper] HTML-Scraper Fallback ebenfalls fehlgeschlagen für '{Symbol}'.", symbol); + } + + // ------------------------------------------------------------- + // 3. Zusammenführen (Merge API & HTML Fallback) + // ------------------------------------------------------------- + if (apiModules == null) return htmlModules; + if (htmlModules == null) return apiModules; + + return MergeModules(apiModules, htmlModules); + } + + /// + /// Prüft, ob das Modul-DTO die wesentlichen Fundamentalblöcke enthält. + /// + private static bool HasSufficientData(YahooQuoteSummaryModulesDto modules) + { + return modules.SummaryDetail != null || + modules.FinancialData != null || + modules.DefaultKeyStatistics != null; + } + + /// + /// Führt API- und Scraper-Daten zusammen, damit Lücken in API-Responses geschlossen werden. + /// + private static YahooQuoteSummaryModulesDto MergeModules( + YahooQuoteSummaryModulesDto primary, + YahooQuoteSummaryModulesDto secondary) + { + return new YahooQuoteSummaryModulesDto( + QuoteType: primary.QuoteType ?? secondary.QuoteType, + AssetProfile: primary.AssetProfile ?? secondary.AssetProfile, + FinancialData: primary.FinancialData ?? secondary.FinancialData, + DefaultKeyStatistics: primary.DefaultKeyStatistics ?? secondary.DefaultKeyStatistics, + SummaryDetail: primary.SummaryDetail ?? secondary.SummaryDetail, + IncomeStatementHistory: primary.IncomeStatementHistory ?? secondary.IncomeStatementHistory, + IncomeStatementHistoryQuarterly: primary.IncomeStatementHistoryQuarterly ?? secondary.IncomeStatementHistoryQuarterly, + BalanceSheetHistory: primary.BalanceSheetHistory ?? secondary.BalanceSheetHistory, + BalanceSheetHistoryQuarterly: primary.BalanceSheetHistoryQuarterly ?? secondary.BalanceSheetHistoryQuarterly, + CashflowStatementHistory: primary.CashflowStatementHistory ?? secondary.CashflowStatementHistory, + CashflowStatementHistoryQuarterly: primary.CashflowStatementHistoryQuarterly ?? secondary.CashflowStatementHistoryQuarterly, + CalendarEvents: primary.CalendarEvents ?? secondary.CalendarEvents + ); + } + + private static bool IsIsin(string value) + { + return value.Length == 12 && + char.IsLetter(value[0]) && + char.IsLetter(value[1]) && + value.All(char.IsLetterOrDigit); + } + + private static int GetExchangePriority(string symbol, string isin) + { + bool isGermanIsin = isin.StartsWith("DE", StringComparison.OrdinalIgnoreCase); + + if (isGermanIsin) + { + if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 1; // Xetra + if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 2; // Frankfurt + if (symbol.EndsWith(".STU", StringComparison.OrdinalIgnoreCase)) return 3; // Stuttgart + if (symbol.EndsWith(".SG", StringComparison.OrdinalIgnoreCase)) return 4; // Stuttgart (alt) + if (symbol.EndsWith(".HM", StringComparison.OrdinalIgnoreCase)) return 5; // Hamburg + if (!symbol.Contains('.')) return 6; // US Primary + } + else + { + if (!symbol.Contains('.')) return 1; // US Primary (NASDAQ, NYSE) + if (symbol.EndsWith(".DE", StringComparison.OrdinalIgnoreCase)) return 2; // Xetra + if (symbol.EndsWith(".F", StringComparison.OrdinalIgnoreCase)) return 3; // Frankfurt + if (symbol.EndsWith(".L", StringComparison.OrdinalIgnoreCase)) return 4; // London + if (symbol.EndsWith(".PA", StringComparison.OrdinalIgnoreCase)) return 5; // Paris + } + + return 10; } } \ No newline at end of file diff --git a/FinlyticFundamentals/Util/FundamentalsMqttClient.cs b/FinlyticFundamentals/Util/FundamentalsMqttClient.cs index bca1def..3bbbd10 100644 --- a/FinlyticFundamentals/Util/FundamentalsMqttClient.cs +++ b/FinlyticFundamentals/Util/FundamentalsMqttClient.cs @@ -1,11 +1,12 @@ using System; -using System.Collections.Generic; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos; using FinlyticCore.Models; +using FinlyticCore.Services; using FinlyticCore.Util; +using FinlyticFundamentals.Database; using FinlyticFundamentals.Services; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -18,18 +19,15 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService { private readonly ILogger _logger; private readonly IConfiguration _configuration; - private readonly IFundamentalsDbService _dbService; private readonly IServiceScopeFactory _scopeFactory; public FundamentalsMqttClient( ILogger logger, IConfiguration configuration, - IFundamentalsDbService dbService, IServiceScopeFactory scopeFactory) : base(logger) { _logger = logger; _configuration = configuration; - _dbService = dbService; _scopeFactory = scopeFactory; } @@ -43,7 +41,7 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService ClientId = _configuration["MQTT:ClientId"] ?? "finlytic_fundamentals_" + Guid.NewGuid().ToString("N") }; - _logger.LogInformation("[{Channel}] Starting Fundamentals MQTT client. Host: {Host}, ClientId: {ClientId}", "FundamentalsChannel", config.Host, config.ClientId); + _logger.LogInformation("[{Channel}] [MQTT_Client] Starting Fundamentals MQTT client. Host: {Host}, ClientId: {ClientId}", "MqttChannel", config.Host, config.ClientId); await ConnectAsync(config); } @@ -51,19 +49,18 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService /// public async Task StopAsync(CancellationToken cancellationToken) { - _logger.LogInformation("[{Channel}] Stopping Fundamentals MQTT client.", "FundamentalsChannel"); + _logger.LogInformation("[{Channel}] [MQTT_Client] Stopping Fundamentals MQTT client.", "MqttChannel"); await DisconnectAsync(); } /// protected override async Task OnConnectedAsync() { - _logger.LogInformation("[{Channel}] Fundamentals MQTT client connected. Subscribing to RPC request topics...", "FundamentalsChannel"); + _logger.LogInformation("[{Channel}] [MQTT_Client] Connected. Subscribing to RPC request topics...", "MqttChannel"); await SubscribeAsync("services/request/fundamentals_Get/#"); await SubscribeAsync("services/request/events_GetAll/#"); await SubscribeAsync("services/request/events_GetByMonth/#"); await SubscribeAsync("services/request/health_Ping/#"); - await SubscribeAsync("services/config/updated/#"); } /// @@ -71,23 +68,11 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService { if (string.IsNullOrWhiteSpace(topic)) return; - // 1. Config update events - if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase)) - { - if (topic.EndsWith("FinlyticFundamentals", StringComparison.OrdinalIgnoreCase)) - { - await OnConfigUpdatedAsync(payload); - } - return; - } - - // Extract correlationId from topic suffix (e.g. services/request/fundamentals_Get/{correlationId}) var lastSlash = topic.LastIndexOf('/'); if (lastSlash < 0 || lastSlash >= topic.Length - 1) return; var correlationId = topic.Substring(lastSlash + 1); - // 2. Dispatch to specific channel handlers if (topic.StartsWith("services/request/fundamentals_Get", StringComparison.OrdinalIgnoreCase)) { await OnFundamentalsGetAsync(payload, correlationId); @@ -106,14 +91,15 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService } } - /// - /// Handles fundamentals_Get RPC requests using source-generated DTO deserialization. - /// private async Task OnFundamentalsGetAsync(string payload, string correlationId) { + using var scope = _scopeFactory.CreateScope(); + var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); + var dbService = scope.ServiceProvider.GetRequiredService(); + if (string.IsNullOrWhiteSpace(payload)) { - _logger.LogWarning("[{Channel}] [FundamentalsMqttClient] Received empty payload for fundamentals_Get request.", "FundamentalsChannel"); + await finlyticLogger.LogWarningAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Received empty payload for fundamentals_Get request."); return; } @@ -122,108 +108,83 @@ public class FundamentalsMqttClient : ManagedMqttClient, IHostedService var request = (IsinRequest?)JsonSerializer.Deserialize(payload, typeof(IsinRequest), FinlyticJsonSerializerContext.Default); if (request == null || string.IsNullOrWhiteSpace(request.Isin)) { - _logger.LogWarning("[{Channel}] [FundamentalsMqttClient] Request missing mandatory ISIN parameter in payload.", "FundamentalsChannel"); + await finlyticLogger.LogWarningAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Request missing mandatory ISIN parameter in payload."); return; } - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Processing RPC fundamentals_Get for ISIN '{Isin}' (forceRefresh={ForceRefresh}) [CorrelationId: {CorrelationId}]", - "FundamentalsChannel", request.Isin, request.ForceRefresh.ToString(), correlationId); + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC fundamentals_Get for ISIN '{Isin}' (forceRefresh={ForceRefresh}) [CorrelationId: {CorrelationId}]", + request.Isin, request.ForceRefresh.ToString(), correlationId); - var fundamentals = await _dbService.GetFundamentalsAsync(request.Isin, request.Ticker, request.ForceRefresh); + var fundamentals = await dbService.GetFundamentalsAsync(request.Isin, request.Ticker, request.ForceRefresh); var responseTopic = $"services/response/fundamentals_Get/{correlationId}"; - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Publishing RPC fundamentals response to '{ResponseTopic}'", "FundamentalsChannel", responseTopic); + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Publishing RPC fundamentals response to '{ResponseTopic}'", responseTopic); await PublishAsync(responseTopic, fundamentals); } catch (Exception ex) { - _logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Failed to process fundamentals_Get request.", "FundamentalsChannel"); + await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [MQTT_Client] Failed to process fundamentals_Get request."); } } - /// - /// Handles events_GetAll RPC requests. - /// private async Task OnEventsGetAllAsync(string correlationId) { - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Processing RPC events_GetAll request [CorrelationId: {CorrelationId}]", "FundamentalsChannel", correlationId); + using var scope = _scopeFactory.CreateScope(); + var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); + var dbService = scope.ServiceProvider.GetRequiredService(); + + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC events_GetAll request [CorrelationId: {CorrelationId}]", correlationId); try { - var events = await _dbService.GetAllEventsAsync(); + var events = await dbService.GetAllEventsAsync(); var responseTopic = $"services/response/events_GetAll/{correlationId}"; - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Publishing events RPC response to '{ResponseTopic}'", "FundamentalsChannel", responseTopic); + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Publishing events RPC response to '{ResponseTopic}'", responseTopic); await PublishAsync(responseTopic, events); } catch (Exception ex) { - _logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Failed to process events_GetAll request.", "FundamentalsChannel"); + await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [MQTT_Client] Failed to process events_GetAll request."); } } - /// - /// Handles events_GetByMonth RPC requests. - /// private async Task OnEventsGetByMonthAsync(string payload, string correlationId) { if (string.IsNullOrWhiteSpace(payload)) return; + using var scope = _scopeFactory.CreateScope(); + var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); + var dbService = scope.ServiceProvider.GetRequiredService(); + try { var request = (GetEventsByMonthRequest?)JsonSerializer.Deserialize(payload, typeof(GetEventsByMonthRequest), FinlyticJsonSerializerContext.Default); if (request == null) return; - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Processing RPC events_GetByMonth request for {Year}/{Month} [CorrelationId: {CorrelationId}]", "FundamentalsChannel", request.Year, request.Month, correlationId); + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Processing RPC events_GetByMonth request for {Year}/{Month} [CorrelationId: {CorrelationId}]", request.Year, request.Month, correlationId); - var events = await _dbService.GetEventsByMonthAsync(request.Year, request.Month); + var events = await dbService.GetEventsByMonthAsync(request.Year, request.Month); var responseTopic = $"services/response/events_GetByMonth/{correlationId}"; - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Publishing monthly events RPC response to '{ResponseTopic}'", "FundamentalsChannel", responseTopic); + await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticFundamentals] [MQTT_Client] Publishing monthly events RPC response to '{ResponseTopic}'", responseTopic); await PublishAsync(responseTopic, events); } catch (Exception ex) { - _logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Failed to process events_GetByMonth request.", "FundamentalsChannel"); + await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticFundamentals] [MQTT_Client] Failed to process events_GetByMonth request."); } } - /// - /// Handles health_Ping RPC requests. - /// private async Task OnHealthPingAsync(string topic, string correlationId) { if (topic.Contains("FinlyticFundamentals", StringComparison.OrdinalIgnoreCase) || !topic.Contains("/", StringComparison.OrdinalIgnoreCase)) { - string respTopic = $"services/response/health_Ping/{correlationId}"; - await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticFundamentals", "Online", DateTime.UtcNow, "Connected")); - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", "FundamentalsChannel", correlationId); - } - } + using var scope = _scopeFactory.CreateScope(); + var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); - /// - /// Handles dynamic service config update events. - /// - private async Task OnConfigUpdatedAsync(string payload) - { - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Received config update event for FinlyticFundamentals.", "FundamentalsChannel"); - try - { - using var doc = JsonDocument.Parse(payload); - if (doc.RootElement.TryGetProperty("settings", out var settingsProp)) - { - var dict = (Dictionary?)JsonSerializer.Deserialize(settingsProp.GetRawText(), typeof(Dictionary), FinlyticJsonSerializerContext.Default); - if (dict != null && dict.Count > 0) - { - using var scope = _scopeFactory.CreateScope(); - var settingsDb = scope.ServiceProvider.GetRequiredService(); - await settingsDb.UpdateSettingsFromDictionaryAsync(dict); - _logger.LogInformation("[{Channel}] [FundamentalsMqttClient] Persisted {Count} updated settings to FinlyticFundamentals database.", "FundamentalsChannel", dict.Count); - } - } - } - catch (Exception ex) - { - _logger.LogError(ex, "[{Channel}] [FundamentalsMqttClient] Error processing MQTT config update event.", "FundamentalsChannel"); + var respTopic = $"services/response/health_Ping/{correlationId}"; + await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticFundamentals", "Online", DateTime.UtcNow, "Connected")); + await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[FinlyticFundamentals] [Health_Ping] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId); } } } diff --git a/FinlyticFundamentals/Util/SettingKeys.cs b/FinlyticFundamentals/Util/SettingKeys.cs index 36a126a..a244bfd 100644 --- a/FinlyticFundamentals/Util/SettingKeys.cs +++ b/FinlyticFundamentals/Util/SettingKeys.cs @@ -1,6 +1,18 @@ -namespace FinlyticFundamentals.Util; +using FinlyticCore.Models.Settings; + +namespace FinlyticFundamentals.Util; public class SettingKeys { - + // --- Logging Channels --- + public static readonly SettingKey HealthPingChannel = new("Logging.Channel.Health", true); + public static readonly SettingKey MqttChannel = new("Logging.Channel.MQTT", true); + public static readonly SettingKey FundamentalsChannel = new("Logging.Channel.Fundamentals", true); + public static readonly SettingKey HtmlScrapperChannel = new("Logging.Channel.HtmlScrapper", true); + public static readonly SettingKey YahooClientChannel = new("Logging.Channel.YahooClient", true); + + // --- Features & Toggles --- + public static readonly SettingKey EnableHtmlFallback = new("Feature.EnableHtmlFallback", true); + public static readonly SettingKey AllowForceRefresh = new("Feature.AllowForceRefresh", true); + public static readonly SettingKey FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30); } \ No newline at end of file