feat(fundamentals): multi-ticker DB caching, parallel html scraper, and frontend mappings
This commit is contained in:
@@ -67,13 +67,16 @@ public class FundamentalsDbContext : DbContext, ISettingsDbContext
|
||||
|
||||
modelBuilder.Entity<FundamentalDataEntity>(entity =>
|
||||
{
|
||||
entity.HasKey(e => e.Isin);
|
||||
entity.HasKey(e => e.Id);
|
||||
|
||||
entity.OwnsOne(e => e.Ticker, t =>
|
||||
{
|
||||
t.Property(p => p.Ticker).HasColumnName("Ticker").HasDefaultValue(string.Empty);
|
||||
t.Property(p => p.Exchange).HasColumnName("TickerExchange").HasDefaultValue(string.Empty);
|
||||
t.HasIndex(p => p.Ticker);
|
||||
});
|
||||
|
||||
entity.HasIndex(e => new { e.AssetDataIsin });
|
||||
});
|
||||
|
||||
modelBuilder.Entity<KeyExecutiveEntity>(entity =>
|
||||
|
||||
@@ -6,7 +6,7 @@ namespace FinlyticFundamentals.Entities;
|
||||
public class FundamentalDataEntity
|
||||
{
|
||||
[Key]
|
||||
public string Isin { get; set; } = string.Empty;
|
||||
public Guid Id { get; set; } = Guid.NewGuid();
|
||||
|
||||
public TickerEntity Ticker { get; set; } = new();
|
||||
|
||||
|
||||
+433
@@ -0,0 +1,433 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using FinlyticFundamentals.Database;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace FinlyticFundamentals.Migrations
|
||||
{
|
||||
[DbContext(typeof(FundamentalsDbContext))]
|
||||
[Migration("20260816103109_MakeFundamentalDataPerTicker")]
|
||||
partial class MakeFundamentalDataPerTicker
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "10.0.9")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 63);
|
||||
|
||||
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
|
||||
|
||||
modelBuilder.Entity("FinlyticCore.Entities.Settings.SettingEntity", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b.Property<string>("Key")
|
||||
.IsRequired()
|
||||
.HasMaxLength(150)
|
||||
.HasColumnType("character varying(150)");
|
||||
|
||||
b.Property<DateTime>("LastUpdatedUtc")
|
||||
.HasColumnType("timestamp with time zone");
|
||||
|
||||
b.Property<string>("ServiceIdentifier")
|
||||
.IsRequired()
|
||||
.HasMaxLength(100)
|
||||
.HasColumnType("character varying(100)");
|
||||
|
||||
b.Property<string>("ValueJson")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("Key")
|
||||
.IsUnique();
|
||||
|
||||
b.ToTable("DynamicSettings");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
|
||||
{
|
||||
b.Property<string>("Isin")
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<string>("Description")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.HasKey("Isin");
|
||||
|
||||
b.ToTable("AssetData");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b.Property<string>("AssetDataIsin")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<DateTime>("Date")
|
||||
.HasColumnType("timestamp with time zone");
|
||||
|
||||
b.Property<string>("Type")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("AssetDataIsin");
|
||||
|
||||
b.ToTable("AssetEvents");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b.Property<string>("AssetDataIsin")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<string>("ConsensusRating")
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<decimal?>("CurrentRatio")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("DebtToEquity")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("DilutedEps")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("Ebitda")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("EnterpriseValue")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("EvToEbitda")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("FiftyTwoWeekHigh")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("FiftyTwoWeekLow")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("ForwardDividendYield")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("ForwardPe")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("FreeCashFlow")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("GrossProfit")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<DateTime>("LastUpdatedUtc")
|
||||
.HasColumnType("timestamp with time zone");
|
||||
|
||||
b.Property<decimal?>("MarketCap")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("NetIncome")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("OperatingCashFlow")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("OperatingIncome")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PayoutRatio")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PegRatio")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PercentHeldByInsiders")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PercentHeldByInstitutions")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PriceTargetHigh")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PriceTargetLow")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PriceTargetMean")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PriceToBook")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("PriceToSales")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("ReturnOnAssets")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("ReturnOnEquity")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("RevenueGrowthYoY")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("ShortPercentOfFloat")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("ShortRatio")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("TotalCash")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("TotalDebt")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("TotalRevenue")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.Property<decimal?>("TrailingPe")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("AssetDataIsin");
|
||||
|
||||
b.ToTable("FundamentalData");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b.Property<string>("AssetDataIsin")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<string>("Payment")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<int>("SortOrder")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("integer")
|
||||
.HasDefaultValue(0);
|
||||
|
||||
b.Property<string>("Title")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("AssetDataIsin", "SortOrder");
|
||||
|
||||
b.ToTable("KeyExecutives");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
|
||||
{
|
||||
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "PrimaryTicker", b1 =>
|
||||
{
|
||||
b1.Property<string>("AssetDataEntityIsin")
|
||||
.HasColumnType("text");
|
||||
|
||||
b1.Property<string>("Exchange")
|
||||
.IsRequired()
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("text")
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("PrimaryTickerExchange");
|
||||
|
||||
b1.Property<string>("Ticker")
|
||||
.IsRequired()
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("text")
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("PrimaryTicker");
|
||||
|
||||
b1.HasKey("AssetDataEntityIsin");
|
||||
|
||||
b1.ToTable("AssetData");
|
||||
|
||||
b1.WithOwner()
|
||||
.HasForeignKey("AssetDataEntityIsin");
|
||||
});
|
||||
|
||||
b.OwnsMany("FinlyticFundamentals.Entities.TickerEntity", "AvailableTickers", b1 =>
|
||||
{
|
||||
b1.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b1.Property<string>("AssetDataIsin")
|
||||
.IsRequired()
|
||||
.HasColumnType("text");
|
||||
|
||||
b1.Property<string>("Exchange")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("Exchange");
|
||||
|
||||
b1.Property<string>("Ticker")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("Ticker");
|
||||
|
||||
b1.HasKey("Id");
|
||||
|
||||
b1.HasIndex("AssetDataIsin");
|
||||
|
||||
b1.HasIndex("Ticker");
|
||||
|
||||
b1.ToTable("Tickers", (string)null);
|
||||
|
||||
b1.WithOwner()
|
||||
.HasForeignKey("AssetDataIsin");
|
||||
});
|
||||
|
||||
b.Navigation("AvailableTickers");
|
||||
|
||||
b.Navigation("PrimaryTicker")
|
||||
.IsRequired();
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetEventEntity", b =>
|
||||
{
|
||||
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
|
||||
.WithMany("AssetEvents")
|
||||
.HasForeignKey("AssetDataIsin")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
|
||||
{
|
||||
b1.Property<Guid>("AssetEventEntityId")
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b1.Property<string>("Exchange")
|
||||
.IsRequired()
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("text")
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("TickerExchange");
|
||||
|
||||
b1.Property<string>("Ticker")
|
||||
.IsRequired()
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("text")
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("Ticker");
|
||||
|
||||
b1.HasKey("AssetEventEntityId");
|
||||
|
||||
b1.ToTable("AssetEvents");
|
||||
|
||||
b1.WithOwner()
|
||||
.HasForeignKey("AssetEventEntityId");
|
||||
});
|
||||
|
||||
b.Navigation("AssetData");
|
||||
|
||||
b.Navigation("Ticker")
|
||||
.IsRequired();
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
|
||||
{
|
||||
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
|
||||
.WithMany("FundamentalData")
|
||||
.HasForeignKey("AssetDataIsin")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
|
||||
{
|
||||
b1.Property<Guid>("FundamentalDataEntityId")
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b1.Property<string>("Exchange")
|
||||
.IsRequired()
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("text")
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("TickerExchange");
|
||||
|
||||
b1.Property<string>("Ticker")
|
||||
.IsRequired()
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("text")
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("Ticker");
|
||||
|
||||
b1.HasKey("FundamentalDataEntityId");
|
||||
|
||||
b1.HasIndex("Ticker");
|
||||
|
||||
b1.ToTable("FundamentalData");
|
||||
|
||||
b1.WithOwner()
|
||||
.HasForeignKey("FundamentalDataEntityId");
|
||||
});
|
||||
|
||||
b.Navigation("AssetData");
|
||||
|
||||
b.Navigation("Ticker")
|
||||
.IsRequired();
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.KeyExecutiveEntity", b =>
|
||||
{
|
||||
b.HasOne("FinlyticFundamentals.Entities.AssetDataEntity", "AssetData")
|
||||
.WithMany("KeyExecutives")
|
||||
.HasForeignKey("AssetDataIsin")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.Navigation("AssetData");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.AssetDataEntity", b =>
|
||||
{
|
||||
b.Navigation("AssetEvents");
|
||||
|
||||
b.Navigation("FundamentalData");
|
||||
|
||||
b.Navigation("KeyExecutives");
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace FinlyticFundamentals.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class MakeFundamentalDataPerTicker : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropPrimaryKey(
|
||||
name: "PK_FundamentalData",
|
||||
table: "FundamentalData");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "Isin",
|
||||
table: "FundamentalData");
|
||||
|
||||
migrationBuilder.AddColumn<Guid>(
|
||||
name: "Id",
|
||||
table: "FundamentalData",
|
||||
type: "uuid",
|
||||
nullable: false,
|
||||
defaultValueSql: "gen_random_uuid()");
|
||||
|
||||
migrationBuilder.AddPrimaryKey(
|
||||
name: "PK_FundamentalData",
|
||||
table: "FundamentalData",
|
||||
column: "Id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_FundamentalData_Ticker",
|
||||
table: "FundamentalData",
|
||||
column: "Ticker");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropPrimaryKey(
|
||||
name: "PK_FundamentalData",
|
||||
table: "FundamentalData");
|
||||
|
||||
migrationBuilder.DropIndex(
|
||||
name: "IX_FundamentalData_Ticker",
|
||||
table: "FundamentalData");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "Id",
|
||||
table: "FundamentalData");
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "Isin",
|
||||
table: "FundamentalData",
|
||||
type: "text",
|
||||
nullable: false,
|
||||
defaultValue: "");
|
||||
|
||||
migrationBuilder.AddPrimaryKey(
|
||||
name: "PK_FundamentalData",
|
||||
table: "FundamentalData",
|
||||
column: "Isin");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -97,8 +97,9 @@ namespace FinlyticFundamentals.Migrations
|
||||
|
||||
modelBuilder.Entity("FinlyticFundamentals.Entities.FundamentalDataEntity", b =>
|
||||
{
|
||||
b.Property<string>("Isin")
|
||||
.HasColumnType("text");
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b.Property<string>("AssetDataIsin")
|
||||
.IsRequired()
|
||||
@@ -212,7 +213,7 @@ namespace FinlyticFundamentals.Migrations
|
||||
b.Property<decimal?>("TrailingPe")
|
||||
.HasColumnType("numeric");
|
||||
|
||||
b.HasKey("Isin");
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("AssetDataIsin");
|
||||
|
||||
@@ -371,8 +372,8 @@ namespace FinlyticFundamentals.Migrations
|
||||
|
||||
b.OwnsOne("FinlyticFundamentals.Entities.TickerEntity", "Ticker", b1 =>
|
||||
{
|
||||
b1.Property<string>("FundamentalDataEntityIsin")
|
||||
.HasColumnType("text");
|
||||
b1.Property<Guid>("FundamentalDataEntityId")
|
||||
.HasColumnType("uuid");
|
||||
|
||||
b1.Property<string>("Exchange")
|
||||
.IsRequired()
|
||||
@@ -388,12 +389,14 @@ namespace FinlyticFundamentals.Migrations
|
||||
.HasDefaultValue("")
|
||||
.HasColumnName("Ticker");
|
||||
|
||||
b1.HasKey("FundamentalDataEntityIsin");
|
||||
b1.HasKey("FundamentalDataEntityId");
|
||||
|
||||
b1.HasIndex("Ticker");
|
||||
|
||||
b1.ToTable("FundamentalData");
|
||||
|
||||
b1.WithOwner()
|
||||
.HasForeignKey("FundamentalDataEntityIsin");
|
||||
.HasForeignKey("FundamentalDataEntityId");
|
||||
});
|
||||
|
||||
b.Navigation("AssetData");
|
||||
|
||||
@@ -31,8 +31,8 @@ builder.Services.AddHttpClient<IYahooFinanceScraper, YahooFinanceScraper>()
|
||||
AllowAutoRedirect = true
|
||||
});
|
||||
|
||||
builder.Services.AddScoped<IPlaywrightBrowserFactory, PlaywrightBrowserFactory>();
|
||||
builder.Services.AddScoped<IPlaywrightExecutionService, PlaywrightExecutionService>();
|
||||
builder.Services.AddSingleton<IPlaywrightBrowserFactory, PlaywrightBrowserFactory>();
|
||||
builder.Services.AddSingleton<IPlaywrightExecutionService, PlaywrightExecutionService>();
|
||||
builder.Services.AddTransient<IYahooFinanceHtmlClient, YahooFinanceHtmlClient>();
|
||||
|
||||
// Register Application Services
|
||||
|
||||
@@ -79,44 +79,53 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
await settingsService.GetSettingAsync(SettingKeys.AllowForceRefresh, cancellationToken);
|
||||
bool enableHtmlFallback =
|
||||
await settingsService.GetSettingAsync(SettingKeys.EnableHtmlFallback, cancellationToken);
|
||||
bool forceHtmlFallback =
|
||||
await settingsService.GetSettingAsync(SettingKeys.ForceHtmlFallback, cancellationToken);
|
||||
int validityDays =
|
||||
await settingsService.GetSettingAsync(SettingKeys.FundamentalDataValidityDays, cancellationToken);
|
||||
|
||||
bool effectiveForceRefresh = forceRefresh && allowForceRefresh;
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
|
||||
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtmlFallback: {Html}",
|
||||
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback);
|
||||
"[DEBUG-START] GetFundamentalsAsync für ISIN: {Isin} | Ticker: {Ticker} | ForceRefresh: {Force} | EnableHtml: {Html} | ForceHtml: {ForceHtml}",
|
||||
cleanIsin, requestedTicker ?? "NULL", forceRefresh, enableHtmlFallback, forceHtmlFallback);
|
||||
|
||||
// 2. Entitäten aus DB laden
|
||||
var assetData = await context.AssetData
|
||||
.Include(a => a.AvailableTickers)
|
||||
.Include(a => a.KeyExecutives)
|
||||
.Include(a => a.AssetEvents)
|
||||
.Include(a => a.FundamentalData)
|
||||
.FirstOrDefaultAsync(a => a.Isin == cleanIsin, cancellationToken);
|
||||
|
||||
var fundamentalData = await context.FundamentalData
|
||||
.FirstOrDefaultAsync(f => f.Isin == cleanIsin, cancellationToken);
|
||||
string targetTicker = !string.IsNullOrWhiteSpace(requestedTicker)
|
||||
? requestedTicker
|
||||
: (assetData?.PrimaryTicker?.Ticker ?? string.Empty);
|
||||
|
||||
var fundamentalData = assetData?.FundamentalData?
|
||||
.FirstOrDefault(f => !string.IsNullOrWhiteSpace(targetTicker) && string.Equals(f.Ticker.Ticker, targetTicker, StringComparison.OrdinalIgnoreCase))
|
||||
?? (string.IsNullOrWhiteSpace(requestedTicker) ? assetData?.FundamentalData?.FirstOrDefault() : null);
|
||||
|
||||
// 3. Prüfen, was aktualisiert werden muss
|
||||
bool assetDataMissing = assetData == null || string.IsNullOrWhiteSpace(assetData.Name);
|
||||
bool executivesMissing = assetData == null || assetData.KeyExecutives == null ||
|
||||
assetData.KeyExecutives.Count == 0;
|
||||
bool fundamentalsExpired = fundamentalData == null ||
|
||||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
|
||||
bool fundamentalsMissingOrExpired = fundamentalData == null ||
|
||||
(fundamentalData.MarketCap == null && fundamentalData.TrailingPe == null) ||
|
||||
(DateTime.UtcNow - fundamentalData.LastUpdatedUtc).TotalDays > validityDays;
|
||||
bool tickersCorruptOrMissing = assetData?.AvailableTickers == null ||
|
||||
assetData.AvailableTickers.Count == 0 ||
|
||||
assetData.AvailableTickers.Any(t => t.Ticker != null && t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// Wenn ein expliziter Ticker übergeben wurde und sich vom gespeicherten unterscheidet,
|
||||
// müssen Asset-Daten und Fundamentals mit dem neuen Ticker neu abgerufen werden.
|
||||
bool tickerChanged = !string.IsNullOrWhiteSpace(requestedTicker)
|
||||
&& assetData?.PrimaryTicker != null
|
||||
&& !string.Equals(assetData.PrimaryTicker.Ticker, requestedTicker,
|
||||
StringComparison.OrdinalIgnoreCase);
|
||||
bool shouldUpdate = assetDataMissing || executivesMissing || fundamentalsMissingOrExpired || tickersCorruptOrMissing || effectiveForceRefresh || forceHtmlFallback;
|
||||
|
||||
bool shouldUpdateAssetData = assetDataMissing || effectiveForceRefresh || tickerChanged;
|
||||
bool shouldUpdateExecutives = executivesMissing || effectiveForceRefresh;
|
||||
bool shouldUpdateFundamentals = fundamentalsExpired || effectiveForceRefresh || tickerChanged;
|
||||
|
||||
if (shouldUpdateAssetData || shouldUpdateExecutives || shouldUpdateFundamentals)
|
||||
if (!shouldUpdate && fundamentalData != null)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
|
||||
"[FundamentalsDbService] Returning valid cached fundamental data for ISIN {Isin} (Ticker: {Ticker}, Updated: {UpdatedUtc}). External API fetch skipped.",
|
||||
cleanIsin, fundamentalData.Ticker.Ticker, fundamentalData.LastUpdatedUtc.ToString("o"));
|
||||
}
|
||||
else
|
||||
{
|
||||
// --- STEP 1: Trade Republic Details ---
|
||||
TradeRepublicStockDetailsResponse? trDetails = null;
|
||||
@@ -150,8 +159,10 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
TickerInfoDto activeQueryTicker;
|
||||
if (!string.IsNullOrWhiteSpace(requestedTicker))
|
||||
{
|
||||
var matchDto = resolvedTickers.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
|
||||
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(a => string.Equals(a.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
|
||||
var matchDto = resolvedTickers.FirstOrDefault(t =>
|
||||
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
|
||||
var matchEntity = assetData?.AvailableTickers?.FirstOrDefault(t =>
|
||||
string.Equals(t.Ticker, requestedTicker, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (matchDto != null)
|
||||
{
|
||||
@@ -197,12 +208,18 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
yahooPrimaryTicker.Ticker, activeQueryTicker.Ticker, cleanIsin);
|
||||
|
||||
// --- STEP 3 & 4: Yahoo Finance API & HTML Fallback über Scraper ---
|
||||
// Profile (Sektor, Industrie, Vorstände) wird gescrapt, wenn weder in DB noch in TR Vorstände/Beschreibungen vorliegen
|
||||
bool hasProfileInDb = assetData != null && !string.IsNullOrWhiteSpace(assetData.Description) && assetData.KeyExecutives != null && assetData.KeyExecutives.Count > 0;
|
||||
bool hasCeoInTr = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
|
||||
bool needProfile = !hasProfileInDb && !hasCeoInTr;
|
||||
|
||||
YahooQuoteSummaryModulesDto? modulesDto = null;
|
||||
if (!string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) && activeQueryTicker.Ticker != cleanIsin)
|
||||
{
|
||||
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
|
||||
activeQueryTicker.Ticker,
|
||||
forceHtmlScrape: false,
|
||||
forceHtmlScrape: forceHtmlFallback,
|
||||
includeProfile: needProfile,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
else
|
||||
@@ -211,51 +228,96 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
"[DEBUG-YAHOO-SKIPPED] Yahoo-Abruf übersprungen. Ticker: '{Ticker}'", activeQueryTicker.Ticker);
|
||||
}
|
||||
|
||||
// --- Update AssetDataEntity ---
|
||||
if (shouldUpdateAssetData)
|
||||
// Falls der Sekundär-Ticker (z. B. APC.DE) überhaupt keine Daten liefert, nutze den PrimaryTicker (z. B. AAPL) als Fallback
|
||||
if (modulesDto == null &&
|
||||
!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) &&
|
||||
yahooPrimaryTicker.Ticker != activeQueryTicker.Ticker &&
|
||||
yahooPrimaryTicker.Ticker != cleanIsin)
|
||||
{
|
||||
if (assetData == null)
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
|
||||
"[DEBUG-FALLBACK-PRIMARY] Sekundär-Ticker '{Active}' lieferte keine Daten. Versuche PrimaryTicker '{Primary}'...",
|
||||
activeQueryTicker.Ticker, yahooPrimaryTicker.Ticker);
|
||||
|
||||
modulesDto = await _scraper.GetQuoteSummaryModulesAsync(
|
||||
yahooPrimaryTicker.Ticker,
|
||||
forceHtmlScrape: forceHtmlFallback,
|
||||
includeProfile: needProfile,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
// --- Update AssetDataEntity ---
|
||||
if (assetData == null)
|
||||
{
|
||||
assetData = new AssetDataEntity
|
||||
{
|
||||
assetData = new AssetDataEntity
|
||||
Isin = cleanIsin,
|
||||
PrimaryTicker = new TickerEntity
|
||||
{
|
||||
Isin = cleanIsin,
|
||||
PrimaryTicker = new TickerEntity
|
||||
{
|
||||
Ticker = yahooPrimaryTicker.Ticker,
|
||||
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
|
||||
},
|
||||
KeyExecutives = new List<KeyExecutiveEntity>(),
|
||||
AssetEvents = new List<AssetEventEntity>()
|
||||
};
|
||||
context.AssetData.Add(assetData);
|
||||
}
|
||||
Ticker = yahooPrimaryTicker.Ticker,
|
||||
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
|
||||
},
|
||||
KeyExecutives = new List<KeyExecutiveEntity>(),
|
||||
AssetEvents = new List<AssetEventEntity>()
|
||||
};
|
||||
context.AssetData.Add(assetData);
|
||||
}
|
||||
|
||||
string trName = trDetails?.Company?.Name ?? string.Empty;
|
||||
string trDescription = trDetails?.Company?.Description ?? string.Empty;
|
||||
string trName = trDetails?.Company?.Name?.Trim() ?? string.Empty;
|
||||
string trDescription = trDetails?.Company?.Description?.Trim() ?? string.Empty;
|
||||
|
||||
string fallbackName = modulesDto?.QuoteType?.ShortName
|
||||
?? modulesDto?.QuoteType?.LongName
|
||||
?? activeQueryTicker.Ticker;
|
||||
string yahooName = modulesDto?.QuoteType?.LongName?.Trim()
|
||||
?? modulesDto?.QuoteType?.ShortName?.Trim()
|
||||
?? string.Empty;
|
||||
string yahooDesc = modulesDto?.AssetProfile?.LongBusinessSummary?.Trim() ?? string.Empty;
|
||||
|
||||
assetData.Name = !string.IsNullOrWhiteSpace(trName) ? trName : fallbackName;
|
||||
assetData.Description = !string.IsNullOrWhiteSpace(trDescription)
|
||||
? trDescription
|
||||
: (modulesDto?.AssetProfile?.LongBusinessSummary ?? string.Empty);
|
||||
// Name nur aktualisieren, wenn ein echter Name vorliegt (Bestandsdaten niemals mit ISIN/Ticker überschreiben)
|
||||
if (!string.IsNullOrWhiteSpace(trName))
|
||||
{
|
||||
assetData.Name = trName;
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(yahooName))
|
||||
{
|
||||
assetData.Name = yahooName;
|
||||
}
|
||||
else if (string.IsNullOrWhiteSpace(assetData.Name))
|
||||
{
|
||||
assetData.Name = !string.IsNullOrWhiteSpace(activeQueryTicker.Ticker) ? activeQueryTicker.Ticker : cleanIsin;
|
||||
}
|
||||
|
||||
// PrimaryTicker ist FEST der erste von Yahoo Finance
|
||||
// Description nur aktualisieren, wenn neue Beschreibung vorhanden ist
|
||||
if (!string.IsNullOrWhiteSpace(trDescription))
|
||||
{
|
||||
assetData.Description = trDescription;
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(yahooDesc))
|
||||
{
|
||||
assetData.Description = yahooDesc;
|
||||
}
|
||||
|
||||
// PrimaryTicker aktualisieren falls vorhanden
|
||||
if (!string.IsNullOrWhiteSpace(yahooPrimaryTicker.Ticker) && yahooPrimaryTicker.Ticker != cleanIsin)
|
||||
{
|
||||
assetData.PrimaryTicker = new TickerEntity
|
||||
{
|
||||
Ticker = yahooPrimaryTicker.Ticker,
|
||||
Exchange = yahooPrimaryTicker.Exchange ?? "Unknown"
|
||||
};
|
||||
}
|
||||
|
||||
if (!resolvedTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
|
||||
// AvailableTickers aktualisieren (nur echte Börsenticker, keine ISINs)
|
||||
var validTickers = resolvedTickers
|
||||
.Where(t => !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(cleanIsin, StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
if (validTickers.Count > 0)
|
||||
{
|
||||
if (!validTickers.Any(t => string.Equals(t.Ticker, yahooPrimaryTicker.Ticker, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
resolvedTickers.Insert(0, yahooPrimaryTicker);
|
||||
validTickers.Insert(0, yahooPrimaryTicker);
|
||||
}
|
||||
|
||||
assetData.AvailableTickers.Clear();
|
||||
foreach (var a in resolvedTickers)
|
||||
foreach (var a in validTickers)
|
||||
{
|
||||
assetData.AvailableTickers.Add(new TickerEntity
|
||||
{
|
||||
@@ -263,21 +325,19 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
Exchange = !string.IsNullOrWhiteSpace(a.Exchange) ? a.Exchange : GetExchangeDisplayName(a.Ticker)
|
||||
});
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
|
||||
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}'",
|
||||
assetData.Name, assetData.PrimaryTicker.Ticker);
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
|
||||
"[DEBUG-ASSET-SAVED] AssetData gesetzt -> Name: '{Name}' | PrimaryTicker: '{Ticker}' | AvailableTickers: {Count}",
|
||||
assetData.Name, assetData.PrimaryTicker?.Ticker ?? "NULL", assetData.AvailableTickers.Count);
|
||||
|
||||
// --- Process Trade Republic Corporate Events ---
|
||||
if (trDetails != null && (shouldUpdateAssetData || effectiveForceRefresh) && assetData != null)
|
||||
if (trDetails != null && assetData != null)
|
||||
{
|
||||
// 1. Alte Events direkt in der DB löschen (bypasses Change Tracker)
|
||||
await context.AssetEvents
|
||||
.Where(e => e.AssetDataIsin == cleanIsin)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
|
||||
// 2. ALLE tracked AssetEventEntity-Einträge aus dem Change Tracker entfernen
|
||||
foreach (var entry in context.ChangeTracker.Entries<AssetEventEntity>()
|
||||
.Where(e => e.Entity.AssetDataIsin == cleanIsin)
|
||||
.ToList())
|
||||
@@ -285,7 +345,6 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
entry.State = EntityState.Detached;
|
||||
}
|
||||
|
||||
// 3. Navigation-Collection zurücksetzen
|
||||
assetData.AssetEvents = new List<AssetEventEntity>();
|
||||
|
||||
var trEventList = new List<TradeRepublicEventDto>();
|
||||
@@ -324,18 +383,18 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
}
|
||||
|
||||
// --- Process Modules DTO (Executives & Fundamental Data) ---
|
||||
if (modulesDto != null)
|
||||
if (modulesDto != null || trDetails?.Company != null)
|
||||
{
|
||||
// Update KeyExecutives
|
||||
if (shouldUpdateExecutives && assetData != null)
|
||||
// Update KeyExecutives wenn Executives aus TR oder Yahoo vorliegen
|
||||
var yahooOfficers = modulesDto?.AssetProfile?.CompanyOfficers;
|
||||
bool hasTrOfficers = trDetails?.Company != null && !string.IsNullOrWhiteSpace(trDetails.Company.CeoName);
|
||||
|
||||
if (((yahooOfficers != null && yahooOfficers.Count > 0) || hasTrOfficers) && assetData != null)
|
||||
{
|
||||
// 1. Alte Executives direkt in der DB löschen (bypasses Change Tracker)
|
||||
await context.KeyExecutives
|
||||
.Where(e => e.AssetDataIsin == cleanIsin)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
|
||||
// 2. ALLE tracked KeyExecutiveEntity-Einträge aus dem Change Tracker entfernen
|
||||
// (nicht nur die in der Navigation-Collection — der Tracker kann mehr halten)
|
||||
foreach (var entry in context.ChangeTracker.Entries<KeyExecutiveEntity>()
|
||||
.Where(e => e.Entity.AssetDataIsin == cleanIsin)
|
||||
.ToList())
|
||||
@@ -343,14 +402,12 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
entry.State = EntityState.Detached;
|
||||
}
|
||||
|
||||
// 3. Navigation-Collection zurücksetzen
|
||||
assetData.KeyExecutives = new List<KeyExecutiveEntity>();
|
||||
|
||||
// 4. Neue Executives aufbauen und direkt über den DbSet hinzufügen
|
||||
if (modulesDto.AssetProfile?.CompanyOfficers != null)
|
||||
if (yahooOfficers != null && yahooOfficers.Count > 0)
|
||||
{
|
||||
int sortIdx = 0;
|
||||
foreach (var officer in modulesDto.AssetProfile.CompanyOfficers)
|
||||
foreach (var officer in yahooOfficers)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(officer.Name))
|
||||
{
|
||||
@@ -368,6 +425,49 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (hasTrOfficers && trDetails?.Company != null)
|
||||
{
|
||||
int sortIdx = 0;
|
||||
if (!string.IsNullOrWhiteSpace(trDetails.Company.CeoName))
|
||||
{
|
||||
var ceo = new KeyExecutiveEntity
|
||||
{
|
||||
AssetDataIsin = cleanIsin,
|
||||
Name = trDetails.Company.CeoName,
|
||||
Title = "CEO",
|
||||
Payment = string.Empty,
|
||||
SortOrder = sortIdx++
|
||||
};
|
||||
context.KeyExecutives.Add(ceo);
|
||||
assetData.KeyExecutives.Add(ceo);
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(trDetails.Company.CfoName))
|
||||
{
|
||||
var cfo = new KeyExecutiveEntity
|
||||
{
|
||||
AssetDataIsin = cleanIsin,
|
||||
Name = trDetails.Company.CfoName,
|
||||
Title = "CFO",
|
||||
Payment = string.Empty,
|
||||
SortOrder = sortIdx++
|
||||
};
|
||||
context.KeyExecutives.Add(cfo);
|
||||
assetData.KeyExecutives.Add(cfo);
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(trDetails.Company.CooName))
|
||||
{
|
||||
var coo = new KeyExecutiveEntity
|
||||
{
|
||||
AssetDataIsin = cleanIsin,
|
||||
Name = trDetails.Company.CooName,
|
||||
Title = "COO",
|
||||
Payment = string.Empty,
|
||||
SortOrder = sortIdx++
|
||||
};
|
||||
context.KeyExecutives.Add(coo);
|
||||
assetData.KeyExecutives.Add(coo);
|
||||
}
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.FundamentalsChannel,
|
||||
"[DEBUG-EXECUTIVES-SAVED] {Count} Executives zu DB hinzugefügt.",
|
||||
@@ -375,16 +475,23 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
}
|
||||
|
||||
// Update FundamentalDataEntity
|
||||
if (shouldUpdateFundamentals)
|
||||
if (modulesDto != null && (modulesDto.SummaryDetail != null || modulesDto.DefaultKeyStatistics != null || modulesDto.FinancialData != null))
|
||||
{
|
||||
if (fundamentalData == null || !string.Equals(fundamentalData.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fundamentalData = assetData?.FundamentalData?
|
||||
.FirstOrDefault(f => string.Equals(f.Ticker.Ticker, activeQueryTicker.Ticker, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (fundamentalData == null)
|
||||
{
|
||||
fundamentalData = new FundamentalDataEntity
|
||||
{
|
||||
Isin = cleanIsin,
|
||||
Id = Guid.NewGuid(),
|
||||
AssetDataIsin = cleanIsin
|
||||
};
|
||||
context.FundamentalData.Add(fundamentalData);
|
||||
assetData?.FundamentalData.Add(fundamentalData);
|
||||
}
|
||||
|
||||
fundamentalData.Ticker = new TickerEntity
|
||||
@@ -542,7 +649,7 @@ public class FundamentalsDbService : IFundamentalsDbService
|
||||
: (assetData.PrimaryTicker != null ? new List<TickerEntity> { assetData.PrimaryTicker } : new List<TickerEntity>());
|
||||
|
||||
var tickerDtos = tickerEntities
|
||||
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker))
|
||||
.Where(t => t != null && !string.IsNullOrWhiteSpace(t.Ticker) && !t.Ticker.Contains(assetData.Isin, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(a => new TickerInfoDto
|
||||
{
|
||||
Ticker = a.Ticker,
|
||||
|
||||
@@ -13,6 +13,7 @@ public class SettingKeys
|
||||
|
||||
// --- Features & Toggles ---
|
||||
public static readonly SettingKey<bool> EnableHtmlFallback = new("Feature.EnableHtmlFallback", true);
|
||||
public static readonly SettingKey<bool> ForceHtmlFallback = new("Scraper.ForceHtmlFallback", false);
|
||||
public static readonly SettingKey<bool> AllowForceRefresh = new("Feature.AllowForceRefresh", true);
|
||||
public static readonly SettingKey<int> FundamentalDataValidityDays = new("Cache.FundamentalDataValidityDays", 30);
|
||||
}
|
||||
Reference in New Issue
Block a user