From 354b6dd8a29ede2d3d6ab6a790354c75bc17c20b Mon Sep 17 00:00:00 2001 From: SonicGD Date: Wed, 30 Oct 2024 13:14:21 +0500 Subject: [PATCH 1/4] feat: use postgres module for storage --- .../ApplicationExtensions.cs | 24 ++++++++++++--- .../PostgresStorageMetadataModule.cs | 29 ++----------------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/Sitko.Core.Storage.Metadata.Postgres/ApplicationExtensions.cs b/src/Sitko.Core.Storage.Metadata.Postgres/ApplicationExtensions.cs index 097b6c118..c959b53db 100644 --- a/src/Sitko.Core.Storage.Metadata.Postgres/ApplicationExtensions.cs +++ b/src/Sitko.Core.Storage.Metadata.Postgres/ApplicationExtensions.cs @@ -1,6 +1,8 @@ using JetBrains.Annotations; using Microsoft.Extensions.Hosting; using Sitko.Core.App; +using Sitko.Core.Db.Postgres; +using Sitko.Core.Storage.Metadata.Postgres.DB; namespace Sitko.Core.Storage.Metadata.Postgres; @@ -30,18 +32,32 @@ public static ISitkoCoreApplicationBuilder AddPostgresStorageMetadata> configure, string? optionsKey = null) - where TStorageOptions : StorageOptions => - applicationBuilder + where TStorageOptions : StorageOptions + { + AddPostgresDatabase(applicationBuilder); + return applicationBuilder .AddModule, PostgresStorageMetadataModuleOptions>( configure, optionsKey); + } public static ISitkoCoreApplicationBuilder AddPostgresStorageMetadata( this ISitkoCoreApplicationBuilder applicationBuilder, Action>? configure = null, string? optionsKey = null) - where TStorageOptions : StorageOptions => - applicationBuilder + where TStorageOptions : StorageOptions + { + AddPostgresDatabase(applicationBuilder); + return applicationBuilder .AddModule, PostgresStorageMetadataModuleOptions>( configure, optionsKey); + } + + private static void AddPostgresDatabase(ISitkoCoreApplicationBuilder applicationBuilder) + where TStorageOptions : StorageOptions => + applicationBuilder.AddPostgresDatabase(options => + { + options.EnableJsonConversion = true; + options.Schema = StorageDbContext.Schema; + }, $"Storage:Metadata:Postgres:{typeof(TStorageOptions).Name}"); } diff --git a/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs b/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs index ba7cbf015..6358c5a17 100644 --- a/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs +++ b/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs @@ -1,35 +1,10 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using Npgsql; -using Sitko.Core.App; -using Sitko.Core.Storage.Metadata.Postgres.DB; - -namespace Sitko.Core.Storage.Metadata.Postgres; +namespace Sitko.Core.Storage.Metadata.Postgres; public class PostgresStorageMetadataModule : BaseStorageMetadataModule, PostgresStorageMetadataModuleOptions> + PostgresStorageMetadataProvider, PostgresStorageMetadataModuleOptions> where TStorageOptions : StorageOptions { public override string OptionsKey => $"Storage:Metadata:Postgres:{typeof(TStorageOptions).Name}"; public override string[] OptionKeys => new[] { "Storage:Metadata:Postgres:Default", OptionsKey }; - - public override void ConfigureServices(IApplicationContext applicationContext, IServiceCollection services, - PostgresStorageMetadataModuleOptions startupOptions) - { - base.ConfigureServices(applicationContext, services, startupOptions); - services.AddDbContextFactory((serviceProvider, builder) => - { - var options = serviceProvider - .GetRequiredService>>(); - var dataSourceBuilder = new NpgsqlDataSourceBuilder(options.Value.GetConnectionString()); - dataSourceBuilder.EnableDynamicJson(); - var dataSource = dataSourceBuilder.Build(); - builder.UseNpgsql(dataSource, optionsBuilder => - { - optionsBuilder.MigrationsHistoryTable("__EFMigrationsHistory", StorageDbContext.Schema); - }); - }); - } } From 7354de855695ee91c36942be09cd589ef724dbe9 Mon Sep 17 00:00:00 2001 From: SonicGD Date: Wed, 30 Oct 2024 16:04:09 +0500 Subject: [PATCH 2/4] fix: use wiki as page provider --- tests/Sitko.Core.Pdf.Tests/PdfTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Sitko.Core.Pdf.Tests/PdfTest.cs b/tests/Sitko.Core.Pdf.Tests/PdfTest.cs index e23dfe5f9..1d3bd27c5 100644 --- a/tests/Sitko.Core.Pdf.Tests/PdfTest.cs +++ b/tests/Sitko.Core.Pdf.Tests/PdfTest.cs @@ -18,7 +18,7 @@ public async Task Url() var scope = await GetScopeAsync(); var renderer = scope.GetService(); - var url = "https://github.com"; + var url = "https://www.wikipedia.org/"; var bytes = await renderer.GetPdfByUrlAsync(url); Assert.NotEmpty(bytes); @@ -26,7 +26,7 @@ public async Task Url() var text = PdfTextExtractor.GetTextFromPage(pdf.GetPage(1), new LocationTextExtractionStrategy()); pdf.Close(); - Assert.Contains("GitHub", text); + Assert.Contains("The Free Encyclopedia", text); } [Fact] From 929c05f0e969dfbd84948ea43e3da113d615790c Mon Sep 17 00:00:00 2001 From: SonicGD Date: Wed, 30 Oct 2024 16:04:34 +0500 Subject: [PATCH 3/4] fix: trim database name cause postgres doesn't like it too long --- src/Sitko.Core.Xunit/DbBaseTestScope.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Sitko.Core.Xunit/DbBaseTestScope.cs b/src/Sitko.Core.Xunit/DbBaseTestScope.cs index 430458ce7..6c134cadd 100644 --- a/src/Sitko.Core.Xunit/DbBaseTestScope.cs +++ b/src/Sitko.Core.Xunit/DbBaseTestScope.cs @@ -45,7 +45,8 @@ protected void AddDbContext(IHostApplicationBuilder applicationBuild { if (GetConfig(applicationContext.Configuration).UsePostgres) { - moduleOptions.Database = $"{Id}_{dbName}"; + var fullDbName = $"{Id.ToString()[..8]}_{dbName}"; + moduleOptions.Database = fullDbName; moduleOptions.EnableSensitiveLogging = true; moduleOptions.IncludeErrorDetails = true; moduleOptions.EnableNpgsqlPooling = false; From 8b8e0608bee4ec27ddfc6aff7ad61f37d87319da Mon Sep 17 00:00:00 2001 From: SonicGD Date: Wed, 30 Oct 2024 16:04:45 +0500 Subject: [PATCH 4/4] refactor: cleanup --- .../PostgresStorageMetadataModule.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs b/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs index 6358c5a17..1a212d9e3 100644 --- a/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs +++ b/src/Sitko.Core.Storage.Metadata.Postgres/PostgresStorageMetadataModule.cs @@ -6,5 +6,5 @@ public class where TStorageOptions : StorageOptions { public override string OptionsKey => $"Storage:Metadata:Postgres:{typeof(TStorageOptions).Name}"; - public override string[] OptionKeys => new[] { "Storage:Metadata:Postgres:Default", OptionsKey }; + public override string[] OptionKeys => ["Storage:Metadata:Postgres:Default", OptionsKey]; }