From 993679baf9733ef7f8fc4ef43ab8f59be5e625a3 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Mon, 22 Apr 2024 17:13:04 +0200 Subject: [PATCH 01/49] chore: only migrate admin api and consumer api if in development mode --- AdminApi/src/AdminApi/Program.cs | 6 ++++-- ConsumerApi/Program.cs | 23 +++++++++++++---------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/AdminApi/src/AdminApi/Program.cs b/AdminApi/src/AdminApi/Program.cs index b7582799cc..d11171b2df 100644 --- a/AdminApi/src/AdminApi/Program.cs +++ b/AdminApi/src/AdminApi/Program.cs @@ -15,6 +15,7 @@ using Backbone.Tooling.Extensions; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Logging; using Serilog; using Serilog.Exceptions; using Serilog.Exceptions.Core; @@ -79,7 +80,8 @@ static WebApplication CreateApp(string[] args) var app = builder.Build(); Configure(app); - app.MigrateDbContext(); + if (app.Environment.IsLocal() || app.Environment.IsDevelopment()) + app.MigrateDbContext(); return app; } @@ -175,7 +177,7 @@ static void Configure(WebApplication app) app.UseSwagger().UseSwaggerUI(); if (app.Environment.IsDevelopment()) - Microsoft.IdentityModel.Logging.IdentityModelEventSource.ShowPII = true; + IdentityModelEventSource.ShowPII = true; app.UseCors(); diff --git a/ConsumerApi/Program.cs b/ConsumerApi/Program.cs index c752ac59e5..69b281e64f 100644 --- a/ConsumerApi/Program.cs +++ b/ConsumerApi/Program.cs @@ -99,16 +99,19 @@ static WebApplication CreateApp(string[] args) var app = builder.Build(); Configure(app); - app - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext() - .MigrateDbContext(); + if (app.Environment.IsLocal() || app.Environment.IsDevelopment()) + { + app + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext() + .MigrateDbContext(); + } app .SeedDbContext() From c353f7cc4e19c8b7b1ca56d70eeae0d534c2395e Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 12:09:49 +0200 Subject: [PATCH 02/49] feat: add DatabaseMigrator --- Backbone.sln | 6 + DatabaseMigrator/Configuration.cs | 24 ++++ DatabaseMigrator/DatabaseMigrator.csproj | 66 ++++++++++ DatabaseMigrator/Executor.cs | 77 ++++++++++++ DatabaseMigrator/Program.cs | 134 +++++++++++++++++++++ DatabaseMigrator/appsettings.json | 23 ++++ DatabaseMigrator/appsettings.override.json | 27 +++++ 7 files changed, 357 insertions(+) create mode 100644 DatabaseMigrator/Configuration.cs create mode 100644 DatabaseMigrator/DatabaseMigrator.csproj create mode 100644 DatabaseMigrator/Executor.cs create mode 100644 DatabaseMigrator/Program.cs create mode 100644 DatabaseMigrator/appsettings.json create mode 100644 DatabaseMigrator/appsettings.override.json diff --git a/Backbone.sln b/Backbone.sln index af7c799340..cb033932ec 100644 --- a/Backbone.sln +++ b/Backbone.sln @@ -287,6 +287,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Files.Infrastructure.Tests" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Messages.Domain.Tests", "Modules\Messages\test\Messages.Domain.Tests\Messages.Domain.Tests.csproj", "{83A86879-670B-4F22-8835-EE1D0AB49AA9}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DatabaseMigrator", "DatabaseMigrator\DatabaseMigrator.csproj", "{143EAC38-A2D0-4F74-88F1-3EF27413A8C6}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -697,6 +699,10 @@ Global {83A86879-670B-4F22-8835-EE1D0AB49AA9}.Debug|Any CPU.Build.0 = Debug|Any CPU {83A86879-670B-4F22-8835-EE1D0AB49AA9}.Release|Any CPU.ActiveCfg = Release|Any CPU {83A86879-670B-4F22-8835-EE1D0AB49AA9}.Release|Any CPU.Build.0 = Release|Any CPU + {143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/DatabaseMigrator/Configuration.cs b/DatabaseMigrator/Configuration.cs new file mode 100644 index 0000000000..5075043b4d --- /dev/null +++ b/DatabaseMigrator/Configuration.cs @@ -0,0 +1,24 @@ +using System.ComponentModel.DataAnnotations; + +namespace Backbone.DatabaseMigrator; + +public class Configuration +{ + [Required] + public InfrastructureConfiguration Infrastructure { get; set; } = null!; +} + +public class InfrastructureConfiguration +{ + [Required] + public SqlDatabaseConfiguration SqlDatabase { get; set; } = null!; +} + +public class SqlDatabaseConfiguration +{ + [Required] + public string Provider { get; set; } = null!; + + [Required] + public string ConnectionString { get; set; } = null!; +} diff --git a/DatabaseMigrator/DatabaseMigrator.csproj b/DatabaseMigrator/DatabaseMigrator.csproj new file mode 100644 index 0000000000..0374b3f016 --- /dev/null +++ b/DatabaseMigrator/DatabaseMigrator.csproj @@ -0,0 +1,66 @@ + + + + Exe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + PreserveNewest + + + diff --git a/DatabaseMigrator/Executor.cs b/DatabaseMigrator/Executor.cs new file mode 100644 index 0000000000..e36528aa8d --- /dev/null +++ b/DatabaseMigrator/Executor.cs @@ -0,0 +1,77 @@ +using System.Data.SqlClient; +using Backbone.Modules.Challenges.Infrastructure.Persistence.Database; +using Backbone.Modules.Devices.Infrastructure.Persistence.Database; +using Backbone.Modules.Files.Infrastructure.Persistence.Database; +using Backbone.Modules.Messages.Infrastructure.Persistence.Database; +using Backbone.Modules.Quotas.Infrastructure.Persistence.Database; +using Backbone.Modules.Relationships.Infrastructure.Persistence.Database; +using Backbone.Modules.Synchronization.Infrastructure.Persistence.Database; +using Backbone.Modules.Tokens.Infrastructure.Persistence.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Npgsql; +using Polly; + +namespace Backbone.DatabaseMigrator; + +public class Executor +{ + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public Executor(IServiceProvider serviceProvider, ILogger logger) + { + _serviceProvider = serviceProvider; + _logger = logger; + } + + public async Task Execute() + { + _logger.LogInformation("Starting migrations..."); + + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); + + _logger.LogInformation("Migrations successfully applied"); + } + + private async Task MigrateDbContext(IServiceProvider serviceProvider, string? targetMigration = null) where TContext : DbContext + { + using var scope = serviceProvider.CreateScope(); + var services = scope.ServiceProvider; + var logger = services.GetRequiredService>(); + var context = services.GetRequiredService(); + + try + { + logger.LogInformation("Migrating database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + + var retry = Policy.Handle().Or() + .WaitAndRetry([ + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(15) + ]); + + var migrator = context.GetInfrastructure().GetRequiredService(); + + await retry.Execute(() => migrator.MigrateAsync(targetMigration)); + + logger.LogInformation("Migrated database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + } + catch (Exception ex) + { + logger.LogError(ex, "An error occurred while migrating the database associated with context {context} to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + throw; + } + } +} diff --git a/DatabaseMigrator/Program.cs b/DatabaseMigrator/Program.cs new file mode 100644 index 0000000000..c1c5d07a50 --- /dev/null +++ b/DatabaseMigrator/Program.cs @@ -0,0 +1,134 @@ +using System.Reflection; +using Autofac.Extensions.DependencyInjection; +using Backbone.DatabaseMigrator; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Serilog; +using Serilog.Exceptions; +using Serilog.Exceptions.Core; +using Serilog.Exceptions.EntityFrameworkCore.Destructurers; +using Serilog.Settings.Configuration; +using IServiceCollectionExtensions = Backbone.Modules.Challenges.Infrastructure.Persistence.IServiceCollectionExtensions; + +Log.Logger = new LoggerConfiguration() + .WriteTo.Console() + .CreateBootstrapLogger(); + +try +{ + var app = CreateHostBuilder(args).Build(); + + var executor = app.Services.GetRequiredService(); + + await executor.Execute(); + + return 0; +} +catch (Exception ex) +{ + Log.Fatal(ex, "Host terminated unexpectedly"); + return 1; +} +finally +{ + await Log.CloseAndFlushAsync(); +} + + +static IHostBuilder CreateHostBuilder(string[] args) +{ + return Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration((hostContext, configuration) => + { + configuration.Sources.Clear(); + var env = hostContext.HostingEnvironment; + + configuration + .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) + .AddJsonFile("appsettings.override.json", optional: true, reloadOnChange: true); + + if (env.IsDevelopment()) + { + var appAssembly = Assembly.Load(new AssemblyName(env.ApplicationName)); + configuration.AddUserSecrets(appAssembly, optional: true); + } + + configuration.AddEnvironmentVariables(); + configuration.AddCommandLine(args); + }) + .ConfigureServices((hostContext, services) => + { + var configuration = hostContext.Configuration; + + services.ConfigureAndValidate(configuration.Bind); + var parsedConfiguration = services.BuildServiceProvider().GetRequiredService>().Value; + + services.AddSingleton(); + + # region Add db contexts + + IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Devices.Infrastructure.Persistence.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.ConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Files.Infrastructure.Persistence.Database.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Messages.Infrastructure.Persistence.Database.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Quotas.Infrastructure.Persistence.Database.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Relationships.Infrastructure.Persistence.Database.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Synchronization.Infrastructure.Persistence.Database.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + Backbone.Modules.Tokens.Infrastructure.Persistence.Database.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + + #endregion + }) + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .UseSerilog((context, configuration) => configuration + .ReadFrom.Configuration(context.Configuration, new ConfigurationReaderOptions { SectionName = "Logging" }) + .Enrich.WithDemystifiedStackTraces() + .Enrich.FromLogContext() + .Enrich.WithProperty("service", "databasemigrator") + .Enrich.WithExceptionDetails(new DestructuringOptionsBuilder() + .WithDefaultDestructurers() + .WithDestructurers(new[] { new DbUpdateExceptionDestructurer() }) + ) + ); +} diff --git a/DatabaseMigrator/appsettings.json b/DatabaseMigrator/appsettings.json new file mode 100644 index 0000000000..fe10b60a13 --- /dev/null +++ b/DatabaseMigrator/appsettings.json @@ -0,0 +1,23 @@ +{ + "Logging": { + "MinimumLevel": { + "Default": "Warning", + "Override": { + "Backbone": "Information", + "Enmeshed": "Information", + "ConsumerApi": "Information", + + "Microsoft": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "WriteTo": { + "Console": { + "Name": "Console", + "Args": { + "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact" + } + } + } + } +} diff --git a/DatabaseMigrator/appsettings.override.json b/DatabaseMigrator/appsettings.override.json new file mode 100644 index 0000000000..a822316508 --- /dev/null +++ b/DatabaseMigrator/appsettings.override.json @@ -0,0 +1,27 @@ +{ + "Infrastructure": { + "SqlDatabase": { + "Provider": "Postgres", + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + } + }, + "Logging": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Backbone": "Debug" + } + }, + "WriteTo": { + "Seq": { + "Name": "Seq", + "Args": { + "ServerUrl": "http://localhost:5341" + } + }, + "Debug": { + "Name": "Debug" + } + } + } +} From a451f72f793cdae3bdc235d9c2ab9d4da57021c6 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 12:33:21 +0200 Subject: [PATCH 03/49] feat: add RunMigration config property --- AdminApi/src/AdminApi/Program.cs | 2 +- AdminApi/src/AdminApi/appsettings.override.json | 1 + ConsumerApi/Program.cs | 2 +- ConsumerApi/appsettings.override.json | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/AdminApi/src/AdminApi/Program.cs b/AdminApi/src/AdminApi/Program.cs index d11171b2df..b6f8b65098 100644 --- a/AdminApi/src/AdminApi/Program.cs +++ b/AdminApi/src/AdminApi/Program.cs @@ -80,7 +80,7 @@ static WebApplication CreateApp(string[] args) var app = builder.Build(); Configure(app); - if (app.Environment.IsLocal() || app.Environment.IsDevelopment()) + if (app.Configuration.GetValue("RunMigrations")) app.MigrateDbContext(); return app; diff --git a/AdminApi/src/AdminApi/appsettings.override.json b/AdminApi/src/AdminApi/appsettings.override.json index 655cbc153c..fd5b87208e 100644 --- a/AdminApi/src/AdminApi/appsettings.override.json +++ b/AdminApi/src/AdminApi/appsettings.override.json @@ -1,4 +1,5 @@ { + "RunMigrations": true, "Authentication": { "ApiKey": "test" }, diff --git a/ConsumerApi/Program.cs b/ConsumerApi/Program.cs index 69b281e64f..f08a3b9f97 100644 --- a/ConsumerApi/Program.cs +++ b/ConsumerApi/Program.cs @@ -99,7 +99,7 @@ static WebApplication CreateApp(string[] args) var app = builder.Build(); Configure(app); - if (app.Environment.IsLocal() || app.Environment.IsDevelopment()) + if (app.Configuration.GetValue("RunMigrations")) { app .MigrateDbContext() diff --git a/ConsumerApi/appsettings.override.json b/ConsumerApi/appsettings.override.json index 367cdb9cc1..8dcf4d0941 100644 --- a/ConsumerApi/appsettings.override.json +++ b/ConsumerApi/appsettings.override.json @@ -1,4 +1,5 @@ { + "RunMigrations": true, "Authentication": { "JwtSigningCertificate": "", // can be empty to use an auto generated development certificate; however, in production scenarios this should definitely be set! Vertical scaling will not work without a fix signing certificate, because each instance would generate its own certificate "JwtLifetimeInSeconds": 300 From 67e9b711b6aa0ac21f5563c104362d412fbc8f5e Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 12:33:50 +0200 Subject: [PATCH 04/49] chore: add Rider run config --- .run/Backbone.run.xml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .run/Backbone.run.xml diff --git a/.run/Backbone.run.xml b/.run/Backbone.run.xml new file mode 100644 index 0000000000..9d3047566d --- /dev/null +++ b/.run/Backbone.run.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + From 5dc034bcb7fadfd8553d0575aa3938d39133580d Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 13:07:09 +0200 Subject: [PATCH 05/49] feat: add Dockerfile --- DatabaseMigrator/DatabaseMigrator.csproj | 15 +---- DatabaseMigrator/Dockerfile | 81 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 13 deletions(-) create mode 100644 DatabaseMigrator/Dockerfile diff --git a/DatabaseMigrator/DatabaseMigrator.csproj b/DatabaseMigrator/DatabaseMigrator.csproj index 0374b3f016..aa074417de 100644 --- a/DatabaseMigrator/DatabaseMigrator.csproj +++ b/DatabaseMigrator/DatabaseMigrator.csproj @@ -1,7 +1,7 @@ - + - Exe + Linux @@ -52,15 +52,4 @@ - - - - - PreserveNewest - - - - PreserveNewest - - diff --git a/DatabaseMigrator/Dockerfile b/DatabaseMigrator/Dockerfile new file mode 100644 index 0000000000..dba08e8166 --- /dev/null +++ b/DatabaseMigrator/Dockerfile @@ -0,0 +1,81 @@ +FROM mcr.microsoft.com/dotnet/runtime:8.0-alpine3.18 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +WORKDIR /src + +COPY ["Directory.Build.props", "."] +COPY ["Modules/Directory.Build.props", "Modules/"] +COPY ["DatabaseMigrator/DatabaseMigrator.csproj", "DatabaseMigrator/"] +COPY ["AdminApi/src/AdminApi.Infrastructure/AdminApi.Infrastructure.csproj", "AdminApi/src/AdminApi.Infrastructure/"] +COPY ["BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj", "BuildingBlocks/src/BuildingBlocks.Infrastructure/"] +COPY ["BuildingBlocks/src/BuildingBlocks.Application.Abstractions/BuildingBlocks.Application.Abstractions.csproj", "BuildingBlocks/src/BuildingBlocks.Application.Abstractions/"] +COPY ["BuildingBlocks/src/DevelopmentKit.Identity/DevelopmentKit.Identity.csproj", "BuildingBlocks/src/DevelopmentKit.Identity/"] +COPY ["BuildingBlocks/src/BuildingBlocks.Domain/BuildingBlocks.Domain.csproj", "BuildingBlocks/src/BuildingBlocks.Domain/"] +COPY ["BuildingBlocks/src/Tooling/Tooling.csproj", "BuildingBlocks/src/Tooling/"] +COPY ["Modules/Tokens/src/Tokens.Infrastructure.Database.SqlServer/Tokens.Infrastructure.Database.SqlServer.csproj", "Modules/Tokens/src/Tokens.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Tokens/src/Tokens.Infrastructure/Tokens.Infrastructure.csproj", "Modules/Tokens/src/Tokens.Infrastructure/"] +COPY ["Modules/Tokens/src/Tokens.Application/Tokens.Application.csproj", "Modules/Tokens/src/Tokens.Application/"] +COPY ["BuildingBlocks/src/BuildingBlocks.Application/BuildingBlocks.Application.csproj", "BuildingBlocks/src/BuildingBlocks.Application/"] +COPY ["Common/src/Common.Infrastructure/Common.Infrastructure.csproj", "Common/src/Common.Infrastructure/"] +COPY ["Modules/Tokens/src/Tokens.Domain/Tokens.Domain.csproj", "Modules/Tokens/src/Tokens.Domain/"] +COPY ["Modules/Tokens/src/Tokens.Infrastructure.Database.Postgres/Tokens.Infrastructure.Database.Postgres.csproj", "Modules/Tokens/src/Tokens.Infrastructure.Database.Postgres/"] +COPY ["Modules/Synchronization/src/Synchronization.Infrastructure.Database.SqlServer/Synchronization.Infrastructure.Database.SqlServer.csproj", "Modules/Synchronization/src/Synchronization.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Synchronization/src/Synchronization.Infrastructure/Synchronization.Infrastructure.csproj", "Modules/Synchronization/src/Synchronization.Infrastructure/"] +COPY ["Modules/Synchronization/src/Synchronization.Application/Synchronization.Application.csproj", "Modules/Synchronization/src/Synchronization.Application/"] +COPY ["Modules/Synchronization/src/Synchronization.Domain/Synchronization.Domain.csproj", "Modules/Synchronization/src/Synchronization.Domain/"] +COPY ["Modules/Synchronization/src/Synchronization.Infrastructure.Database.Postgres/Synchronization.Infrastructure.Database.Postgres.csproj", "Modules/Synchronization/src/Synchronization.Infrastructure.Database.Postgres/"] +COPY ["Modules/Challenges/src/Challenges.Infrastructure/Challenges.Infrastructure.csproj", "Modules/Challenges/src/Challenges.Infrastructure/"] +COPY ["Modules/Challenges/src/Challenges.Application/Challenges.Application.csproj", "Modules/Challenges/src/Challenges.Application/"] +COPY ["Modules/Challenges/src/Challenges.Domain/Challenges.Domain.csproj", "Modules/Challenges/src/Challenges.Domain/"] +COPY ["Modules/Devices/src/Devices.Infrastructure/Devices.Infrastructure.csproj", "Modules/Devices/src/Devices.Infrastructure/"] +COPY ["Modules/Devices/src/Devices.Application/Devices.Application.csproj", "Modules/Devices/src/Devices.Application/"] +COPY ["BuildingBlocks/src/Crypto/Crypto.csproj", "BuildingBlocks/src/Crypto/"] +COPY ["Modules/Devices/src/Devices.Domain/Devices.Domain.csproj", "Modules/Devices/src/Devices.Domain/"] +COPY ["Modules/Files/src/Files.Infrastructure/Files.Infrastructure.csproj", "Modules/Files/src/Files.Infrastructure/"] +COPY ["Modules/Files/src/Files.Application/Files.Application.csproj", "Modules/Files/src/Files.Application/"] +COPY ["Modules/Files/src/Files.Domain/Files.Domain.csproj", "Modules/Files/src/Files.Domain/"] +COPY ["Modules/Messages/src/Messages.Infrastructure/Messages.Infrastructure.csproj", "Modules/Messages/src/Messages.Infrastructure/"] +COPY ["Modules/Messages/src/Messages.Application/Messages.Application.csproj", "Modules/Messages/src/Messages.Application/"] +COPY ["Modules/Messages/src/Messages.Domain/Messages.Domain.csproj", "Modules/Messages/src/Messages.Domain/"] +COPY ["Modules/Quotas/src/Quotas.Infrastructure/Quotas.Infrastructure.csproj", "Modules/Quotas/src/Quotas.Infrastructure/"] +COPY ["Modules/Quotas/src/Quotas.Application/Quotas.Application.csproj", "Modules/Quotas/src/Quotas.Application/"] +COPY ["Modules/Quotas/src/Quotas.Domain/Quotas.Domain.csproj", "Modules/Quotas/src/Quotas.Domain/"] +COPY ["Modules/Relationships/src/Relationships.Infrastructure/Relationships.Infrastructure.csproj", "Modules/Relationships/src/Relationships.Infrastructure/"] +COPY ["Modules/Relationships/src/Relationships.Application/Relationships.Application.csproj", "Modules/Relationships/src/Relationships.Application/"] +COPY ["Modules/Relationships/src/Relationships.Common/Relationships.Common.csproj", "Modules/Relationships/src/Relationships.Common/"] +COPY ["Modules/Relationships/src/Relationships.Domain/Relationships.Domain.csproj", "Modules/Relationships/src/Relationships.Domain/"] +COPY ["Modules/Relationships/src/Relationships.Infrastructure.Database.SqlServer/Relationships.Infrastructure.Database.SqlServer.csproj", "Modules/Relationships/src/Relationships.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Relationships/src/Relationships.Infrastructure.Database.Postgres/Relationships.Infrastructure.Database.Postgres.csproj", "Modules/Relationships/src/Relationships.Infrastructure.Database.Postgres/"] +COPY ["Modules/Quotas/src/Quotas.Infrastructure.Database.SqlServer/Quotas.Infrastructure.Database.SqlServer.csproj", "Modules/Quotas/src/Quotas.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Quotas/src/Quotas.Infrastructure.Database.Postgres/Quotas.Infrastructure.Database.Postgres.csproj", "Modules/Quotas/src/Quotas.Infrastructure.Database.Postgres/"] +COPY ["Modules/Messages/src/Messages.Infrastructure.Database.SqlServer/Messages.Infrastructure.Database.SqlServer.csproj", "Modules/Messages/src/Messages.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Messages/src/Messages.Infrastructure.Database.Postgres/Messages.Infrastructure.Database.Postgres.csproj", "Modules/Messages/src/Messages.Infrastructure.Database.Postgres/"] +COPY ["Modules/Files/src/Files.Infrastructure.Database.SqlServer/Files.Infrastructure.Database.SqlServer.csproj", "Modules/Files/src/Files.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Files/src/Files.Infrastructure.Database.Postgres/Files.Infrastructure.Database.Postgres.csproj", "Modules/Files/src/Files.Infrastructure.Database.Postgres/"] +COPY ["Modules/Devices/src/Devices.Infrastructure.Database.SqlServer/Devices.Infrastructure.Database.SqlServer.csproj", "Modules/Devices/src/Devices.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Devices/src/Devices.Infrastructure.Database.Postgres/Devices.Infrastructure.Database.Postgres.csproj", "Modules/Devices/src/Devices.Infrastructure.Database.Postgres/"] +COPY ["Modules/Challenges/src/Challenges.Infrastructure.Database.SqlServer/Challenges.Infrastructure.Database.SqlServer.csproj", "Modules/Challenges/src/Challenges.Infrastructure.Database.SqlServer/"] +COPY ["Modules/Challenges/src/Challenges.Infrastructure.Database.Postgres/Challenges.Infrastructure.Database.Postgres.csproj", "Modules/Challenges/src/Challenges.Infrastructure.Database.Postgres/"] +COPY ["AdminApi/src/AdminApi.Infrastructure.Database.SqlServer/AdminApi.Infrastructure.Database.SqlServer.csproj", "AdminApi/src/AdminApi.Infrastructure.Database.SqlServer/"] +COPY ["AdminApi/src/AdminApi.Infrastructure.Database.Postgres/AdminApi.Infrastructure.Database.Postgres.csproj", "AdminApi/src/AdminApi.Infrastructure.Database.Postgres/"] + +RUN dotnet restore "DatabaseMigrator/DatabaseMigrator.csproj" + +COPY . . + +WORKDIR "/src/DatabaseMigrator" +RUN dotnet build "DatabaseMigrator.csproj" -c Release --output /app/build --no-restore + +FROM build AS publish + +RUN dotnet publish -c Release --output /app/publish --no-restore "DatabaseMigrator.csproj" + +FROM base AS final + +RUN apk add icu-libs +ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=0 + +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Backbone.DatabaseMigrator.dll"] From 4b87a6683741f765c69cf5569320c93ac5290deb Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 13:07:28 +0200 Subject: [PATCH 06/49] ci: add database migrator to docker-compose.test.yml --- .ci/docker-compose.test.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index ae822013af..824a993d98 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -11,12 +11,10 @@ services: ports: - "5000:8080" depends_on: - database: - condition: service_started + database-migrator: + condition: service_completed_successfully rabbitmq: condition: service_started - seed-database: - condition: service_completed_successfully azure-storage-emulator: condition: service_started configs: @@ -34,12 +32,10 @@ services: ports: - "5173:8080" depends_on: - database: - condition: service_started + database-migrator: + condition: service_completed_successfully rabbitmq: condition: service_started - consumer-api: - condition: service_healthy configs: - source: Config target: app/appsettings.override.json @@ -60,6 +56,19 @@ services: - source: Config target: app/appsettings.override.json + database-migrator: + container_name: database-migrator-test + build: + context: .. + dockerfile: DatabaseMigrator/Dockerfile + environment: + - ASPNETCORE_ENVIRONMENT=Development + depends_on: + database: + condition: service_started + seed-database: + condition: service_completed_successfully + ### infrastructure ### azure-storage-emulator: From b6f8c8bc5a1d2490437e515d1c00b3f31370b591 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 13:35:02 +0200 Subject: [PATCH 07/49] feat: also migrate AdminApiDbContext --- DatabaseMigrator/Dockerfile | 2 ++ DatabaseMigrator/Executor.cs | 4 ++-- DatabaseMigrator/Program.cs | 9 ++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/DatabaseMigrator/Dockerfile b/DatabaseMigrator/Dockerfile index dba08e8166..b794d644c5 100644 --- a/DatabaseMigrator/Dockerfile +++ b/DatabaseMigrator/Dockerfile @@ -8,6 +8,8 @@ COPY ["Directory.Build.props", "."] COPY ["Modules/Directory.Build.props", "Modules/"] COPY ["DatabaseMigrator/DatabaseMigrator.csproj", "DatabaseMigrator/"] COPY ["AdminApi/src/AdminApi.Infrastructure/AdminApi.Infrastructure.csproj", "AdminApi/src/AdminApi.Infrastructure/"] +COPY ["AdminApi/src/AdminApi.Infrastructure.Database.Postgres/AdminApi.Infrastructure.Database.Postgres.csproj", "AdminApi/src/AdminApi.Infrastructure.Database.Postgres/"] +COPY ["AdminApi/src/AdminApi.Infrastructure.Database.SqlServer/AdminApi.Infrastructure.Database.SqlServer.csproj", "AdminApi/src/AdminApi.Infrastructure.Database.SqlServer/"] COPY ["BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj", "BuildingBlocks/src/BuildingBlocks.Infrastructure/"] COPY ["BuildingBlocks/src/BuildingBlocks.Application.Abstractions/BuildingBlocks.Application.Abstractions.csproj", "BuildingBlocks/src/BuildingBlocks.Application.Abstractions/"] COPY ["BuildingBlocks/src/DevelopmentKit.Identity/DevelopmentKit.Identity.csproj", "BuildingBlocks/src/DevelopmentKit.Identity/"] diff --git a/DatabaseMigrator/Executor.cs b/DatabaseMigrator/Executor.cs index e36528aa8d..64beb79058 100644 --- a/DatabaseMigrator/Executor.cs +++ b/DatabaseMigrator/Executor.cs @@ -1,4 +1,5 @@ using System.Data.SqlClient; +using Backbone.AdminApi.Infrastructure.Persistence.Database; using Backbone.Modules.Challenges.Infrastructure.Persistence.Database; using Backbone.Modules.Devices.Infrastructure.Persistence.Database; using Backbone.Modules.Files.Infrastructure.Persistence.Database; @@ -10,8 +11,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Npgsql; using Polly; @@ -40,6 +39,7 @@ public async Task Execute() await MigrateDbContext(_serviceProvider); await MigrateDbContext(_serviceProvider); await MigrateDbContext(_serviceProvider); + await MigrateDbContext(_serviceProvider); _logger.LogInformation("Migrations successfully applied"); } diff --git a/DatabaseMigrator/Program.cs b/DatabaseMigrator/Program.cs index c1c5d07a50..56e7c5f7c5 100644 --- a/DatabaseMigrator/Program.cs +++ b/DatabaseMigrator/Program.cs @@ -1,9 +1,6 @@ using System.Reflection; using Autofac.Extensions.DependencyInjection; using Backbone.DatabaseMigrator; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; using Serilog; using Serilog.Exceptions; @@ -118,6 +115,12 @@ static IHostBuilder CreateHostBuilder(string[] args) options.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; }); + Backbone.AdminApi.Infrastructure.Persistence.IServiceCollectionExtensions.AddDatabase(services, options => + { + options.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider; + options.ConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString; + }); + #endregion }) .UseServiceProviderFactory(new AutofacServiceProviderFactory()) From 09e7a80cbfef6fe0811df0b19e927d31217585d7 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 13:35:39 +0200 Subject: [PATCH 08/49] chore: add rolling update configuration to eventhandler deployment --- helm/charts/eventhandler/templates/deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helm/charts/eventhandler/templates/deployment.yaml b/helm/charts/eventhandler/templates/deployment.yaml index 2ece25396b..fd1eba2df0 100644 --- a/helm/charts/eventhandler/templates/deployment.yaml +++ b/helm/charts/eventhandler/templates/deployment.yaml @@ -7,6 +7,11 @@ metadata: app: {{ include "global.name" . }} spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: {{ .Values.maxSurge }} + maxUnavailable: {{ .Values.maxUnavailable }} selector: matchLabels: app: {{ include "global.name" . }} From ad0c855e4c2192161bc9752b405ad894965cd70d Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 13:36:41 +0200 Subject: [PATCH 09/49] feat: add databasemigrator to helm chart --- DatabaseMigrator/Executor.cs | 6 +- helm/charts/databasemigrator/Chart.yaml | 9 +++ .../databasemigrator/templates/job.yaml | 65 +++++++++++++++++++ helm/values.yaml | 38 +++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 helm/charts/databasemigrator/Chart.yaml create mode 100644 helm/charts/databasemigrator/templates/job.yaml diff --git a/DatabaseMigrator/Executor.cs b/DatabaseMigrator/Executor.cs index 64beb79058..cf3d8bd3d3 100644 --- a/DatabaseMigrator/Executor.cs +++ b/DatabaseMigrator/Executor.cs @@ -31,6 +31,10 @@ public async Task Execute() { _logger.LogInformation("Starting migrations..."); + // vor Migrationen merken, welcher DB Context auf welcher Migration war + // try catch um alle Migrationen + // im catch Rollback zu gemerkter Migration + await MigrateDbContext(_serviceProvider); await MigrateDbContext(_serviceProvider); await MigrateDbContext(_serviceProvider); @@ -63,7 +67,7 @@ private async Task MigrateDbContext(IServiceProvider serviceProvider, ]); var migrator = context.GetInfrastructure().GetRequiredService(); - + await retry.Execute(() => migrator.MigrateAsync(targetMigration)); logger.LogInformation("Migrated database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); diff --git a/helm/charts/databasemigrator/Chart.yaml b/helm/charts/databasemigrator/Chart.yaml new file mode 100644 index 0000000000..089b44ce23 --- /dev/null +++ b/helm/charts/databasemigrator/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: databasemigrator +description: A Helm chart for Kubernetes + +type: application + +version: "1.0.0" + +appVersion: "v1.0.0" diff --git a/helm/charts/databasemigrator/templates/job.yaml b/helm/charts/databasemigrator/templates/job.yaml new file mode 100644 index 0000000000..04ae3d2a36 --- /dev/null +++ b/helm/charts/databasemigrator/templates/job.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "global.name" . }} + labels: + {{- include "global.labels" . | nindent 4 }} + app: {{ include "global.name" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade +spec: + selector: + matchLabels: + app: {{ include "global.name" . }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: {{ include "global.name" . }} + spec: + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + {{- with .Values.image.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: settings-override + configMap: + name: configuration + containers: + - name: {{ include "global.name" . }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + image: "{{ .Values.image.repository }}:{{- default .Chart.AppVersion .Values.image.tagOverride }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: settings-override + mountPath: /app/appsettings.override.json + subPath: appsettings.override.json + readOnly: true + env: + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index b60b707fce..73475b78df 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -236,6 +236,44 @@ eventhandler: targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: "" +databasemigrator: + nameOverride: "" + + image: + repository: "ghcr.io/nmshd/backbone-database-migrator" + tagOverride: "" + pullPolicy: "IfNotPresent" + imagePullSecrets: [] + + # resources - the resources for the Event Handler container (see https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources) + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "200m" + memory: "512Mi" + + # securityContext - securityContext for the Consumer API container (see https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-1) + securityContext: {} + + # podSecurityContext - securityContext for the pods deployed by the Deployment (see https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) + podSecurityContext: {} + + podAnnotations: {} + + # env - environment variables for the Event Handler container (see https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) + env: [] + + # the nodeSelector for the pods deployed by the Deployment (see https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) + nodeSelector: {} + + # the tolerations for the pods deployed by the Deployment (see https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) + tolerations: [] + + # the affinity for the pods deployed by the Deployment (see https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#NodeAffinity) + affinity: {} + admincli: nameOverride: "" From ae765622507a00c674dd84a9162f71c929455b17 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 16:40:05 +0200 Subject: [PATCH 10/49] ci: add database migrator to publish pipeline --- .ci/dbm/buildContainerImage.js | 11 ++++++++++ .github/workflows/publish.yml | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 .ci/dbm/buildContainerImage.js diff --git a/.ci/dbm/buildContainerImage.js b/.ci/dbm/buildContainerImage.js new file mode 100644 index 0000000000..f962b2ba8f --- /dev/null +++ b/.ci/dbm/buildContainerImage.js @@ -0,0 +1,11 @@ +#!/usr/bin/env node + +import { $ } from "zx"; +import { getRequiredEnvVar } from "../lib.js"; + +const tag = getRequiredEnvVar("TAG"); + +const platforms = process.env.PLATFORMS ?? "linux/amd64,linux/arm64"; +const push = process.env.PUSH === "1" ? ["--push", "--provenance=true", "--sbom=true"] : ""; + +await $`docker buildx build --file ./DatabaseMigrator/Dockerfile --tag ghcr.io/nmshd/backbone-database-migrator:${tag} --platform ${platforms} ${push} .`; diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eef6319349..adcca3bc9e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -161,6 +161,45 @@ jobs: TAG: ${{ steps.extract-version-from-git-tag.outputs.VERSION }} PUSH: 1 + publish-database-migrator: + name: Publish Database Migrator Container Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install script dependencies + run: npm install --prefix ./.ci + - name: Docker Login + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract version from git tag + id: extract-version-from-git-tag + run: echo "VERSION=$(./.ci/extractVersionFromGitTag.js)" >> $GITHUB_OUTPUT + env: + GIT_TAG: ${{ github.ref_name }} + - name: Log in to Docker Hub for accessing the cloud builder + uses: docker/login-action@v3 + with: + username: ${{ secrets.CLOUD_BUILDER_USERNAME }} + password: ${{ secrets.CLOUD_BUILDER_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: "lab:latest" + driver: cloud + endpoint: "jssoft/js-soft" + - name: Build and Push Container Image + run: ./.ci/dbm/buildContainerImage.js + env: + TAG: ${{ steps.extract-version-from-git-tag.outputs.VERSION }} + PUSH: 1 + publish-helm-chart: name: Publish Helm Chart runs-on: ubuntu-latest From 9103131169d35681e8137360975785bd981fbb7f Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Tue, 23 Apr 2024 16:41:08 +0200 Subject: [PATCH 11/49] ci: add database migrator to test pipeline --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe5a80906c..f6526c8821 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -221,6 +221,22 @@ jobs: TAG: test PLATFORMS: linux/amd64 + build-dbm-container-image: + name: Build Database Migrator Container Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install script dependencies + run: npm install --prefix ./.ci + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build image + run: ./.ci/dbm/buildContainerImage.js + env: + TAG: test + PLATFORMS: linux/amd64 + build-aui-container-image: name: Build Admin UI Container Image runs-on: ubuntu-latest From 99d93e00addbe794f558e6c8d6db110d96f1f7f4 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 10:42:16 +0200 Subject: [PATCH 12/49] feat: implement rollback behavior --- DatabaseMigrator/Executor.cs | 130 ++++++++++++++++++++++++++--------- 1 file changed, 98 insertions(+), 32 deletions(-) diff --git a/DatabaseMigrator/Executor.cs b/DatabaseMigrator/Executor.cs index cf3d8bd3d3..322e9bab42 100644 --- a/DatabaseMigrator/Executor.cs +++ b/DatabaseMigrator/Executor.cs @@ -21,6 +21,8 @@ public class Executor private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; + private readonly Stack<(Type dbContextType, string? migrationBeforeChanges)> _migratedDbContexts = new(); + public Executor(IServiceProvider serviceProvider, ILogger logger) { _serviceProvider = serviceProvider; @@ -31,51 +33,115 @@ public async Task Execute() { _logger.LogInformation("Starting migrations..."); - // vor Migrationen merken, welcher DB Context auf welcher Migration war - // try catch um alle Migrationen - // im catch Rollback zu gemerkter Migration - - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); - await MigrateDbContext(_serviceProvider); + try + { + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + await MigrateDbContext(); + } + catch (Exception ex) + { + _logger.LogCritical(ex, "An error occurred while migrating the database. Rolling back already migrated DbContexts..."); + await RollbackAlreadyMigratedDbContexts(); + Environment.Exit(1); + } _logger.LogInformation("Migrations successfully applied"); } - private async Task MigrateDbContext(IServiceProvider serviceProvider, string? targetMigration = null) where TContext : DbContext + private async Task MigrateDbContext(string? targetMigration = null) where TContext : DbContext { - using var scope = serviceProvider.CreateScope(); - var services = scope.ServiceProvider; - var logger = services.GetRequiredService>(); - var context = services.GetRequiredService(); + using var scope = _serviceProvider.CreateScope(); + var logger = scope.ServiceProvider.GetRequiredService>(); + var context = scope.ServiceProvider.GetRequiredService(); - try - { - logger.LogInformation("Migrating database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + if (targetMigration == null) + logger.LogInformation("Migrating database associated with context '{context}' to the latest migration...", typeof(TContext).Name); + else + logger.LogInformation("Migrating database associated with context '{context}' to target migration '{targetMigration}'...", typeof(TContext).Name, targetMigration); + + var retry = Policy.Handle().Or() + .WaitAndRetry([ + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(15) + ]); - var retry = Policy.Handle().Or() - .WaitAndRetry([ - TimeSpan.FromSeconds(5), - TimeSpan.FromSeconds(10), - TimeSpan.FromSeconds(15) - ]); + var pendingMigrations = await context.Database.GetPendingMigrationsAsync(); + if (!pendingMigrations.Any()) + return; - var migrator = context.GetInfrastructure().GetRequiredService(); - - await retry.Execute(() => migrator.MigrateAsync(targetMigration)); + var migrationBeforeChanges = await context.GetLastAppliedMigration(); - logger.LogInformation("Migrated database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + try + { + await retry.Execute(() => context.MigrateTo(targetMigration)); } catch (Exception ex) { - logger.LogError(ex, "An error occurred while migrating the database associated with context {context} to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + logger.LogCritical(ex, "An error occurred while migrating the database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, + targetMigration); throw; } + finally + { + _migratedDbContexts.Push((context.GetType(), migrationBeforeChanges)); + } + + if (targetMigration == null) + logger.LogInformation("Successfully migrated database associated with context '{context}' to the latest migration.", typeof(TContext).Name); + else + logger.LogInformation("Successfully migrated database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + } + + private async Task RollbackAlreadyMigratedDbContexts() + { + while (_migratedDbContexts.Count != 0) + { + var (contextType, migrationBeforeChanges) = _migratedDbContexts.Peek(); + var scope = _serviceProvider.CreateScope(); + var context = (DbContext)scope.ServiceProvider.GetRequiredService(contextType); + + // if the migration before changes is null, this means that it was the first migration; EF Core doesn't allow unapplying the first migration. But it's not problem because + // if there is only one migration, it means that in case of a rollback of the application version, there will be nothing that needs the tables from the first migration. + if (migrationBeforeChanges == null) + continue; + + try + { + await context.MigrateTo(migrationBeforeChanges); + _migratedDbContexts.Pop(); + } + catch (Exception ex) + { + var remainingDbContexts = string.Join(", ", _migratedDbContexts.Select(c => c.dbContextType.Name)); + + _logger.LogCritical(ex, + "There was an error while rolling back the migration of context '{context}' to migration '{migrationBeforeChanges}'. The following DbContexts couldn't be rolled back: {dbContexts}", + context.GetType().Name, migrationBeforeChanges, remainingDbContexts); + } + } + } +} + +file static class DbContextExtensions +{ + public static async Task MigrateTo(this DbContext dbContext, string? targetMigration = null) + { + var migrator = dbContext.GetInfrastructure().GetRequiredService(); + await migrator.MigrateAsync(targetMigration); + } + + public static async Task GetLastAppliedMigration(this DbContext dbContext) + { + var stateBeforeMigration = await dbContext.Database.GetAppliedMigrationsAsync(); + var migrationBeforeChanges = stateBeforeMigration.LastOrDefault(); + return migrationBeforeChanges; } } From b872739add8a9c6a5bb1946f38c33023341d0cf5 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 10:52:04 +0200 Subject: [PATCH 13/49] ci: make buildContainerImage script executable --- .ci/dbm/buildContainerImage.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .ci/dbm/buildContainerImage.js diff --git a/.ci/dbm/buildContainerImage.js b/.ci/dbm/buildContainerImage.js old mode 100644 new mode 100755 From 634e20516ef9d8e4c6e882cc6d8b80860c4e0263 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 10:53:50 +0200 Subject: [PATCH 14/49] ci: map config into database migrator --- .ci/docker-compose.test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index 824a993d98..fb99d0833a 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -68,6 +68,9 @@ services: condition: service_started seed-database: condition: service_completed_successfully + configs: + - source: Config + target: app/appsettings.override.json ### infrastructure ### From 960ddcca210a5b4bf7d73d04d40541e6108cfb14 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 12:24:58 +0200 Subject: [PATCH 15/49] feat: update setup-postgres.sql --- setup-db/setup-postgres.sql | 40 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/setup-db/setup-postgres.sql b/setup-db/setup-postgres.sql index 3a7f87ff03..9e8e8894b8 100644 --- a/setup-db/setup-postgres.sql +++ b/setup-db/setup-postgres.sql @@ -130,17 +130,25 @@ $$; /*+++++++++++++++++++++++++++++++++++++++++++++++++ Authorizations +++++++++++++++++++++++++++++++++++++++++++++++++*/ -/*GRANT CREATE ON SCHEMA Challenges, Devices, Messages, Synchronization, Tokens, Relationships, Files TO challenges, devices, messages, synchronization, tokens, relationships, files, Quotas; -GRANT CREATE ON SCHEMA Relationships TO relationships;*/ - -REVOKE USAGE, CREATE ON SCHEMA "Challenges" FROM synchronization, devices, messages, tokens, relationships, files, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Synchronization" FROM challenges, devices, messages, tokens, relationships, files, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Messages" FROM challenges, synchronization, devices, tokens, relationships, files, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Devices" FROM challenges, synchronization, messages, tokens, relationships, files, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Tokens" FROM challenges, synchronization, devices, messages, relationships, files, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Relationships" FROM challenges, synchronization, devices, messages, tokens, files, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Files" FROM challenges, synchronization, devices, messages, tokens, relationships, quotas; -REVOKE USAGE, CREATE ON SCHEMA "Quotas" FROM challenges, synchronization, devices, messages, tokens, relationships, files; +GRANT USAGE ON SCHEMA "Challenges" TO challenges; +GRANT USAGE ON SCHEMA "Devices" TO devices; +GRANT USAGE ON SCHEMA "Files" TO files; +GRANT USAGE ON SCHEMA "Messages" TO messages; +GRANT USAGE ON SCHEMA "Quotas" TO quotas; +GRANT USAGE ON SCHEMA "Relationships" TO relationships; +GRANT USAGE ON SCHEMA "Synchronization" TO synchronization; +GRANT USAGE ON SCHEMA "Tokens" TO tokens; +GRANT USAGE ON SCHEMA "AdminUi" TO "adminUi"; + +ALTER DEFAULT PRIVILEGES IN SCHEMA "Challenges" GRANT ALL ON TABLES TO challenges; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Devices" GRANT ALL ON TABLES TO devices; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Files" GRANT ALL ON TABLES TO files; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Messages" GRANT ALL ON TABLES TO messages; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Quotas" GRANT ALL ON TABLES TO quotas; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Relationships" GRANT ALL ON TABLES TO relationships; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Synchronization" GRANT ALL ON TABLES TO synchronization; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Tokens" GRANT ALL ON TABLES TO tokens; +ALTER DEFAULT PRIVILEGES IN SCHEMA "AdminUi" GRANT ALL ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Relationships" TO messages; GRANT SELECT, REFERENCES, TRIGGER, TRUNCATE ON ALL TABLES IN SCHEMA "Relationships" TO messages; @@ -282,13 +290,3 @@ GRANT relationships TO "nmshdAdmin"; GRANT files TO "nmshdAdmin"; GRANT quotas TO "nmshdAdmin"; GRANT "adminUi" TO "nmshdAdmin"; - -ALTER SCHEMA "Challenges" OWNER TO challenges; -ALTER SCHEMA "Devices" OWNER TO devices; -ALTER SCHEMA "Messages" OWNER TO messages; -ALTER SCHEMA "Synchronization" OWNER TO synchronization; -ALTER SCHEMA "Tokens" OWNER TO tokens; -ALTER SCHEMA "Relationships" OWNER TO relationships; -ALTER SCHEMA "Files" OWNER TO files; -ALTER SCHEMA "Quotas" OWNER TO quotas; -ALTER SCHEMA "AdminUi" OWNER TO "adminUi"; From 11ed61646af1f8929ebbafb96e5dce3f8cc9234d Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 12:25:14 +0200 Subject: [PATCH 16/49] chore: remove unused using directive --- .../src/BuildingBlocks.API/Extensions/HostExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/BuildingBlocks/src/BuildingBlocks.API/Extensions/HostExtensions.cs b/BuildingBlocks/src/BuildingBlocks.API/Extensions/HostExtensions.cs index 896869d690..87b0c4d78e 100644 --- a/BuildingBlocks/src/BuildingBlocks.API/Extensions/HostExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.API/Extensions/HostExtensions.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Logging; using Npgsql; using Polly; -using Polly.Retry; namespace Backbone.BuildingBlocks.API.Extensions; From 8481df4f8a0008ffe6a8f0d51921095bf970f5b0 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 12:25:40 +0200 Subject: [PATCH 17/49] refactor: change structure of files in helm folder --- helm/charts/databasemigrator/Chart.yaml | 9 ------- .../admincli/{templates => }/deployment.yaml | 0 .../adminui/{templates => }/deployment.yaml | 0 .../adminui/{templates => }/hpa.yaml | 0 .../adminui/{templates => }/service.yaml | 0 .../{templates => }/backendconfig.yaml | 0 .../{templates => }/deployment.yaml | 0 .../consumerapi/{templates => }/hpa.yaml | 0 .../{templates => }/httproute.yaml | 0 .../consumerapi/{templates => }/ingress.yaml | 0 .../consumerapi/{templates => }/service.yaml | 0 .../databasemigrator}/job.yaml | 26 +++++++++---------- .../{templates => }/deployment.yaml | 0 13 files changed, 13 insertions(+), 22 deletions(-) delete mode 100644 helm/charts/databasemigrator/Chart.yaml rename helm/templates/admincli/{templates => }/deployment.yaml (100%) rename helm/templates/adminui/{templates => }/deployment.yaml (100%) rename helm/templates/adminui/{templates => }/hpa.yaml (100%) rename helm/templates/adminui/{templates => }/service.yaml (100%) rename helm/templates/consumerapi/{templates => }/backendconfig.yaml (100%) rename helm/templates/consumerapi/{templates => }/deployment.yaml (100%) rename helm/templates/consumerapi/{templates => }/hpa.yaml (100%) rename helm/templates/consumerapi/{templates => }/httproute.yaml (100%) rename helm/templates/consumerapi/{templates => }/ingress.yaml (100%) rename helm/templates/consumerapi/{templates => }/service.yaml (100%) rename helm/{charts/databasemigrator/templates => templates/databasemigrator}/job.yaml (59%) rename helm/templates/eventhandler/{templates => }/deployment.yaml (100%) diff --git a/helm/charts/databasemigrator/Chart.yaml b/helm/charts/databasemigrator/Chart.yaml deleted file mode 100644 index 089b44ce23..0000000000 --- a/helm/charts/databasemigrator/Chart.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v2 -name: databasemigrator -description: A Helm chart for Kubernetes - -type: application - -version: "1.0.0" - -appVersion: "v1.0.0" diff --git a/helm/templates/admincli/templates/deployment.yaml b/helm/templates/admincli/deployment.yaml similarity index 100% rename from helm/templates/admincli/templates/deployment.yaml rename to helm/templates/admincli/deployment.yaml diff --git a/helm/templates/adminui/templates/deployment.yaml b/helm/templates/adminui/deployment.yaml similarity index 100% rename from helm/templates/adminui/templates/deployment.yaml rename to helm/templates/adminui/deployment.yaml diff --git a/helm/templates/adminui/templates/hpa.yaml b/helm/templates/adminui/hpa.yaml similarity index 100% rename from helm/templates/adminui/templates/hpa.yaml rename to helm/templates/adminui/hpa.yaml diff --git a/helm/templates/adminui/templates/service.yaml b/helm/templates/adminui/service.yaml similarity index 100% rename from helm/templates/adminui/templates/service.yaml rename to helm/templates/adminui/service.yaml diff --git a/helm/templates/consumerapi/templates/backendconfig.yaml b/helm/templates/consumerapi/backendconfig.yaml similarity index 100% rename from helm/templates/consumerapi/templates/backendconfig.yaml rename to helm/templates/consumerapi/backendconfig.yaml diff --git a/helm/templates/consumerapi/templates/deployment.yaml b/helm/templates/consumerapi/deployment.yaml similarity index 100% rename from helm/templates/consumerapi/templates/deployment.yaml rename to helm/templates/consumerapi/deployment.yaml diff --git a/helm/templates/consumerapi/templates/hpa.yaml b/helm/templates/consumerapi/hpa.yaml similarity index 100% rename from helm/templates/consumerapi/templates/hpa.yaml rename to helm/templates/consumerapi/hpa.yaml diff --git a/helm/templates/consumerapi/templates/httproute.yaml b/helm/templates/consumerapi/httproute.yaml similarity index 100% rename from helm/templates/consumerapi/templates/httproute.yaml rename to helm/templates/consumerapi/httproute.yaml diff --git a/helm/templates/consumerapi/templates/ingress.yaml b/helm/templates/consumerapi/ingress.yaml similarity index 100% rename from helm/templates/consumerapi/templates/ingress.yaml rename to helm/templates/consumerapi/ingress.yaml diff --git a/helm/templates/consumerapi/templates/service.yaml b/helm/templates/consumerapi/service.yaml similarity index 100% rename from helm/templates/consumerapi/templates/service.yaml rename to helm/templates/consumerapi/service.yaml diff --git a/helm/charts/databasemigrator/templates/job.yaml b/helm/templates/databasemigrator/job.yaml similarity index 59% rename from helm/charts/databasemigrator/templates/job.yaml rename to helm/templates/databasemigrator/job.yaml index 04ae3d2a36..fb8819e8b0 100644 --- a/helm/charts/databasemigrator/templates/job.yaml +++ b/helm/templates/databasemigrator/job.yaml @@ -13,18 +13,18 @@ spec: app: {{ include "global.name" . }} template: metadata: - {{- with .Values.podAnnotations }} + {{- with .Values.databasemigrator.podAnnotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} labels: app: {{ include "global.name" . }} spec: - {{- with .Values.podSecurityContext }} + {{- with .Values.databasemigrator.podSecurityContext }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.databasemigrator.podSecurityContext | nindent 8 }} {{- end }} - {{- with .Values.image.imagePullSecrets }} + {{- with .Values.databasemigrator.image.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} @@ -34,32 +34,32 @@ spec: name: configuration containers: - name: {{ include "global.name" . }} - {{- with .Values.securityContext }} + {{- with .Values.databasemigrator.securityContext }} securityContext: - {{- toYaml .Values.securityContext | nindent 8 }} + {{- toYaml .Values.databasemigrator.securityContext | nindent 8 }} {{- end }} resources: - {{- toYaml .Values.resources | nindent 12 }} - image: "{{ .Values.image.repository }}:{{- default .Chart.AppVersion .Values.image.tagOverride }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- toYaml .Values.databasemigrator.resources | nindent 12 }} + image: "{{ .Values.databasemigrator.image.repository }}:{{- default .Chart.AppVersion .Values.databasemigrator.image.tagOverride }}" + imagePullPolicy: {{ .Values.databasemigrator.image.pullPolicy }} volumeMounts: - name: settings-override mountPath: /app/appsettings.override.json subPath: appsettings.override.json readOnly: true env: - {{- with .Values.env }} + {{- with .Values.databasemigrator.env }} {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.nodeSelector }} + {{- with .Values.databasemigrator.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with .Values.databasemigrator.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with .Values.databasemigrator.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/eventhandler/templates/deployment.yaml b/helm/templates/eventhandler/deployment.yaml similarity index 100% rename from helm/templates/eventhandler/templates/deployment.yaml rename to helm/templates/eventhandler/deployment.yaml From e8a767e707bea6401a5cbc58c3f7664987130328 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 12:30:58 +0200 Subject: [PATCH 18/49] feat: remove CREATE TABLE permission from setup-sqlserver script --- setup-db/setup-sqlserver.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup-db/setup-sqlserver.sql b/setup-db/setup-sqlserver.sql index 3e75712272..8a312effe1 100644 --- a/setup-db/setup-sqlserver.sql +++ b/setup-db/setup-sqlserver.sql @@ -243,10 +243,6 @@ GO /*+++++++++++++++++++++++++++++++++++++++++++++++++ Authorizations +++++++++++++++++++++++++++++++++++++++++++++++++*/ PRINT 'Start changing authorizations' ; -GRANT CREATE TABLE TO challenges, devices, messages, synchronization, tokens, relationships, files, quotas, adminUi -GRANT CREATE FUNCTION TO relationships -GO - DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Challenges TO synchronization, devices, messages, tokens, relationships, files, quotas DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Synchronization TO challenges, devices, messages, tokens, relationships, files, quotas DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Messages TO challenges, synchronization, devices, tokens, relationships, files, quotas From 808b5f49367f1ab4ca792b21deef6edf2061335e Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 15:12:33 +0200 Subject: [PATCH 19/49] feat: don't set schema owners for sqlserver --- setup-db/setup-sqlserver.sql | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/setup-db/setup-sqlserver.sql b/setup-db/setup-sqlserver.sql index 8a312effe1..74ba423337 100644 --- a/setup-db/setup-sqlserver.sql +++ b/setup-db/setup-sqlserver.sql @@ -226,20 +226,6 @@ END GO -/*+++++++++++++++++++++++++++++++++++++++++++++++++ Schema Owners ++++++++++++++++++++++++++++++++++++++++++++++++++*/ -PRINT 'Start changing schema owners' ; -ALTER AUTHORIZATION ON SCHEMA::Challenges TO challenges -ALTER AUTHORIZATION ON SCHEMA::Devices TO devices -ALTER AUTHORIZATION ON SCHEMA::Messages TO messages -ALTER AUTHORIZATION ON SCHEMA::Synchronization TO synchronization -ALTER AUTHORIZATION ON SCHEMA::Tokens TO tokens -ALTER AUTHORIZATION ON SCHEMA::Relationships TO relationships -ALTER AUTHORIZATION ON SCHEMA::Files TO files -ALTER AUTHORIZATION ON SCHEMA::Quotas TO quotas -ALTER AUTHORIZATION ON SCHEMA::AdminUi TO adminUi -PRINT 'Finished changing schema owners' ; -GO - /*+++++++++++++++++++++++++++++++++++++++++++++++++ Authorizations +++++++++++++++++++++++++++++++++++++++++++++++++*/ PRINT 'Start changing authorizations' ; From cdb0729847658cb8ec6b1b0145c5979668a3a135 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 17:08:25 +0200 Subject: [PATCH 20/49] chore: use default user (sa/postgres) in all modules for local development --- .../src/AdminApi/appsettings.override.json | 12 +++---- ConsumerApi/appsettings.override.json | 32 +++++++++---------- DatabaseMigrator/appsettings.override.json | 3 +- .../appsettings.override.json | 32 +++++++++---------- docker-compose/docker-compose.yml | 2 +- 5 files changed, 41 insertions(+), 40 deletions(-) diff --git a/AdminApi/src/AdminApi/appsettings.override.json b/AdminApi/src/AdminApi/appsettings.override.json index fd5b87208e..fac7faae2a 100644 --- a/AdminApi/src/AdminApi/appsettings.override.json +++ b/AdminApi/src/AdminApi/appsettings.override.json @@ -25,8 +25,8 @@ }, "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=adminUi;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - //"ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=adminUi;Password=Passw0rd;TrustServerCertificate=True" + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" } }, "Modules": { @@ -41,8 +41,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=devices;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - //"ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" }, "PushNotifications": { "Provider": "Dummy" @@ -59,8 +59,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=quotas;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - //"ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=quotas;Password=Passw0rd;TrustServerCertificate=True" + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" } } } diff --git a/ConsumerApi/appsettings.override.json b/ConsumerApi/appsettings.override.json index 8dcf4d0941..057fcb9f23 100644 --- a/ConsumerApi/appsettings.override.json +++ b/ConsumerApi/appsettings.override.json @@ -30,8 +30,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=challenges;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=challenges;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -39,8 +39,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=quotas;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=quotas;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -48,8 +48,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=devices;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver }, "PushNotifications": { "Provider": "Dummy" @@ -60,8 +60,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=files;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=files;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver }, "BlobStorage": { "CloudProvider": "Azure", @@ -74,8 +74,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=messages;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=messages;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -83,8 +83,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=relationships;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=relationships;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -92,8 +92,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=synchronization;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=synchronization;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -101,8 +101,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=tokens;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=tokens;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } } diff --git a/DatabaseMigrator/appsettings.override.json b/DatabaseMigrator/appsettings.override.json index a822316508..f9955cd341 100644 --- a/DatabaseMigrator/appsettings.override.json +++ b/DatabaseMigrator/appsettings.override.json @@ -2,7 +2,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres, +// "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } }, "Logging": { diff --git a/EventHandlerService/src/EventHandlerService/appsettings.override.json b/EventHandlerService/src/EventHandlerService/appsettings.override.json index 40cf03c8d0..7bef826456 100644 --- a/EventHandlerService/src/EventHandlerService/appsettings.override.json +++ b/EventHandlerService/src/EventHandlerService/appsettings.override.json @@ -17,8 +17,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=challenges;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=challenges;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -26,8 +26,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=quotas;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=quotas;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -35,8 +35,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=devices;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver }, "PushNotifications": { "Provider": "Dummy" @@ -47,8 +47,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=files;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=files;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver }, "BlobStorage": { "CloudProvider": "Azure", @@ -61,8 +61,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=messages;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=messages;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -70,8 +70,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=relationships;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=relationships;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -79,8 +79,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=synchronization;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=synchronization;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } }, @@ -88,8 +88,8 @@ "Infrastructure": { "SqlDatabase": { "Provider": "Postgres", - "ConnectionString": "User ID=tokens;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres - // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=tokens;Password=Passw0rd;TrustServerCertificate=True" // sqlserver + "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres + // "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" // sqlserver } } } diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index b4bb341a09..7480bc5ac8 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -34,7 +34,7 @@ services: - postgres-volume:/var/lib/postgresql/data ports: - 5432:5432 - + rabbitmq: container_name: bkb-rabbitmq hostname: rabbitmq From 87d71e70219ad8e658ee0f2b84c45cb519d7da24 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 24 Apr 2024 17:26:03 +0200 Subject: [PATCH 21/49] ci: set environment variables with admin connection string for database-migrator --- .ci/docker-compose.test.postgres.yml | 5 +++++ .ci/docker-compose.test.sqlserver.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.ci/docker-compose.test.postgres.yml b/.ci/docker-compose.test.postgres.yml index 67ce0ffaf7..0c4a0045b6 100644 --- a/.ci/docker-compose.test.postgres.yml +++ b/.ci/docker-compose.test.postgres.yml @@ -35,6 +35,11 @@ services: Database__Provider: Postgres Database__ConnectionString: "Server=postgres;Database=enmeshed;User Id=devices;Password=Passw0rd;Port=5432" + database-migrator: + environment: + Infrastructure__SqlDatabase__Provider: Postgres + Infrastructure__SqlDatabase__ConnectionString: "Server=postgres;Database=enmeshed;User Id=postgres;Password=admin;Port=5432" + configs: Config: file: appsettings.override.postgres.json diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index b2d032b350..5b76f04a22 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -28,6 +28,11 @@ services: Database__Provider: SqlServer Database__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True" + database-migrator: + environment: + Infrastructure__SqlDatabase__Provider: SqlServer + Infrastructure__SqlDatabase__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True" + configs: Config: file: appsettings.override.sqlserver.json From a115a86ff5f3a223db49af2130f7194525c35220 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 07:58:40 +0200 Subject: [PATCH 22/49] fix: base dockerfile on aspnet instead of runtime --- DatabaseMigrator/DatabaseMigrator.csproj | 1 + DatabaseMigrator/Dockerfile | 2 +- .../src/EventHandlerService/EventHandlerService.csproj | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/DatabaseMigrator/DatabaseMigrator.csproj b/DatabaseMigrator/DatabaseMigrator.csproj index aa074417de..e35173675b 100644 --- a/DatabaseMigrator/DatabaseMigrator.csproj +++ b/DatabaseMigrator/DatabaseMigrator.csproj @@ -1,6 +1,7 @@  + Exe Linux diff --git a/DatabaseMigrator/Dockerfile b/DatabaseMigrator/Dockerfile index b794d644c5..c5a6dcddfb 100644 --- a/DatabaseMigrator/Dockerfile +++ b/DatabaseMigrator/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/runtime:8.0-alpine3.18 AS base +FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine3.18 AS base WORKDIR /app FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build diff --git a/EventHandlerService/src/EventHandlerService/EventHandlerService.csproj b/EventHandlerService/src/EventHandlerService/EventHandlerService.csproj index c511107aa1..43d6c8b872 100644 --- a/EventHandlerService/src/EventHandlerService/EventHandlerService.csproj +++ b/EventHandlerService/src/EventHandlerService/EventHandlerService.csproj @@ -1,7 +1,7 @@  - Exe + Exe Linux ..\..\.. From eba08d692df49253c37e58f22b29b09a6f81aefd Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 07:59:45 +0200 Subject: [PATCH 23/49] fix: add name for all resources to values --- helm/templates/admincli/deployment.yaml | 10 +++++----- helm/templates/adminui/deployment.yaml | 10 +++++----- helm/templates/adminui/hpa.yaml | 4 ++-- helm/templates/adminui/service.yaml | 8 ++++---- helm/templates/consumerapi/backendconfig.yaml | 2 +- helm/templates/consumerapi/deployment.yaml | 10 +++++----- helm/templates/consumerapi/hpa.yaml | 4 ++-- helm/templates/consumerapi/httproute.yaml | 4 ++-- helm/templates/consumerapi/ingress.yaml | 6 +++--- helm/templates/consumerapi/service.yaml | 8 ++++---- helm/templates/databasemigrator/job.yaml | 15 ++++++++------- helm/templates/eventhandler/deployment.yaml | 10 +++++----- helm/values.yaml | 10 +++++----- 13 files changed, 51 insertions(+), 50 deletions(-) diff --git a/helm/templates/admincli/deployment.yaml b/helm/templates/admincli/deployment.yaml index 5b872db345..b913d78906 100644 --- a/helm/templates/admincli/deployment.yaml +++ b/helm/templates/admincli/deployment.yaml @@ -1,15 +1,15 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "global.name" . }} + name: {{ .Values.admincli.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.admincli.name }} spec: replicas: 1 selector: matchLabels: - app: {{ include "global.name" . }} + app: {{ .Values.admincli.name }} template: metadata: {{- with .Values.admincli.podAnnotations }} @@ -17,7 +17,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app: {{ include "global.name" . }} + app: {{ .Values.admincli.name }} spec: {{- with .Values.admincli.podSecurityContext }} securityContext: @@ -28,7 +28,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} containers: - - name: {{ include "global.name" . }} + - name: {{ .Values.admincli.name }} {{- with .Values.admincli.securityContext }} securityContext: {{- toYaml .Values.admincli.securityContext | nindent 8 }} diff --git a/helm/templates/adminui/deployment.yaml b/helm/templates/adminui/deployment.yaml index 43aba0b988..40a64bb5f9 100644 --- a/helm/templates/adminui/deployment.yaml +++ b/helm/templates/adminui/deployment.yaml @@ -1,10 +1,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "global.name" . }} + name: {{ .Values.adminui.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.adminui.name }} spec: replicas: {{ .Values.adminui.replicas }} strategy: @@ -14,7 +14,7 @@ spec: maxUnavailable: {{ .Values.adminui.maxUnavailable }} selector: matchLabels: - app: {{ include "global.name" . }} + app: {{ .Values.adminui.name }} template: metadata: annotations: @@ -24,7 +24,7 @@ spec: {{- end }} labels: {{- include "global.labels" . | nindent 8 }} - app: {{ include "global.name" . }} + app: {{ .Values.adminui.name }} spec: {{- with .Values.global.serviceAccount.name}} serviceAccountName: {{ . }} @@ -46,7 +46,7 @@ spec: configMap: name: configuration containers: - - name: {{ include "global.name" . }} + - name: {{ .Values.adminui.name }} {{- with .Values.adminui.securityContext }} securityContext: {{- toYaml .Values.adminui.securityContext | nindent 8 }} diff --git a/helm/templates/adminui/hpa.yaml b/helm/templates/adminui/hpa.yaml index a58f4203c3..badb60579e 100644 --- a/helm/templates/adminui/hpa.yaml +++ b/helm/templates/adminui/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "global.name" . }} + name: {{ .Values.adminui.name }} labels: {{- include "global.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "global.name" . }} + name: {{ .Values.adminui.name }} minReplicas: {{ .Values.adminui.autoscaling.minReplicas }} maxReplicas: {{ .Values.adminui.autoscaling.maxReplicas }} metrics: diff --git a/helm/templates/adminui/service.yaml b/helm/templates/adminui/service.yaml index 4ef1b52b64..2bc84f5ef7 100644 --- a/helm/templates/adminui/service.yaml +++ b/helm/templates/adminui/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "global.name" . }} + name: {{ .Values.adminui.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.adminui.name }} {{- if eq .Values.global.provider "GoogleCloud" }} annotations: - cloud.google.com/backend-config: '{"default": "{{ include "global.name" . }}"}' + cloud.google.com/backend-config: '{"default": "{{ .Values.adminui.name }}"}' {{- end }} spec: type: {{ .Values.adminui.service.type }} @@ -15,4 +15,4 @@ spec: - port: 8080 targetPort: 8080 selector: - app: {{ include "global.name" . }} + app: {{ .Values.adminui.name }} diff --git a/helm/templates/consumerapi/backendconfig.yaml b/helm/templates/consumerapi/backendconfig.yaml index 05c998b615..fb416b9389 100644 --- a/helm/templates/consumerapi/backendconfig.yaml +++ b/helm/templates/consumerapi/backendconfig.yaml @@ -4,7 +4,7 @@ kind: BackendConfig metadata: labels: {{- include "global.labels" . | nindent 4 }} - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} spec: healthCheck: checkIntervalSec: {{ .Values.consumerapi.backendConfig.healthCheck.checkIntervalSec }} diff --git a/helm/templates/consumerapi/deployment.yaml b/helm/templates/consumerapi/deployment.yaml index eeb699c772..bdd1d2fd9f 100644 --- a/helm/templates/consumerapi/deployment.yaml +++ b/helm/templates/consumerapi/deployment.yaml @@ -1,10 +1,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.consumerapi.name }} spec: replicas: {{ .Values.consumerapi.replicas }} strategy: @@ -14,7 +14,7 @@ spec: maxUnavailable: {{ .Values.consumerapi.maxUnavailable }} selector: matchLabels: - app: {{ include "global.name" . }} + app: {{ .Values.consumerapi.name }} template: metadata: annotations: @@ -24,7 +24,7 @@ spec: {{- end }} labels: {{- include "global.labels" . | nindent 8 }} - app: {{ include "global.name" . }} + app: {{ .Values.consumerapi.name }} spec: {{- with .Values.global.serviceAccount.name}} serviceAccountName: {{ . }} @@ -46,7 +46,7 @@ spec: configMap: name: configuration containers: - - name: {{ include "global.name" . }} + - name: {{ .Values.consumerapi.name }} {{- with .Values.consumerapi.securityContext }} securityContext: {{- toYaml .Values.consumerapi.securityContext | nindent 8 }} diff --git a/helm/templates/consumerapi/hpa.yaml b/helm/templates/consumerapi/hpa.yaml index 816ad94b62..18c00417d9 100644 --- a/helm/templates/consumerapi/hpa.yaml +++ b/helm/templates/consumerapi/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} labels: {{- include "global.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} minReplicas: {{ .Values.consumerapi.autoscaling.minReplicas }} maxReplicas: {{ .Values.consumerapi.autoscaling.maxReplicas }} metrics: diff --git a/helm/templates/consumerapi/httproute.yaml b/helm/templates/consumerapi/httproute.yaml index c1c7a7c46d..d59b0da7c2 100644 --- a/helm/templates/consumerapi/httproute.yaml +++ b/helm/templates/consumerapi/httproute.yaml @@ -2,7 +2,7 @@ apiVersion: gateway.networking.k8s.io/v1beta1 kind: HTTPRoute metadata: - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} labels: {{- include "global.labels" . | nindent 4 }} spec: @@ -20,6 +20,6 @@ spec: type: PathPrefix value: / backendRefs: - - name: {{ include "global.name" . }} + - name: {{ .Values.consumerapi.name }} port: 8080 {{- end }} diff --git a/helm/templates/consumerapi/ingress.yaml b/helm/templates/consumerapi/ingress.yaml index ed39cf7cca..6df847853e 100644 --- a/helm/templates/consumerapi/ingress.yaml +++ b/helm/templates/consumerapi/ingress.yaml @@ -2,10 +2,10 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.consumerapi.name }} annotations: kubernetes.io/ingress.class: azure/application-gateway appgw.ingress.kubernetes.io/appgw-ssl-certificate: {{ .Values.consumerapi.ingress.certName }} @@ -20,7 +20,7 @@ spec: pathType: Prefix backend: service: - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} port: number: 8080 {{- end }} diff --git a/helm/templates/consumerapi/service.yaml b/helm/templates/consumerapi/service.yaml index b1de961c48..dcf5524d85 100644 --- a/helm/templates/consumerapi/service.yaml +++ b/helm/templates/consumerapi/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "global.name" . }} + name: {{ .Values.consumerapi.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.consumerapi.name }} {{- if eq .Values.global.provider "GoogleCloud" }} annotations: - cloud.google.com/backend-config: '{"default": "{{ include "global.name" . }}"}' + cloud.google.com/backend-config: '{"default": "{{ .Values.consumerapi.name }}"}' {{- end }} {{- if eq .Values.consumerapi.service.type "LoadBalancer" }} annotations: @@ -22,4 +22,4 @@ spec: - port: 8080 targetPort: 8080 selector: - app: {{ include "global.name" . }} + app: {{ .Values.consumerapi.name }} diff --git a/helm/templates/databasemigrator/job.yaml b/helm/templates/databasemigrator/job.yaml index fb8819e8b0..cef1bf16bf 100644 --- a/helm/templates/databasemigrator/job.yaml +++ b/helm/templates/databasemigrator/job.yaml @@ -1,16 +1,16 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ include "global.name" . }} + name: {{ .Values.databasemigrator.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.databasemigrator.name }} annotations: "helm.sh/hook": pre-install,pre-upgrade spec: - selector: - matchLabels: - app: {{ include "global.name" . }} + # selector: + # matchLabels: + # app: {{ .Values.databasemigrator.name }} template: metadata: {{- with .Values.databasemigrator.podAnnotations }} @@ -18,8 +18,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app: {{ include "global.name" . }} + app: {{ .Values.databasemigrator.name }} spec: + restartPolicy: Never {{- with .Values.databasemigrator.podSecurityContext }} securityContext: {{- toYaml .Values.databasemigrator.podSecurityContext | nindent 8 }} @@ -33,7 +34,7 @@ spec: configMap: name: configuration containers: - - name: {{ include "global.name" . }} + - name: {{ .Values.databasemigrator.name }} {{- with .Values.databasemigrator.securityContext }} securityContext: {{- toYaml .Values.databasemigrator.securityContext | nindent 8 }} diff --git a/helm/templates/eventhandler/deployment.yaml b/helm/templates/eventhandler/deployment.yaml index f1726d3430..2ede81542d 100644 --- a/helm/templates/eventhandler/deployment.yaml +++ b/helm/templates/eventhandler/deployment.yaml @@ -1,10 +1,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "global.name" . }} + name: {{ .Values.eventhandler.name }} labels: {{- include "global.labels" . | nindent 4 }} - app: {{ include "global.name" . }} + app: {{ .Values.eventhandler.name }} spec: replicas: 1 strategy: @@ -14,7 +14,7 @@ spec: maxUnavailable: {{ .Values.maxUnavailable }} selector: matchLabels: - app: {{ include "global.name" . }} + app: {{ .Values.eventhandler.name }} template: metadata: {{- with .Values.eventhandler.podAnnotations }} @@ -22,7 +22,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - app: {{ include "global.name" . }} + app: {{ .Values.eventhandler.name }} spec: {{- with .Values.eventhandler.podSecurityContext }} securityContext: @@ -37,7 +37,7 @@ spec: configMap: name: configuration containers: - - name: {{ include "global.name" . }} + - name: {{ .Values.eventhandler.name }} {{- with .Values.eventhandler.securityContext }} securityContext: {{- toYaml .Values.eventhandler.securityContext | nindent 8 }} diff --git a/helm/values.yaml b/helm/values.yaml index 73475b78df..f46b6d3b24 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,5 +1,5 @@ consumerapi: - nameOverride: "" + name: "consumerapi" replicas: 1 maxSurge: 2 maxUnavailable: 0 @@ -99,7 +99,7 @@ consumerapi: unhealthyThreshold: 2 adminui: - nameOverride: "" + name: "adminui" replicas: 1 maxSurge: 2 maxUnavailable: 0 @@ -189,7 +189,7 @@ adminui: unhealthyThreshold: 2 eventhandler: - nameOverride: "" + name: "eventhandler" replicas: 1 maxSurge: 2 maxUnavailable: 0 @@ -237,7 +237,7 @@ eventhandler: targetMemoryUtilizationPercentage: "" databasemigrator: - nameOverride: "" + name: "databasemigrator" image: repository: "ghcr.io/nmshd/backbone-database-migrator" @@ -275,7 +275,7 @@ databasemigrator: affinity: {} admincli: - nameOverride: "" + name: "admincli" configuration: Database: From 228a520eafc7f1a4b63b8a747560f3baebc5b449 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 08:59:14 +0200 Subject: [PATCH 24/49] ci: change dependencies in docker-compsoe files --- .ci/docker-compose.test.postgres.yml | 4 +--- .ci/docker-compose.test.sqlserver.yml | 5 ++--- .ci/docker-compose.test.yml | 19 +++++++++---------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.ci/docker-compose.test.postgres.yml b/.ci/docker-compose.test.postgres.yml index 0c4a0045b6..c451e62194 100644 --- a/.ci/docker-compose.test.postgres.yml +++ b/.ci/docker-compose.test.postgres.yml @@ -21,8 +21,6 @@ services: seed-database: image: postgres - volumes: - - ../setup-db:/app/setup-db environment: - PGPASSWORD=Passw0rd command: psql -h postgres -U postgres -d enmeshed -f /app/setup-db/setup-postgres.sql @@ -38,7 +36,7 @@ services: database-migrator: environment: Infrastructure__SqlDatabase__Provider: Postgres - Infrastructure__SqlDatabase__ConnectionString: "Server=postgres;Database=enmeshed;User Id=postgres;Password=admin;Port=5432" + Infrastructure__SqlDatabase__ConnectionString: "Server=postgres;Database=enmeshed;User Id=postgres;Password=Passw0rd;Port=5432" configs: Config: diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index 5b76f04a22..20aaa05273 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -17,11 +17,10 @@ services: seed-database: image: mcr.microsoft.com/mssql-tools - volumes: - - ../setup-db:/app/setup-db command: bash -c " sleep 20 && /opt/mssql-tools/bin/sqlcmd -S sqlserver -U SA -P Passw0rd -i /app/setup-db/setup-sqlserver.sql " depends_on: - - database + database: + condition: service_started seed-client: environment: diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index fb99d0833a..79de4be547 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -11,7 +11,7 @@ services: ports: - "5000:8080" depends_on: - database-migrator: + seed-client: condition: service_completed_successfully rabbitmq: condition: service_started @@ -48,8 +48,8 @@ services: environment: - ASPNETCORE_ENVIRONMENT=Development depends_on: - database: - condition: service_started + database-migrator: + condition: service_completed_successfully rabbitmq: condition: service_started configs: @@ -63,11 +63,6 @@ services: dockerfile: DatabaseMigrator/Dockerfile environment: - ASPNETCORE_ENVIRONMENT=Development - depends_on: - database: - condition: service_started - seed-database: - condition: service_completed_successfully configs: - source: Config target: app/appsettings.override.json @@ -91,14 +86,18 @@ services: ### seeds ### + seed-database: + volumes: + - ../setup-db:/app/setup-db + seed-client: container_name: seed-client-test build: context: .. dockerfile: Modules/Devices/src/Devices.AdminCli/Dockerfile depends_on: - consumer-api: - condition: service_healthy + database-migrator: + condition: service_completed_successfully command: backbone client create --clientId test --clientSecret test --defaultTier Basic networks: From f023331037ebe4041c909388106f9747912a0dd5 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 09:22:08 +0200 Subject: [PATCH 25/49] feat: add special configmap for database migrator --- helm/templates/databasemigrator/configmap.yaml | 12 ++++++++++++ helm/templates/databasemigrator/job.yaml | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 helm/templates/databasemigrator/configmap.yaml diff --git a/helm/templates/databasemigrator/configmap.yaml b/helm/templates/databasemigrator/configmap.yaml new file mode 100644 index 0000000000..1ab3cb282b --- /dev/null +++ b/helm/templates/databasemigrator/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: configuration-for-databasemigrator + labels: {{- include "global.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation +data: + appsettings.override.json: |- + {{- mustToPrettyJson .Values.global.configuration | toString | nindent 4 }} diff --git a/helm/templates/databasemigrator/job.yaml b/helm/templates/databasemigrator/job.yaml index cef1bf16bf..373ffefe78 100644 --- a/helm/templates/databasemigrator/job.yaml +++ b/helm/templates/databasemigrator/job.yaml @@ -7,6 +7,7 @@ metadata: app: {{ .Values.databasemigrator.name }} annotations: "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" spec: # selector: # matchLabels: @@ -32,7 +33,7 @@ spec: volumes: - name: settings-override configMap: - name: configuration + name: configuration-for-databasemigrator containers: - name: {{ .Values.databasemigrator.name }} {{- with .Values.databasemigrator.securityContext }} From dd755854f32e69fed98d372c1182950a47892a34 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 11:49:16 +0200 Subject: [PATCH 26/49] feat: improve logging for Executor --- DatabaseMigrator/Executor.cs | 155 ++++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 28 deletions(-) diff --git a/DatabaseMigrator/Executor.cs b/DatabaseMigrator/Executor.cs index 322e9bab42..dc63e7f3d3 100644 --- a/DatabaseMigrator/Executor.cs +++ b/DatabaseMigrator/Executor.cs @@ -13,6 +13,7 @@ using Microsoft.EntityFrameworkCore.Migrations; using Npgsql; using Polly; +using Polly.Retry; namespace Backbone.DatabaseMigrator; @@ -22,16 +23,24 @@ public class Executor private readonly ILogger _logger; private readonly Stack<(Type dbContextType, string? migrationBeforeChanges)> _migratedDbContexts = new(); + private readonly RetryPolicy _retryPolicy; public Executor(IServiceProvider serviceProvider, ILogger logger) { _serviceProvider = serviceProvider; _logger = logger; + + _retryPolicy = Policy.Handle().Or() + .WaitAndRetry([ + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(15) + ]); } public async Task Execute() { - _logger.LogInformation("Starting migrations..."); + _logger.StartApplyingMigrations(); try { @@ -47,31 +56,22 @@ public async Task Execute() } catch (Exception ex) { - _logger.LogCritical(ex, "An error occurred while migrating the database. Rolling back already migrated DbContexts..."); + _logger.ErrorWhileApplyingMigrations(ex); await RollbackAlreadyMigratedDbContexts(); Environment.Exit(1); } - _logger.LogInformation("Migrations successfully applied"); + _logger.SuccessfullyAppliedMigrations(); } private async Task MigrateDbContext(string? targetMigration = null) where TContext : DbContext { - using var scope = _serviceProvider.CreateScope(); - var logger = scope.ServiceProvider.GetRequiredService>(); - var context = scope.ServiceProvider.GetRequiredService(); + await using var context = _serviceProvider.GetDbContext(); if (targetMigration == null) - logger.LogInformation("Migrating database associated with context '{context}' to the latest migration...", typeof(TContext).Name); + _logger.MigratingDbContext(typeof(TContext).Name); else - logger.LogInformation("Migrating database associated with context '{context}' to target migration '{targetMigration}'...", typeof(TContext).Name, targetMigration); - - var retry = Policy.Handle().Or() - .WaitAndRetry([ - TimeSpan.FromSeconds(5), - TimeSpan.FromSeconds(10), - TimeSpan.FromSeconds(15) - ]); + _logger.MigratingDbContext(typeof(TContext).Name, targetMigration); var pendingMigrations = await context.Database.GetPendingMigrationsAsync(); if (!pendingMigrations.Any()) @@ -81,12 +81,14 @@ private async Task MigrateDbContext(string? targetMigration = null) wh try { - await retry.Execute(() => context.MigrateTo(targetMigration)); + await _retryPolicy.Execute(() => context.MigrateTo(targetMigration)); } catch (Exception ex) { - logger.LogCritical(ex, "An error occurred while migrating the database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, - targetMigration); + if (targetMigration == null) + _logger.ErrorWhileApplyingMigration(ex, typeof(TContext).Name); + else + _logger.ErrorWhileApplyingMigration(ex, typeof(TContext).Name, targetMigration); throw; } finally @@ -95,9 +97,9 @@ private async Task MigrateDbContext(string? targetMigration = null) wh } if (targetMigration == null) - logger.LogInformation("Successfully migrated database associated with context '{context}' to the latest migration.", typeof(TContext).Name); + _logger.SuccessfullyMigratedDbContext(typeof(TContext).Name); else - logger.LogInformation("Successfully migrated database associated with context '{context}' to target migration '{targetMigration}'", typeof(TContext).Name, targetMigration); + _logger.SuccessfullyMigratedDbContext(typeof(TContext).Name, targetMigration); } private async Task RollbackAlreadyMigratedDbContexts() @@ -105,32 +107,33 @@ private async Task RollbackAlreadyMigratedDbContexts() while (_migratedDbContexts.Count != 0) { var (contextType, migrationBeforeChanges) = _migratedDbContexts.Peek(); - var scope = _serviceProvider.CreateScope(); - var context = (DbContext)scope.ServiceProvider.GetRequiredService(contextType); - // if the migration before changes is null, this means that it was the first migration; EF Core doesn't allow unapplying the first migration. But it's not problem because + await using var context = _serviceProvider.GetDbContext(contextType); + + // if the migrationBeforeChanges is null, this means that it was the first migration; EF Core doesn't allow unapplying the first migration. But it's no problem because // if there is only one migration, it means that in case of a rollback of the application version, there will be nothing that needs the tables from the first migration. if (migrationBeforeChanges == null) continue; + _logger.RollingBackDbContext(contextType.Name, migrationBeforeChanges); + try { await context.MigrateTo(migrationBeforeChanges); - _migratedDbContexts.Pop(); } catch (Exception ex) { var remainingDbContexts = string.Join(", ", _migratedDbContexts.Select(c => c.dbContextType.Name)); - _logger.LogCritical(ex, - "There was an error while rolling back the migration of context '{context}' to migration '{migrationBeforeChanges}'. The following DbContexts couldn't be rolled back: {dbContexts}", - context.GetType().Name, migrationBeforeChanges, remainingDbContexts); + _logger.ErrorOnRollback(ex, context.GetType().Name, migrationBeforeChanges, remainingDbContexts); } + + _migratedDbContexts.Pop(); } } } -file static class DbContextExtensions +file static class Extensions { public static async Task MigrateTo(this DbContext dbContext, string? targetMigration = null) { @@ -144,4 +147,100 @@ public static async Task MigrateTo(this DbContext dbContext, string? targetMigra var migrationBeforeChanges = stateBeforeMigration.LastOrDefault(); return migrationBeforeChanges; } + + public static DbContext GetDbContext(this IServiceProvider serviceProvider) where T : DbContext + { + return serviceProvider.GetDbContext(typeof(T)); + } + + public static DbContext GetDbContext(this IServiceProvider serviceProvider, Type type) + { + var scope = serviceProvider.CreateScope(); + var context = (DbContext)scope.ServiceProvider.GetRequiredService(type); + return context; + } +} + +internal static partial class ExecutorLogs +{ + [LoggerMessage( + EventId = 561100, + EventName = "DatabaseMigrator.Executor.StartApplyingMigrations", + Level = LogLevel.Information, + Message = "Start applying migrations...")] + public static partial void StartApplyingMigrations(this ILogger logger); + + [LoggerMessage( + EventId = 561101, + EventName = "DatabaseMigrator.Executor.SuccessfullyAppliedMigrations", + Level = LogLevel.Information, + Message = "All migrations were successfully applied")] + public static partial void SuccessfullyAppliedMigrations(this ILogger logger); + + [LoggerMessage( + EventId = 561102, + EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigrations", + Level = LogLevel.Critical, + Message = "An error occurred while migrating the database. Rolling back already migrated DbContexts...")] + public static partial void ErrorWhileApplyingMigrations(this ILogger logger, Exception ex); + + [LoggerMessage( + EventId = 561103, + EventName = "DatabaseMigrator.Executor.MigratingDbContext", + Level = LogLevel.Information, + Message = "Migrating database associated with context '{context}' to the latest migration...")] + public static partial void MigratingDbContext(this ILogger logger, string context); + + [LoggerMessage( + EventId = 561104, + EventName = "DatabaseMigrator.Executor.MigratingDbContext", + Level = LogLevel.Information, + Message = "Migrating database associated with context '{context}' to target migration '{targetMigration}'...")] + public static partial void MigratingDbContext(this ILogger logger, string context, string targetMigration); + + [LoggerMessage( + EventId = 561105, + EventName = "DatabaseMigrator.Executor.SuccessfullyMigratedDbContext", + Level = LogLevel.Information, + Message = "Migrating database associated with context '{context}' to the latest migration...")] + public static partial void SuccessfullyMigratedDbContext(this ILogger logger, string context); + + [LoggerMessage( + EventId = 561106, + EventName = "DatabaseMigrator.Executor.SuccessfullyMigratedDbContext", + Level = LogLevel.Information, + Message = "Migrating database associated with context '{context}' to target migration '{targetMigration}'...")] + public static partial void SuccessfullyMigratedDbContext(this ILogger logger, string context, string targetMigration); + + [LoggerMessage( + EventId = 561107, + EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigration", + Level = LogLevel.Critical, + Message = + "An error occurred while migrating the database associated with context '{context}' to target migration '{targetMigration}'")] + public static partial void ErrorWhileApplyingMigration(this ILogger logger, Exception ex, string context, string targetMigration); + + [LoggerMessage( + EventId = 561108, + EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigration", + Level = LogLevel.Critical, + Message = + "An error occurred while migrating the database associated with context '{context}' to the latest migration'")] + public static partial void ErrorWhileApplyingMigration(this ILogger logger, Exception ex, string context); + + [LoggerMessage( + EventId = 561109, + EventName = "DatabaseMigrator.Executor.RollingBackDbContext", + Level = LogLevel.Information, + Message = + "Rolling back context '{context}' to migration '{migrationBeforeChanges}'....")] + public static partial void RollingBackDbContext(this ILogger logger, string context, string migrationBeforeChanges); + + [LoggerMessage( + EventId = 561110, + EventName = "DatabaseMigrator.Executor.ErrorOnRollback", + Level = LogLevel.Critical, + Message = + "There was an error while rolling back the migration of context '{context}' to migration '{migrationBeforeChanges}'. The following DbContexts couldn't be rolled back: {remainingDbContexts}")] + public static partial void ErrorOnRollback(this ILogger logger, Exception ex, string context, string migrationBeforeChanges, string remainingDbContexts); } From 186ed62c4107a863cf8b9edfc53148b2e1c00715 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 11:55:35 +0200 Subject: [PATCH 27/49] chore: remove commented out code --- helm/templates/databasemigrator/job.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/helm/templates/databasemigrator/job.yaml b/helm/templates/databasemigrator/job.yaml index 373ffefe78..afd2473255 100644 --- a/helm/templates/databasemigrator/job.yaml +++ b/helm/templates/databasemigrator/job.yaml @@ -9,9 +9,6 @@ metadata: "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-weight": "1" spec: - # selector: - # matchLabels: - # app: {{ .Values.databasemigrator.name }} template: metadata: {{- with .Values.databasemigrator.podAnnotations }} From aaff036e8c14b69003e00d85f803dc61c3851faf Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 11:55:51 +0200 Subject: [PATCH 28/49] feat: set default log level of DatabaseMigrator to Information --- DatabaseMigrator/appsettings.json | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/DatabaseMigrator/appsettings.json b/DatabaseMigrator/appsettings.json index fe10b60a13..ee2039925d 100644 --- a/DatabaseMigrator/appsettings.json +++ b/DatabaseMigrator/appsettings.json @@ -1,15 +1,7 @@ { "Logging": { "MinimumLevel": { - "Default": "Warning", - "Override": { - "Backbone": "Information", - "Enmeshed": "Information", - "ConsumerApi": "Information", - - "Microsoft": "Information", - "Microsoft.AspNetCore": "Warning" - } + "Default": "Information", }, "WriteTo": { "Console": { From 1df1e006003ba3e40effedb60bf7e01919bdf15c Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 11:56:40 +0200 Subject: [PATCH 29/49] chore: explicitly set helm.sh/hook-delete-policy annotation for databasemigrator job --- helm/templates/databasemigrator/job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/templates/databasemigrator/job.yaml b/helm/templates/databasemigrator/job.yaml index afd2473255..0ef615e075 100644 --- a/helm/templates/databasemigrator/job.yaml +++ b/helm/templates/databasemigrator/job.yaml @@ -8,6 +8,7 @@ metadata: annotations: "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation spec: template: metadata: From 76b853ab9cdaa8685e38cffe56bd3acf6d4b760c Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 11:58:07 +0200 Subject: [PATCH 30/49] chore: remove redundant blank --- .ci/helm/buildChart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/helm/buildChart.js b/.ci/helm/buildChart.js index 2a93da2ca8..dbe390960a 100755 --- a/.ci/helm/buildChart.js +++ b/.ci/helm/buildChart.js @@ -7,4 +7,4 @@ const version = getRequiredEnvVar("VERSION"); await $`helm dependency update helm`; -await $`helm package --version ${version} --app-version ${version} helm`; +await $`helm package --version ${version} --app-version ${version} helm`; From d90bec2056ea02a13a8b2556f5d2786fa94a18aa Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 12:48:27 +0200 Subject: [PATCH 31/49] ci: add missing condition in docker-compose.test.yml --- .ci/docker-compose.test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index 79de4be547..94c095771b 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -61,6 +61,9 @@ services: build: context: .. dockerfile: DatabaseMigrator/Dockerfile + depends_on: + seed-database: + condition: service_completed_successfully environment: - ASPNETCORE_ENVIRONMENT=Development configs: From 790cb75b2e842899c3996da9204e8cebaa9e4cfa Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 13:05:01 +0200 Subject: [PATCH 32/49] ci: add health check for sqlserver docker compose service --- .ci/docker-compose.test.postgres.yml | 3 --- .ci/docker-compose.test.sqlserver.yml | 8 +++++--- .ci/docker-compose.test.yml | 3 +++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.ci/docker-compose.test.postgres.yml b/.ci/docker-compose.test.postgres.yml index c451e62194..96da41adae 100644 --- a/.ci/docker-compose.test.postgres.yml +++ b/.ci/docker-compose.test.postgres.yml @@ -24,9 +24,6 @@ services: environment: - PGPASSWORD=Passw0rd command: psql -h postgres -U postgres -d enmeshed -f /app/setup-db/setup-postgres.sql - depends_on: - database: - condition: service_healthy seed-client: environment: diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index 20aaa05273..378604a457 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -10,6 +10,11 @@ services: - MSSQL_SA_PASSWORD=Passw0rd - ACCEPT_EULA=Y - MSSQL_PID=Express + healthcheck: + test: ["CMD", "sqlcmd", "-S", "localhost", "-U", "sa", "-P", "Passw0rd", "-Q", "SELECT 1"] + interval: 5s + timeout: 5s + retries: 5 ports: - 1433:1433 @@ -18,9 +23,6 @@ services: seed-database: image: mcr.microsoft.com/mssql-tools command: bash -c " sleep 20 && /opt/mssql-tools/bin/sqlcmd -S sqlserver -U SA -P Passw0rd -i /app/setup-db/setup-sqlserver.sql " - depends_on: - database: - condition: service_started seed-client: environment: diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index 94c095771b..c043f224a5 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -92,6 +92,9 @@ services: seed-database: volumes: - ../setup-db:/app/setup-db + depends_on: + database: + condition: service_healthy seed-client: container_name: seed-client-test From faa417aaf9b8661ebe0ec7ab08c105db73036feb Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 13:10:43 +0200 Subject: [PATCH 33/49] ci: let seed-client depend on consumer-api health --- .ci/docker-compose.test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index c043f224a5..4cff1ee466 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -102,8 +102,8 @@ services: context: .. dockerfile: Modules/Devices/src/Devices.AdminCli/Dockerfile depends_on: - database-migrator: - condition: service_completed_successfully + consumer-api: + condition: service_healthy command: backbone client create --clientId test --clientSecret test --defaultTier Basic networks: From 9e3ef174f79068c977bb2c785b7f6783f79806d6 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 13:12:19 +0200 Subject: [PATCH 34/49] ci: fix cyclic reference --- .ci/docker-compose.test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index 4cff1ee466..19c8ee1033 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -11,8 +11,6 @@ services: ports: - "5000:8080" depends_on: - seed-client: - condition: service_completed_successfully rabbitmq: condition: service_started azure-storage-emulator: From 4320f868d7a3dd12027c7145a8e627468dcf443d Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 13:28:05 +0200 Subject: [PATCH 35/49] ci: let consumer-api depend on database-migrator --- .ci/docker-compose.test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/docker-compose.test.yml b/.ci/docker-compose.test.yml index 19c8ee1033..84a994b46c 100644 --- a/.ci/docker-compose.test.yml +++ b/.ci/docker-compose.test.yml @@ -11,6 +11,8 @@ services: ports: - "5000:8080" depends_on: + database-migrator: + condition: service_completed_successfully rabbitmq: condition: service_started azure-storage-emulator: From 66de070dfce1caec0c3cd7d5b5dbd7efcd9c4e2d Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 13:35:13 +0200 Subject: [PATCH 36/49] refactor: fix duplicated log event names --- DatabaseMigrator/Executor.cs | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/DatabaseMigrator/Executor.cs b/DatabaseMigrator/Executor.cs index dc63e7f3d3..2bc8366ed7 100644 --- a/DatabaseMigrator/Executor.cs +++ b/DatabaseMigrator/Executor.cs @@ -69,9 +69,9 @@ private async Task MigrateDbContext(string? targetMigration = null) wh await using var context = _serviceProvider.GetDbContext(); if (targetMigration == null) - _logger.MigratingDbContext(typeof(TContext).Name); + _logger.MigratingDbContextToLatestMigration(typeof(TContext).Name); else - _logger.MigratingDbContext(typeof(TContext).Name, targetMigration); + _logger.MigratingDbContextToTargetMigration(typeof(TContext).Name, targetMigration); var pendingMigrations = await context.Database.GetPendingMigrationsAsync(); if (!pendingMigrations.Any()) @@ -86,9 +86,9 @@ private async Task MigrateDbContext(string? targetMigration = null) wh catch (Exception ex) { if (targetMigration == null) - _logger.ErrorWhileApplyingMigration(ex, typeof(TContext).Name); + _logger.ErrorWhileApplyingMigrationToLatestMigration(ex, typeof(TContext).Name); else - _logger.ErrorWhileApplyingMigration(ex, typeof(TContext).Name, targetMigration); + _logger.ErrorWhileApplyingMigrationToTargetMigration(ex, typeof(TContext).Name, targetMigration); throw; } finally @@ -97,9 +97,9 @@ private async Task MigrateDbContext(string? targetMigration = null) wh } if (targetMigration == null) - _logger.SuccessfullyMigratedDbContext(typeof(TContext).Name); + _logger.SuccessfullyMigratedDbContextToLatestMigration(typeof(TContext).Name); else - _logger.SuccessfullyMigratedDbContext(typeof(TContext).Name, targetMigration); + _logger.SuccessfullyMigratedDbContextToTargetMigration(typeof(TContext).Name, targetMigration); } private async Task RollbackAlreadyMigratedDbContexts() @@ -186,47 +186,47 @@ internal static partial class ExecutorLogs [LoggerMessage( EventId = 561103, - EventName = "DatabaseMigrator.Executor.MigratingDbContext", + EventName = "DatabaseMigrator.Executor.MigratingDbContextToLatestMigration", Level = LogLevel.Information, Message = "Migrating database associated with context '{context}' to the latest migration...")] - public static partial void MigratingDbContext(this ILogger logger, string context); + public static partial void MigratingDbContextToLatestMigration(this ILogger logger, string context); [LoggerMessage( EventId = 561104, - EventName = "DatabaseMigrator.Executor.MigratingDbContext", + EventName = "DatabaseMigrator.Executor.MigratingDbContextToTargetMigration", Level = LogLevel.Information, Message = "Migrating database associated with context '{context}' to target migration '{targetMigration}'...")] - public static partial void MigratingDbContext(this ILogger logger, string context, string targetMigration); + public static partial void MigratingDbContextToTargetMigration(this ILogger logger, string context, string targetMigration); [LoggerMessage( EventId = 561105, - EventName = "DatabaseMigrator.Executor.SuccessfullyMigratedDbContext", + EventName = "DatabaseMigrator.Executor.SuccessfullyMigratedDbContextToLatestMigration", Level = LogLevel.Information, Message = "Migrating database associated with context '{context}' to the latest migration...")] - public static partial void SuccessfullyMigratedDbContext(this ILogger logger, string context); + public static partial void SuccessfullyMigratedDbContextToLatestMigration(this ILogger logger, string context); [LoggerMessage( EventId = 561106, - EventName = "DatabaseMigrator.Executor.SuccessfullyMigratedDbContext", + EventName = "DatabaseMigrator.Executor.SuccessfullyMigratedDbContextToTargetMigration", Level = LogLevel.Information, Message = "Migrating database associated with context '{context}' to target migration '{targetMigration}'...")] - public static partial void SuccessfullyMigratedDbContext(this ILogger logger, string context, string targetMigration); + public static partial void SuccessfullyMigratedDbContextToTargetMigration(this ILogger logger, string context, string targetMigration); [LoggerMessage( - EventId = 561107, - EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigration", + EventId = 561108, + EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigrationToLatestMigration", Level = LogLevel.Critical, Message = - "An error occurred while migrating the database associated with context '{context}' to target migration '{targetMigration}'")] - public static partial void ErrorWhileApplyingMigration(this ILogger logger, Exception ex, string context, string targetMigration); + "An error occurred while migrating the database associated with context '{context}' to the latest migration'")] + public static partial void ErrorWhileApplyingMigrationToLatestMigration(this ILogger logger, Exception ex, string context); [LoggerMessage( - EventId = 561108, - EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigration", + EventId = 561107, + EventName = "DatabaseMigrator.Executor.ErrorWhileApplyingMigrationToTargetMigration", Level = LogLevel.Critical, Message = - "An error occurred while migrating the database associated with context '{context}' to the latest migration'")] - public static partial void ErrorWhileApplyingMigration(this ILogger logger, Exception ex, string context); + "An error occurred while migrating the database associated with context '{context}' to target migration '{targetMigration}'")] + public static partial void ErrorWhileApplyingMigrationToTargetMigration(this ILogger logger, Exception ex, string context, string targetMigration); [LoggerMessage( EventId = 561109, From 5458903e60fd8a9bbac0c13bb14b1bdb10da3685 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 14:17:51 +0200 Subject: [PATCH 37/49] ci: test --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6526c8821..239ce8745c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,6 +62,7 @@ jobs: name: Run Integration Tests (on SQL Server) runs-on: ubuntu-latest steps: + - run: docker compose version - name: Checkout uses: actions/checkout@v4 - name: Install script dependencies From c8a7f2af1a039ad740e04b987f45404454bcb9fe Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 14:21:36 +0200 Subject: [PATCH 38/49] ci: test --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 239ce8745c..24c56a15b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,6 +62,10 @@ jobs: name: Run Integration Tests (on SQL Server) runs-on: ubuntu-latest steps: + - run: | + DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} + mkdir -p $DOCKER_CONFIG/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose - run: docker compose version - name: Checkout uses: actions/checkout@v4 From 2db5a538278bb1442c0bdde89f933cbcadda1bae Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Thu, 25 Apr 2024 14:23:16 +0200 Subject: [PATCH 39/49] ci: test --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24c56a15b2..2b89c6ff1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,6 +66,7 @@ jobs: DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} mkdir -p $DOCKER_CONFIG/cli-plugins curl -SL https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose + chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose - run: docker compose version - name: Checkout uses: actions/checkout@v4 From 26effe4cdce7be5754312162c2c7dc205ecf06a7 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 08:27:42 +0200 Subject: [PATCH 40/49] ci: change health check of sql server --- .ci/docker-compose.test.sqlserver.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index 378604a457..31d6db9b4a 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -11,10 +11,10 @@ services: - ACCEPT_EULA=Y - MSSQL_PID=Express healthcheck: - test: ["CMD", "sqlcmd", "-S", "localhost", "-U", "sa", "-P", "Passw0rd", "-Q", "SELECT 1"] - interval: 5s - timeout: 5s - retries: 5 + test: ["sleep", "10"] + interval: 1s + timeout: 1s + retries: 1 ports: - 1433:1433 From ff01d1c8ebd1b2b1ca0aa399b31debf77c36b88b Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 08:30:43 +0200 Subject: [PATCH 41/49] ci: fix health check --- .ci/docker-compose.test.sqlserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index 31d6db9b4a..5a02d35c14 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -11,7 +11,7 @@ services: - ACCEPT_EULA=Y - MSSQL_PID=Express healthcheck: - test: ["sleep", "10"] + test: ["CMD", "sleep", "10"] interval: 1s timeout: 1s retries: 1 From ab5f49dac21081a8a1e49f0112b7aae7e5c606fb Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 09:32:44 +0200 Subject: [PATCH 42/49] fix: remove duplicated role in ALTER DEFAULT PRIVILEGES commands --- setup-db/setup-postgres.sql | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/setup-db/setup-postgres.sql b/setup-db/setup-postgres.sql index 9e8e8894b8..47dae177a0 100644 --- a/setup-db/setup-postgres.sql +++ b/setup-db/setup-postgres.sql @@ -152,59 +152,59 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA "AdminUi" GRANT ALL ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Relationships" TO messages; GRANT SELECT, REFERENCES, TRIGGER, TRUNCATE ON ALL TABLES IN SCHEMA "Relationships" TO messages; -ALTER DEFAULT PRIVILEGES FOR ROLE relationships IN SCHEMA "Relationships" GRANT SELECT, REFERENCES, TRIGGER, TRUNCATE ON TABLES TO messages; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Relationships" GRANT SELECT, REFERENCES, TRIGGER, TRUNCATE ON TABLES TO messages; GRANT USAGE ON SCHEMA "Challenges" TO devices; GRANT SELECT ON ALL TABLES IN SCHEMA "Challenges" TO devices; -ALTER DEFAULT PRIVILEGES FOR ROLE challenges IN SCHEMA "Challenges" GRANT SELECT ON TABLES TO devices; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Challenges" GRANT SELECT ON TABLES TO devices; GRANT USAGE ON SCHEMA "Messages" TO quotas; GRANT SELECT ON ALL TABLES IN SCHEMA "Messages" TO quotas; -ALTER DEFAULT PRIVILEGES FOR ROLE messages IN SCHEMA "Messages" GRANT SELECT ON TABLES TO quotas; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Messages" GRANT SELECT ON TABLES TO quotas; GRANT USAGE ON SCHEMA "Files" TO quotas; GRANT SELECT ON ALL TABLES IN SCHEMA "Files" TO quotas; -ALTER DEFAULT PRIVILEGES FOR ROLE files IN SCHEMA "Files" GRANT SELECT ON TABLES TO quotas; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Files" GRANT SELECT ON TABLES TO quotas; GRANT USAGE ON SCHEMA "Relationships" TO quotas; GRANT SELECT ON ALL TABLES IN SCHEMA "Relationships" TO quotas; -ALTER DEFAULT PRIVILEGES FOR ROLE relationships IN SCHEMA "Relationships" GRANT SELECT ON TABLES TO quotas; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Relationships" GRANT SELECT ON TABLES TO quotas; GRANT USAGE ON SCHEMA "Challenges" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Challenges" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE challenges IN SCHEMA "Challenges" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Challenges" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Synchronization" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Synchronization" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE synchronization IN SCHEMA "Synchronization" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Synchronization" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Messages" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Messages" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE messages IN SCHEMA "Messages" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Messages" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Devices" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Devices" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE devices IN SCHEMA "Devices" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Devices" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Tokens" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Tokens" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE tokens IN SCHEMA "Tokens" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Tokens" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Relationships" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Relationships" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE relationships IN SCHEMA "Relationships" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Relationships" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Files" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Files" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE files IN SCHEMA "Files" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Files" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Quotas" TO "adminUi"; GRANT SELECT ON ALL TABLES IN SCHEMA "Quotas" TO "adminUi"; -ALTER DEFAULT PRIVILEGES FOR ROLE quotas IN SCHEMA "Quotas" GRANT SELECT ON TABLES TO "adminUi"; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Quotas" GRANT SELECT ON TABLES TO "adminUi"; GRANT USAGE ON SCHEMA "Tokens" TO quotas; GRANT SELECT ON ALL TABLES IN SCHEMA "Tokens" TO quotas; -ALTER DEFAULT PRIVILEGES FOR ROLE tokens IN SCHEMA "Tokens" GRANT SELECT ON TABLES TO quotas; +ALTER DEFAULT PRIVILEGES IN SCHEMA "Tokens" GRANT SELECT ON TABLES TO quotas; CREATE TABLE IF NOT EXISTS "Challenges"."__EFMigrationsHistory" From 398cfd5c12a0028eb51181cab25a8fbe093a1120 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 09:41:41 +0200 Subject: [PATCH 43/49] ci: reduce sleep time for sqlserver start --- .ci/docker-compose.test.sqlserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index 5a02d35c14..d8ba83405e 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -11,7 +11,7 @@ services: - ACCEPT_EULA=Y - MSSQL_PID=Express healthcheck: - test: ["CMD", "sleep", "10"] + test: ["CMD", "sleep", "1"] interval: 1s timeout: 1s retries: 1 From e989e360c3b6458109c2330d6bbb41cfdbbe3b63 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 09:50:56 +0200 Subject: [PATCH 44/49] ci: further reduce sleep time for sqlserver start --- .ci/docker-compose.test.sqlserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index d8ba83405e..f7473b7d33 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -11,7 +11,7 @@ services: - ACCEPT_EULA=Y - MSSQL_PID=Express healthcheck: - test: ["CMD", "sleep", "1"] + test: ["CMD", "sleep", "0.5"] interval: 1s timeout: 1s retries: 1 From 5bace69327ed6c379c7ec12a0259efdb84cc3077 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 09:51:41 +0200 Subject: [PATCH 45/49] ci: reduce sleep --- .ci/docker-compose.test.sqlserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose.test.sqlserver.yml b/.ci/docker-compose.test.sqlserver.yml index f7473b7d33..2e71f260b9 100644 --- a/.ci/docker-compose.test.sqlserver.yml +++ b/.ci/docker-compose.test.sqlserver.yml @@ -11,7 +11,7 @@ services: - ACCEPT_EULA=Y - MSSQL_PID=Express healthcheck: - test: ["CMD", "sleep", "0.5"] + test: ["CMD", "echo"] interval: 1s timeout: 1s retries: 1 From d9fd5c0fbdb0616b70ca6501879cad285e962dfc Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 11:47:49 +0200 Subject: [PATCH 46/49] feat: update permissions in setup-sqlserver.sql script --- setup-db/setup-sqlserver.sql | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/setup-db/setup-sqlserver.sql b/setup-db/setup-sqlserver.sql index 74ba423337..f9ad347520 100644 --- a/setup-db/setup-sqlserver.sql +++ b/setup-db/setup-sqlserver.sql @@ -229,14 +229,15 @@ GO /*+++++++++++++++++++++++++++++++++++++++++++++++++ Authorizations +++++++++++++++++++++++++++++++++++++++++++++++++*/ PRINT 'Start changing authorizations' ; -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Challenges TO synchronization, devices, messages, tokens, relationships, files, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Synchronization TO challenges, devices, messages, tokens, relationships, files, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Messages TO challenges, synchronization, devices, tokens, relationships, files, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Devices TO challenges, synchronization, messages, tokens, relationships, files, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Tokens TO challenges, synchronization, devices, messages, relationships, files, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Relationships TO challenges, synchronization, devices, messages, tokens, files, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Files TO challenges, synchronization, devices, messages, tokens, relationships, quotas -DENY SELECT, INSERT, UPDATE, DELETE ON SCHEMA::Quotas TO challenges, synchronization, devices, messages, tokens, relationships, files +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Challenges TO challenges; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Devices TO devices; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Files TO files; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Messages TO messages; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Quotas TO quotas; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Relationships TO relationships; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Synchronization TO synchronization; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::Tokens TO tokens; +GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE, REFERENCES, VIEW DEFINITION ON SCHEMA::AdminUi TO adminUi; GRANT SELECT, REFERENCES ON SCHEMA::Relationships TO messages GRANT SELECT, REFERENCES ON SCHEMA::Challenges TO devices From caaca2612da02332be1625cdc98a26cf8d0da7fd Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 12:09:15 +0200 Subject: [PATCH 47/49] ci: try to not update docker-compose --- .github/workflows/test.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b89c6ff1c..239ce8745c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,11 +62,6 @@ jobs: name: Run Integration Tests (on SQL Server) runs-on: ubuntu-latest steps: - - run: | - DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} - mkdir -p $DOCKER_CONFIG/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose - run: docker compose version - name: Checkout uses: actions/checkout@v4 From 9599c40ec2579526b8a2a04247156852566fee04 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 12:56:33 +0200 Subject: [PATCH 48/49] ci: remove debug statement --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 239ce8745c..f6526c8821 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,6 @@ jobs: name: Run Integration Tests (on SQL Server) runs-on: ubuntu-latest steps: - - run: docker compose version - name: Checkout uses: actions/checkout@v4 - name: Install script dependencies From ee5b62946c57084ff6c2f2ce463a93b34d6a4930 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Fri, 26 Apr 2024 13:10:54 +0200 Subject: [PATCH 49/49] chore: only allow RunMigrations flag if environment is Development or Local --- AdminApi/src/AdminApi/Program.cs | 2 +- ConsumerApi/Program.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AdminApi/src/AdminApi/Program.cs b/AdminApi/src/AdminApi/Program.cs index b6f8b65098..b77bb14770 100644 --- a/AdminApi/src/AdminApi/Program.cs +++ b/AdminApi/src/AdminApi/Program.cs @@ -80,7 +80,7 @@ static WebApplication CreateApp(string[] args) var app = builder.Build(); Configure(app); - if (app.Configuration.GetValue("RunMigrations")) + if ((app.Environment.IsLocal() || app.Environment.IsDevelopment()) && app.Configuration.GetValue("RunMigrations")) app.MigrateDbContext(); return app; diff --git a/ConsumerApi/Program.cs b/ConsumerApi/Program.cs index 557f8c4d56..3ebc9804d7 100644 --- a/ConsumerApi/Program.cs +++ b/ConsumerApi/Program.cs @@ -98,7 +98,7 @@ static WebApplication CreateApp(string[] args) var app = builder.Build(); Configure(app); - if (app.Configuration.GetValue("RunMigrations")) + if ((app.Environment.IsLocal() || app.Environment.IsDevelopment()) && app.Configuration.GetValue("RunMigrations")) { app .MigrateDbContext()