Database Migrator

.ci/dbm/buildContainerImage.js

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+import { $ } from "zx";
+import { getRequiredEnvVar } from "../lib.js";
+
+const tag = getRequiredEnvVar("TAG");
+
+const platforms = process.env.PLATFORMS ?? "linux/amd64,linux/arm64";
+const push = process.env.PUSH === "1" ? ["--push", "--provenance=true", "--sbom=true"] : "";
+
+await $`docker buildx build --file ./DatabaseMigrator/Dockerfile --tag ghcr.io/nmshd/backbone-database-migrator:${tag} --platform ${platforms} ${push} .`;

.ci/docker-compose.test.postgres.yml

@@ -21,20 +21,20 @@ services:
 
   seed-database:
     image: postgres
-    volumes:
-      - ../setup-db:/app/setup-db
     environment:
       - PGPASSWORD=Passw0rd
     command: psql -h postgres -U postgres -d enmeshed -f /app/setup-db/setup-postgres.sql
-    depends_on:
-      database:
-        condition: service_healthy
 
   seed-client:
     environment:
       Database__Provider: Postgres
       Database__ConnectionString: "Server=postgres;Database=enmeshed;User Id=devices;Password=Passw0rd;Port=5432"
 
+  database-migrator:
+    environment:
+      Infrastructure__SqlDatabase__Provider: Postgres
+      Infrastructure__SqlDatabase__ConnectionString: "Server=postgres;Database=enmeshed;User Id=postgres;Password=Passw0rd;Port=5432"
+
 configs:
   Config:
     file: appsettings.override.postgres.json

.ci/docker-compose.test.sqlserver.yml

@@ -10,24 +10,30 @@ services:
       - MSSQL_SA_PASSWORD=Passw0rd
       - ACCEPT_EULA=Y
       - MSSQL_PID=Express
+    healthcheck:
+      test: ["CMD", "echo"]
+      interval: 1s
+      timeout: 1s
+      retries: 1
     ports:
       - 1433:1433
 
   ### seeds ###
 
   seed-database:
     image: mcr.microsoft.com/mssql-tools
-    volumes:
-      - ../setup-db:/app/setup-db
     command: bash -c " sleep 20 && /opt/mssql-tools/bin/sqlcmd -S sqlserver -U SA -P Passw0rd -i /app/setup-db/setup-sqlserver.sql "
-    depends_on:
-      - database
 
   seed-client:
     environment:
       Database__Provider: SqlServer
       Database__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True"
 
+  database-migrator:
+    environment:
+      Infrastructure__SqlDatabase__Provider: SqlServer
+      Infrastructure__SqlDatabase__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True"
+
 configs:
   Config:
     file: appsettings.override.sqlserver.json

.ci/docker-compose.test.yml

@@ -11,12 +11,10 @@ services:
     ports:
       - "5000:8080"
     depends_on:
-      database:
-        condition: service_started
+      database-migrator:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_started
-      seed-database:
-        condition: service_completed_successfully
       azure-storage-emulator:
         condition: service_started
     configs:
@@ -34,12 +32,10 @@ services:
     ports:
       - "5173:8080"
     depends_on:
-      database:
-        condition: service_started
+      database-migrator:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_started
-      consumer-api:
-        condition: service_healthy
     configs:
       - source: Config
         target: app/appsettings.override.json
@@ -52,14 +48,28 @@ services:
     environment:
       - ASPNETCORE_ENVIRONMENT=Development
     depends_on:
-      database:
-        condition: service_started
+      database-migrator:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_started
     configs:
       - source: Config
         target: app/appsettings.override.json
 
+  database-migrator:
+    container_name: database-migrator-test
+    build:
+      context: ..
+      dockerfile: DatabaseMigrator/Dockerfile
+    depends_on:
+      seed-database:
+        condition: service_completed_successfully
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Development
+    configs:
+      - source: Config
+        target: app/appsettings.override.json
+
   ### infrastructure ###
 
   azure-storage-emulator:
@@ -79,6 +89,13 @@ services:
 
   ### seeds ###
 
+  seed-database:
+    volumes:
+      - ../setup-db:/app/setup-db
+    depends_on:
+      database:
+        condition: service_healthy
+
   seed-client:
     container_name: seed-client-test
     build:

.ci/helm/buildChart.js

@@ -7,4 +7,4 @@ const version = getRequiredEnvVar("VERSION");
 
 await $`helm dependency update helm`;
 
-await $`helm package --version ${version} --app-version ${version}  helm`;
+await $`helm package --version ${version} --app-version ${version} helm`;

.github/workflows/publish.yml

@@ -161,6 +161,45 @@ jobs:
           TAG: ${{ steps.extract-version-from-git-tag.outputs.VERSION }}
           PUSH: 1
 
+  publish-database-migrator:
+    name: Publish Database Migrator Container Image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install script dependencies
+        run: npm install --prefix ./.ci
+      - name: Docker Login
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract version from git tag
+        id: extract-version-from-git-tag
+        run: echo "VERSION=$(./.ci/extractVersionFromGitTag.js)" >> $GITHUB_OUTPUT
+        env:
+          GIT_TAG: ${{ github.ref_name }}
+      - name: Log in to Docker Hub for accessing the cloud builder
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.CLOUD_BUILDER_USERNAME }}
+          password: ${{ secrets.CLOUD_BUILDER_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: "lab:latest"
+          driver: cloud
+          endpoint: "jssoft/js-soft"
+      - name: Build and Push Container Image
+        run: ./.ci/dbm/buildContainerImage.js
+        env:
+          TAG: ${{ steps.extract-version-from-git-tag.outputs.VERSION }}
+          PUSH: 1
+
   publish-helm-chart:
     name: Publish Helm Chart
     runs-on: ubuntu-latest

.github/workflows/test.yml

@@ -62,6 +62,7 @@ jobs:
     name: Run Integration Tests (on SQL Server)
     runs-on: ubuntu-latest
     steps:
+      - run: docker compose version
       - name: Checkout
         uses: actions/checkout@v4
       - name: Install script dependencies
@@ -221,6 +222,22 @@ jobs:
           TAG: test
           PLATFORMS: linux/amd64
 
+  build-dbm-container-image:
+    name: Build Database Migrator Container Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install script dependencies
+        run: npm install --prefix ./.ci
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build image
+        run: ./.ci/dbm/buildContainerImage.js
+        env:
+          TAG: test
+          PLATFORMS: linux/amd64
+
   build-aui-container-image:
     name: Build Admin UI Container Image
     runs-on: ubuntu-latest

.run/Backbone.run.xml

@@ -0,0 +1,43 @@
+<component name="ProjectRunConfigurationManager">
+    <configuration default="false" name="Backbone" type="com.intellij.execution.configurations.multilaunch" factoryName="MultiLaunchConfiguration">
+        <rows>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="immediately"/>
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:Docker.docker-compose: Compose Deployment"/>
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="immediately"/>
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:.NET Launch Settings Profile.AdminApi: Default"/>
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="immediately"/>
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:.NET Launch Settings Profile.ConsumerApi: Default"/>
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+        </rows>
+        <method v="2"/>
+    </configuration>
+</component>

AdminApi/src/AdminApi/Program.cs

@@ -15,6 +15,7 @@
 using Backbone.Tooling.Extensions;
 using Microsoft.AspNetCore.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Options;
+using Microsoft.IdentityModel.Logging;
 using Serilog;
 using Serilog.Exceptions;
 using Serilog.Exceptions.Core;
@@ -79,7 +80,8 @@ static WebApplication CreateApp(string[] args)
     var app = builder.Build();
     Configure(app);
 
-    app.MigrateDbContext<AdminApiDbContext>();
+    if (app.Configuration.GetValue<bool>("RunMigrations"))
+        app.MigrateDbContext<AdminApiDbContext>();
 
     return app;
 }
@@ -175,7 +177,7 @@ static void Configure(WebApplication app)
         app.UseSwagger().UseSwaggerUI();
 
     if (app.Environment.IsDevelopment())
-        Microsoft.IdentityModel.Logging.IdentityModelEventSource.ShowPII = true;
+        IdentityModelEventSource.ShowPII = true;
 
     app.UseCors();
 

AdminApi/src/AdminApi/appsettings.override.json

@@ -1,4 +1,5 @@
 {
+    "RunMigrations": true,
     "Authentication": {
         "ApiKey": "test"
     },
@@ -24,8 +25,8 @@
         },
         "SqlDatabase": {
             "Provider": "Postgres",
-            "ConnectionString": "User ID=adminUi;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres
-            //"ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=adminUi;Password=Passw0rd;TrustServerCertificate=True"
+            "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres
+//            "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True"
         }
     },
     "Modules": {
@@ -40,8 +41,8 @@
             "Infrastructure": {
                 "SqlDatabase": {
                     "Provider": "Postgres",
-                    "ConnectionString": "User ID=devices;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres
-                    //"ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True"
+                    "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres
+//                    "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True"
                 },
                 "PushNotifications": {
                     "Provider": "Dummy"
@@ -58,8 +59,8 @@
             "Infrastructure": {
                 "SqlDatabase": {
                     "Provider": "Postgres",
-                    "ConnectionString": "User ID=quotas;Password=Passw0rd;Server=localhost;Port=5432;Database=enmeshed;" // postgres
-                    //"ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=quotas;Password=Passw0rd;TrustServerCertificate=True"
+                    "ConnectionString": "User ID=postgres;Password=admin;Server=localhost;Port=5432;Database=enmeshed;" // postgres
+//                    "ConnectionString": "Server=localhost;Database=enmeshed;User Id=sa;Password=Passw0rd;TrustServerCertificate=True"
                 }
             }
         }

Backbone.sln

@@ -297,6 +297,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EventHandlerService", "Even
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EventHandlerService.Tests", "EventHandlerService\test\EventHandlerService.Tests\EventHandlerService.Tests.csproj", "{D47E0FE1-23A0-4A96-AF57-E3CE71A132AD}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DatabaseMigrator", "DatabaseMigrator\DatabaseMigrator.csproj", "{143EAC38-A2D0-4F74-88F1-3EF27413A8C6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -715,6 +717,10 @@ Global
 		{D47E0FE1-23A0-4A96-AF57-E3CE71A132AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D47E0FE1-23A0-4A96-AF57-E3CE71A132AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D47E0FE1-23A0-4A96-AF57-E3CE71A132AD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{143EAC38-A2D0-4F74-88F1-3EF27413A8C6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

BuildingBlocks/src/BuildingBlocks.API/Extensions/HostExtensions.cs

@@ -5,7 +5,6 @@
 using Microsoft.Extensions.Logging;
 using Npgsql;
 using Polly;
-using Polly.Retry;
 
 namespace Backbone.BuildingBlocks.API.Extensions;
 

ConsumerApi/Program.cs

@@ -98,16 +98,19 @@ static WebApplication CreateApp(string[] args)
     var app = builder.Build();
     Configure(app);
 
-    app
-        .MigrateDbContext<ChallengesDbContext>()
-        .MigrateDbContext<DevicesDbContext>()
-        .MigrateDbContext<FilesDbContext>()
-        .MigrateDbContext<RelationshipsDbContext>()
-        .MigrateDbContext<QuotasDbContext>()
-        .MigrateDbContext<MessagesDbContext>()
-        .MigrateDbContext<SynchronizationDbContext>()
-        .MigrateDbContext<TokensDbContext>()
-        .MigrateDbContext<QuotasDbContext>();
+    if (app.Configuration.GetValue<bool>("RunMigrations"))
+    {
+        app
+            .MigrateDbContext<ChallengesDbContext>()
+            .MigrateDbContext<DevicesDbContext>()
+            .MigrateDbContext<FilesDbContext>()
+            .MigrateDbContext<RelationshipsDbContext>()
+            .MigrateDbContext<QuotasDbContext>()
+            .MigrateDbContext<MessagesDbContext>()
+            .MigrateDbContext<SynchronizationDbContext>()
+            .MigrateDbContext<TokensDbContext>()
+            .MigrateDbContext<QuotasDbContext>();
+    }
 
     app
         .SeedDbContext<DevicesDbContext, DevicesDbContextSeeder>()