chore: Implement caching for integration tests and transport tests

.github/workflows/test.yml

@@ -103,34 +103,138 @@ jobs:
         run: ./.ci/test.sh
         env:
           DOTNET_CONSOLE_ANSI_COLOR: true
+
+  container-test-builds:
+    name: Build ${{matrix.tag}} Container
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - dockerfile: ConsumerApi/Dockerfile
+            tag: consumer-api
+          - dockerfile: AdminApi/src/AdminApi/Dockerfile
+            tag: admin-ui
+          - dockerfile: Modules/Devices/src/Devices.AdminCli/Dockerfile
+            tag: seed-client
+          - dockerfile: DatabaseMigrator/Dockerfile
+            tag: database-migrator
+          - dockerfile: EventHandlerService/src/EventHandlerService/Dockerfile
+            tag: event-handler-service
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          check-latest: true
+          cache: "npm"
+          cache-dependency-path: "**/package-lock.json"
+
+      - name: Setup Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup NuGet
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.x
+
+      - name: Cache Image
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-${{ matrix.tag }}-cache
+          key: buildx-${{ matrix.tag }}-cache-${{ hashFiles(matrix.dockerfile) }}
+          restore-keys: |
+            buildx-${{ matrix.tag }}-cache-
+            
+      - name: Install script dependencies
+        run: npm install --prefix ./.ci
+
+      - name: Build Container
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          cache-from: type=local,src=/tmp/.buildx-${{ matrix.tag }}-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-${{ matrix.tag }}-cache-new
+          tags: ${{ matrix.tag }}
+          load: false
+          outputs: type=docker,dest=/tmp/${{ matrix.tag }}.tar
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-${{ matrix.tag }}
+          path: /tmp/${{ matrix.tag }}.tar
+          retention-days: 1
+
+      - name: Save Docker Logs
+        if: failure()
+        run: docker compose -f ./.ci/docker-compose.test.yml -f ./.ci/docker-compose.test.postgres.yml logs > docker-log.txt
+
+      - name: Archive logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-postgres-docker-logs
+          path: docker-log.txt
+        env:
+          DOTNET_CONSOLE_ANSI_COLOR: true
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-${{ matrix.tag }}-cache
+          mv /tmp/.buildx-${{ matrix.tag }}-cache-new /tmp/.buildx-${{ matrix.tag }}-cache
 
   integration-test-sqlserver:
     name: Run Integration Tests (on SQL Server)
     runs-on: ubuntu-latest
+    needs: container-test-builds
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
       - uses: actions/setup-node@v3
         with:
           node-version: 20
           check-latest: true
           cache: "npm"
           cache-dependency-path: "**/package-lock.json"
-      - name: Cache npm dependencies
-        uses: actions/cache@v3
+
+      - name: Setup NuGet
+        uses: actions/setup-dotnet@v4
         with:
-          path: |
-            ./runtime/node_modules
-          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-npm-
+          dotnet-version: 8.x
+
       - name: Install script dependencies
         run: npm install --prefix ./.ci
+
+      - name: Download Docker images
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp
+          pattern: docker-*
+          merge-multiple: true
+
+      - name: Load Docker images
+        run: |
+          docker load --input /tmp/consumer-api.tar
+          docker load --input /tmp/admin-ui.tar
+          docker load --input /tmp/seed-client.tar
+          docker load --input /tmp/database-migrator.tar
+          docker load --input /tmp/event-handler-service.tar
+      
       - name: Run integration tests
         run: ./.ci/integrationTest.sqlserver.sh
+
       - name: Save Docker Logs
         if: failure()
         run: docker compose -f ./.ci/docker-compose.test.yml -f ./.ci/docker-compose.test.sqlserver.yml logs > docker-log.txt
+
       - name: Archive logs
         if: failure()
         uses: actions/upload-artifact@v4
@@ -143,30 +247,48 @@ jobs:
   integration-test-postgres:
     name: Run Integration Tests (on Postgres)
     runs-on: ubuntu-latest
+    needs: container-test-builds
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
       - uses: actions/setup-node@v3
         with:
           node-version: 20
           check-latest: true
           cache: "npm"
           cache-dependency-path: "**/package-lock.json"
-      - name: Cache npm dependencies
-        uses: actions/cache@v3
+
+      - name: Setup NuGet
+        uses: actions/setup-dotnet@v4
         with:
-          path: |
-            ./runtime/node_modules
-          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-npm-
+          dotnet-version: 8.x
+
       - name: Install script dependencies
         run: npm install --prefix ./.ci
+
+      - name: Download Docker images
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp
+          pattern: docker-*
+          merge-multiple: true
+
+      - name: Load Docker images
+        run: |
+          docker load --input /tmp/consumer-api.tar
+          docker load --input /tmp/admin-ui.tar
+          docker load --input /tmp/seed-client.tar
+          docker load --input /tmp/database-migrator.tar
+          docker load --input /tmp/event-handler-service.tar
+      
       - name: Run integration tests
         run: ./.ci/integrationTest.postgres.sh
+
       - name: Save Docker Logs
         if: failure()
         run: docker compose -f ./.ci/docker-compose.test.yml -f ./.ci/docker-compose.test.postgres.yml logs > docker-log.txt
+
       - name: Archive logs
         if: failure()
         uses: actions/upload-artifact@v4
@@ -179,31 +301,48 @@ jobs:
   transport-test-sqlserver:
     name: Run transport Tests (on SQL Server)
     runs-on: ubuntu-latest
+    needs: container-test-builds
     steps:
       - name: Checkout backbone repository
         uses: actions/checkout@v4
         with:
           path: backbone
+
       - name: Checkout runtime repository
         uses: actions/checkout@v4
         with:
           repository: nmshd/runtime
           path: runtime
-      - name: Cache npm dependencies
-        uses: actions/cache@v3
+
+      - uses: actions/setup-node@v3
         with:
-          path: |
-            ./runtime/node_modules
-          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-npm-
-      - name: Build Consumer API and Admin API
-        run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.sqlserver.yml build --parallel
+          node-version: 20
+          check-latest: true
+          cache: "npm"
+          cache-dependency-path: "**/package-lock.json"
+
+      - name: Download Docker images
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp
+          pattern: docker-*
+          merge-multiple: true
+
+      - name: Load Docker images
+        run: |
+          docker load --input /tmp/consumer-api.tar
+          docker load --input /tmp/admin-ui.tar
+          docker load --input /tmp/seed-client.tar
+          docker load --input /tmp/database-migrator.tar
+          docker load --input /tmp/event-handler-service.tar
+      
       - name: Run Consumer API & Admin API
         run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.sqlserver.yml up -d --no-build
+
       - name: Install runtime dependencies
         working-directory: ./runtime
         run: npm install
+
       - name: Run transport Tests
         working-directory: ./runtime/packages/transport
         env:
@@ -213,46 +352,66 @@ jobs:
           NMSHD_TEST_BASEURL_ADMIN_API: http://localhost:5173
           NMSHD_TEST_ADMIN_API_KEY: test
         run: npm run test:local:lokijs
+
       - name: Save Docker Logs
         if: failure()
         run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.sqlserver.yml logs > docker-log.txt
+
       - name: Archive logs
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: transport-test-sqlserver-docker-logs
           path: docker-log.txt
+
       - name: Stop Consumer API
         run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.sqlserver.yml down
 
   transport-test-postgres:
     name: Run transport Tests (on Postgres)
     runs-on: ubuntu-latest
+    needs: container-test-builds
     steps:
       - name: Checkout backbone repository
         uses: actions/checkout@v4
         with:
           path: backbone
+
       - name: Checkout runtime repository
         uses: actions/checkout@v4
         with:
           repository: nmshd/runtime
           path: runtime
-      - name: Cache npm dependencies
-        uses: actions/cache@v3
+
+      - uses: actions/setup-node@v3
         with:
-          path: |
-            ./runtime/node_modules
-          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-npm-
-      - name: Build Consumer API and Admin API
-        run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.postgres.yml build --parallel
+          node-version: 20
+          check-latest: true
+          cache: "npm"
+          cache-dependency-path: "**/package-lock.json"
+
+      - name: Download Docker images
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp
+          pattern: docker-*
+          merge-multiple: true
+
+      - name: Load Docker images
+        run: |
+          docker load --input /tmp/consumer-api.tar
+          docker load --input /tmp/admin-ui.tar
+          docker load --input /tmp/seed-client.tar
+          docker load --input /tmp/database-migrator.tar
+          docker load --input /tmp/event-handler-service.tar
+      
       - name: Run Consumer API & Admin API
         run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.postgres.yml up -d --no-build
+
       - name: Install runtime dependencies
         working-directory: ./runtime
         run: npm install
+
       - name: Run transport Tests
         working-directory: ./runtime/packages/transport
         env:
@@ -262,15 +421,18 @@ jobs:
           NMSHD_TEST_BASEURL_ADMIN_API: http://localhost:5173
           NMSHD_TEST_ADMIN_API_KEY: test
         run: npm run test:local:lokijs
+
       - name: Save Docker Logs
         if: failure()
         run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.postgres.yml logs > docker-log.txt
+
       - name: Archive logs
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: transport-test-postgres-docker-logs
           path: docker-log.txt
+
       - name: Stop Consumer API
         run: docker compose -f ./backbone/.ci/docker-compose.test.yml -f ./backbone/.ci/docker-compose.test.postgres.yml down