ci: github actions for tileserver (#17)

* ci: fix kv-warmer, format files and run tests in CI * chore: pin node version * ci: fix incorrect node path * ci: fix linting and test running * ci: build tiles code * chore: remove cf version metadata as it can't be used from terraform * chore: remove file hash and use file name instead as unique identifier * ci: deploy tiles gha * ci: deploy tiles gha * ci: deploy tiles gha * ci: deploy tiles gha * chore: test push-o-matic * Update tile server file to 20240823.pmtiles * ci: deploy tiles gha * ci: run kv warmer and check if already warm * chore: revert to node 18 because of network problems nodejs/node#47822 * chore: revert back to v1.pmtiles until v4 spec is finalised by protomaps * Update deployment/modules/cloudflare/tiles/workers.tf Fix incorrect environment variable in worker deployment Co-authored-by: bo0tzz <[email protected]> * fix: kv-warming deploy on main only --------- Co-authored-by: zackpollard <[email protected]> Co-authored-by: bo0tzz <[email protected]>
immich-app · Aug 27, 2024 · 517134c · 517134c
1 parent 46d4129
commit 517134c
Show file tree

Hide file tree

Showing 30 changed files with 1,297 additions and 1,788 deletions.
diff --git a/.github/workflows/terragrunt.yaml b/.github/workflows/terragrunt.yaml
diff --git a/.github/workflows/tiles-file.yml b/.github/workflows/tiles-file.yml
@@ -0,0 +1,64 @@
+name: 'Deploy Tiles'
+on:
+  workflow_dispatch:
+  pull_request:
+      branches: [ "main" ]
+      paths:
+        - ".github/workflows/deploy-tiles.yml"
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Find latest pmtiles build
+        id: latest_build
+        run: |
+          latest_build=$(curl -s https://build-metadata.protomaps.dev/builds.json | jq -r 'map(.key) | sort | last')
+          echo "KEY=$latest_build"
+          echo "KEY=$latest_build" >> $GITHUB_OUTPUT
+
+      - name: Setup Rclone
+        uses: AnimMouse/setup-rclone@v1
+        with:
+          disable_base64: true
+          rclone_config: |
+            [r2]
+            type = s3
+            provider = Cloudflare
+            region = auto
+            endpoint = https://601adaaa1eab190cdfe2945f9a6c89d0.r2.cloudflarestorage.com
+            access_key_id = ${{ secrets.CLOUDFLARE_TILES_R2_KV_TOKEN_ID }}
+            secret_access_key = ${{ secrets.CLOUDFLARE_TILES_R2_KV_TOKEN_HASHED_VALUE }}
+            [pmtiles]
+            type = http
+            url = https://build.protomaps.com/
+
+      - name: Copy latest tiles to R2
+        run: rclone --progress copyto pmtiles:/${{ steps.latest_build.outputs.KEY }} r2:/tiles/${{ steps.latest_build.outputs.KEY }} --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
+
+      - name: Generate a token
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.PUSH_O_MATIC_APP_ID }}
+          private-key: ${{ secrets.PUSH_O_MATIC_APP_KEY }}
+
+      - name: 'Checkout'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Update deployment file name
+        run: |
+          sed -i -E 's/"pmtiles_file_name": "(.*)"/"pmtiles_file_name": "${{ steps.latest_build.outputs.KEY }}"/' deployment/modules/cloudflare/tiles/tiles.tfvars.json
+
+      - name: Push updated wrangler.toml to repo
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: deployment/modules/cloudflare/tiles/tiles.tfvars.json
+          message: Update tile server file to ${{ steps.latest_build.outputs.KEY }}
+          github-token: ${{ steps.generate-token.outputs.token }}
diff --git a/.github/workflows/tiles-worker.yml b/.github/workflows/tiles-worker.yml
@@ -0,0 +1,223 @@
+name: Tiles Worker
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  tofu_version: '1.7.1'
+  tg_version: '0.58.12'
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./tiles
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: './tiles/.nvmrc'
+
+      - name: Run npm install
+        run: npm ci
+
+      - name: Run linter
+        run: npm run lint
+        if: ${{ !cancelled() }}
+
+      - name: Run formatter
+        run: npm run format
+        if: ${{ !cancelled() }}
+
+      - name: Run tsc
+        run: npm run check
+        if: ${{ !cancelled() }}
+
+      - name: Run unit tests & coverage
+        run: npm run test:cov
+        if: ${{ !cancelled() }}
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./tiles
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: './tiles/.nvmrc'
+
+      - name: Run npm install
+        run: npm ci
+
+      - name: Run tsc
+        run: npm run check
+        if: ${{ !cancelled() }}
+
+      - name: Wrangler build
+        run: npm run build
+        if: ${{ !cancelled() }}
+
+      - name: Upload build output
+        uses: actions/upload-artifact@v4
+        with:
+          name: tiles-build-output
+          path: tiles/dist
+          retention-days: 14
+        if: ${{ !cancelled() }}
+
+  check-terragrunt:
+    name: Check Terragrunt
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./deployment/cloudflare/tiles
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@main
+
+      - name: Check Formatting
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tofu_version: ${{ env.tofu_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.working_dir }}
+          tg_command: 'hclfmt --terragrunt-check --terragrunt-diff'
+
+      - name: Check terraform fmt
+        uses: gruntwork-io/terragrunt-action@v2
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          TILES_BUILD_DIR: dist
+          VMETRICS_API_TOKEN: ${{ secrets.VMETRICS_API_TOKEN }}
+          TF_STATE_POSTGRES_CONN_STR: ${{ secrets.TF_STATE_POSTGRES_CONN_STR }}
+        with:
+          tofu_version: ${{ env.tofu_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.working_dir }}
+          tg_command: 'run-all fmt -diff -check'
+  plan-terragrunt:
+    needs: build
+    name: Plan Terragrunt
+    runs-on: ubuntu-latest
+    if: github.ref != 'refs/heads/main'
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@main
+
+      - name: Install Terragrunt
+        uses: eLco/setup-terragrunt@v1
+        with:
+          terragrunt_version: ${{ env.tg_version }}
+
+      - name: 'Install OpenTofu'
+        uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: ${{ env.tofu_version }}
+          tofu_wrapper: false
+
+      - name: 'Get build artifact'
+        uses: actions/download-artifact@v4
+        with:
+          name: tiles-build-output
+          path: "${{ github.workspace }}/dist"
+
+      - name: Plan All
+        working-directory: ${{ env.working_dir }}
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          VMETRICS_API_TOKEN: ${{ secrets.VMETRICS_API_TOKEN }}
+          TILES_BUILD_DIR: ${{ github.workspace }}/dist
+          TF_STATE_POSTGRES_CONN_STR: ${{ secrets.TF_STATE_POSTGRES_CONN_STR }}
+        run: terragrunt run-all plan -no-color 2>&1 | tee "${{github.workspace}}/plan_output.txt" && exit ${PIPESTATUS[0]};
+
+  kv-warming:
+    needs: [build, test]
+    name: KV Warming
+    runs-on: mich
+    if: github.ref == 'refs/heads/main'
+    defaults:
+      run:
+        working-directory: ./tiles
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: './tiles/.nvmrc'
+
+      - name: Run npm install
+        run: npm ci
+
+      - name: Get tiles.json
+        run: echo "TILES_JSON=$(jq -c . < ${{ github.workspace }}/deployment/modules/cloudflare/tiles/tiles.tfvars.json)" >> $GITHUB_ENV
+
+      - name: Run kv warming
+        env:
+          S3_ACCESS_KEY: ${{ secrets.CLOUDFLARE_TILES_R2_KV_TOKEN_ID }}
+          S3_SECRET_KEY: ${{ secrets.CLOUDFLARE_TILES_R2_KV_TOKEN_HASHED_VALUE }}
+          S3_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          KV_API_KEY: ${{ secrets.CLOUDFLARE_TILES_R2_KV_TOKEN_VALUE }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          # Figure out how to extract this from terraform at some point or get it into github vars
+          KV_NAMESPACE_ID: 5a4b82694e8b490db8b8904cdaea4f00
+          BUCKET_KEY: tiles
+          FILE_NAME: ${{ fromJson(env.TILES_JSON).pmtiles_file_name }}
+        run: npm run kv:warm
+
+  deploy-terragrunt:
+    needs: [build, test]
+    name: Deploy Terragrunt
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@main
+
+      - name: Install Terragrunt
+        uses: eLco/setup-terragrunt@v1
+        with:
+          terragrunt_version: ${{ env.tg_version }}
+
+      - name: 'Install OpenTofu'
+        uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: ${{ env.tofu_version }}
+          tofu_wrapper: false
+
+      - name: 'Get build artifact'
+        uses: actions/download-artifact@v4
+        with:
+          name: tiles-build-output
+          path: "${{ github.workspace }}/dist"
+      - name: Deploy All
+        working-directory: ${{ env.working_dir }}
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          VMETRICS_API_TOKEN: ${{ secrets.VMETRICS_API_TOKEN }}
+          TILES_BUILD_DIR: "${{ github.workspace }}/dist"
+          TF_STATE_POSTGRES_CONN_STR: ${{ secrets.TF_STATE_POSTGRES_CONN_STR }}
+        run: terragrunt run-all apply --terragrunt-non-interactive