From 38a6c41f908993ec25a3ca709c63086b55fe1496 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Mon, 2 Dec 2024 16:25:20 +0000 Subject: [PATCH 1/2] feat: liquibase runs in containerised environments via aws batch. WIP --- .github/workflows/run-liquibase.yaml | 76 ++++++++++++++++++++++++++++ infra/docker/liquibase/Dockerfile | 19 +++++++ infra/docker/liquibase/entrypoint.sh | 22 ++++++++ 3 files changed, 117 insertions(+) create mode 100644 .github/workflows/run-liquibase.yaml create mode 100644 infra/docker/liquibase/Dockerfile create mode 100644 infra/docker/liquibase/entrypoint.sh diff --git a/.github/workflows/run-liquibase.yaml b/.github/workflows/run-liquibase.yaml new file mode 100644 index 0000000000..130fce95fb --- /dev/null +++ b/.github/workflows/run-liquibase.yaml @@ -0,0 +1,76 @@ +name: Run Database Migrations + +on: + workflow_call: + inputs: + environment: + type: string + required: true + ref: + type: string + required: false + dry-run: + type: boolean + required: false + default: true + workflow_dispatch: + inputs: + environment: + description: "Environment to run migrations against" + type: choice + options: [dev, int, prep] + required: true + ref: + description: "Branch to run migrations from" + type: string + required: false + dry-run: + description: "Run migrations without applying changes" + type: boolean + required: true + default: true + +permissions: + id-token: write + contents: read + +jobs: + migrate: + runs-on: ubuntu-latest + environment: ${{ !inputs.dry-run && inputs.environment || null }} + env: + AWS_REGION: ${{ vars.DVSA_AWS_REGION }} + AWS_OIDC_ROLE: ${{ vars[format('ENV_{0}_TF_OIDC_ROLE', inputs.environment)] || vars[(inputs.environment == 'prep' || inputs.environment == 'prod') && 'ACCOUNT_PROD_TF_OIDC_ROLE' || 'ACCOUNT_NONPROD_TF_OIDC_ROLE'] }} + BATCH_QUEUE: ${{ vars[format('ACCOUNT_{0}_BATCH_JOB_QUEUE', (inputs.environment == 'prep' || inputs.environment == 'prod') && 'PROD' || 'NONPROD')] }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || null }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE }} + aws-region: ${{ env.AWS_REGION }} + + - name: Submit Batch Job + id: submit-job + run: | + JOB_ID=$(aws batch submit-job \ + --job-name "db-migration-${{ inputs.environment }}-${{ github.run_id }}" \ + --job-queue "${{ env.BATCH_QUEUE }}" \ + --job-definition "db-migration-${{ inputs.environment }}" \ + --parameters environment="${{ inputs.environment }}" \ + --query 'jobId' --output text) + + echo "job-id=${JOB_ID}" >> $GITHUB_OUTPUT + + - name: Wait for job completion + run: | + aws batch wait job-complete --jobs ${{ steps.submit-job.outputs.job-id }} + STATUS=$(aws batch describe-jobs --jobs ${{ steps.submit-job.outputs.job-id }} --query 'jobs[0].status' --output text) + if [ "$STATUS" != "SUCCEEDED" ]; then + echo "Job failed with status: ${STATUS}" + exit 1 + fi diff --git a/infra/docker/liquibase/Dockerfile b/infra/docker/liquibase/Dockerfile new file mode 100644 index 0000000000..ed72eb9783 --- /dev/null +++ b/infra/docker/liquibase/Dockerfile @@ -0,0 +1,19 @@ +FROM alpine/git AS repo +WORKDIR /app +RUN git clone https://github.com/dvsa/olcs-etl . + +FROM liquibase/liquibase + +RUN apt-get update && apt-get install -y \ + aws-cli \ + jq \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=repo /app /liquibase/changelog + +COPY entrypoint.sh /liquibase/ +RUN chmod +x /liquibase/entrypoint.sh + +ENV INSTALL_MYSQL=true + +ENTRYPOINT ["/liquibase/entrypoint.sh"] diff --git a/infra/docker/liquibase/entrypoint.sh b/infra/docker/liquibase/entrypoint.sh new file mode 100644 index 0000000000..bd9dbe1880 --- /dev/null +++ b/infra/docker/liquibase/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +cat > /liquibase/liquibase.properties << EOF +driver=com.mysql.cj.jdbc.Driver +url=jdbc:mysql://${DB_HOST}:${DB_PORT}/${DB_NAME} +username=${DB_USER} +password=${DB_PASSWORD} +classpath=/liquibase/changelog/mysql-connector-java-8.0.21/mysql-connector-java-8.0.21.jar +changeLogFile=changesets/OLCS.xml +logLevel=info +liquibase.hub.mode=off +EOF + +if [[ "$1" == "--dry-run" ]]; then + echo "Running in dry-run mode - showing pending changes:" + liquibase status --verbose + liquibase update-sql +else + echo "Running migrations..." + liquibase update +fi From 7de9ca5b10a34aec4b9171c4563a01db5f73b0fe Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Fri, 13 Dec 2024 16:33:18 +0000 Subject: [PATCH 2/2] feat: liquibase container work --- .github/workflows/run-liquibase.yaml | 136 ++++++++++++++++----------- infra/docker/liquibase/Dockerfile | 14 +-- infra/docker/liquibase/entrypoint.sh | 25 +++-- 3 files changed, 99 insertions(+), 76 deletions(-) diff --git a/.github/workflows/run-liquibase.yaml b/.github/workflows/run-liquibase.yaml index 130fce95fb..c71616d62c 100644 --- a/.github/workflows/run-liquibase.yaml +++ b/.github/workflows/run-liquibase.yaml @@ -1,76 +1,106 @@ -name: Run Database Migrations - +name: Run Liquibase Migrations on: - workflow_call: + workflow_dispatch: inputs: - environment: + version: + description: "Version tag for the container" + required: true type: string + account: + description: "Target AWS account" + required: true + type: choice + options: + - nonprod + - prod + dry_run: + description: "Run in dry-run mode (show pending changes only)" required: true + type: boolean + default: true ref: - type: string + description: "Git ref to checkout (branch, tag, or commit SHA)" required: false - dry-run: + type: string + default: "main" + submit_job: + description: "Submit AWS Batch job after building" + required: true type: boolean - required: false - default: true - workflow_dispatch: + default: false + workflow_call: inputs: - environment: - description: "Environment to run migrations against" - type: choice - options: [dev, int, prep] + version: + type: string required: true + push: + type: boolean + required: true + default: false + account: + type: string + required: true + enum: ["nonprod", "prod"] ref: - description: "Branch to run migrations from" type: string required: false - dry-run: - description: "Run migrations without applying changes" - type: boolean - required: true - default: true - -permissions: - id-token: write - contents: read - + default: "main" +env: + REGISTRY: 054614622558.dkr.ecr.eu-west-1.amazonaws.com + AWS_REGION: ${{ vars.DVSA_AWS_REGION }} + AWS_OIDC_ROLE: ${{ (inputs.account || github.event.inputs.account) == 'prod' && vars.ACCOUNT_PROD_TF_OIDC_ROLE || vars.ACCOUNT_NONPROD_TF_OIDC_ROLE }} jobs: - migrate: + build: runs-on: ubuntu-latest - environment: ${{ !inputs.dry-run && inputs.environment || null }} - env: - AWS_REGION: ${{ vars.DVSA_AWS_REGION }} - AWS_OIDC_ROLE: ${{ vars[format('ENV_{0}_TF_OIDC_ROLE', inputs.environment)] || vars[(inputs.environment == 'prep' || inputs.environment == 'prod') && 'ACCOUNT_PROD_TF_OIDC_ROLE' || 'ACCOUNT_NONPROD_TF_OIDC_ROLE'] }} - BATCH_QUEUE: ${{ vars[format('ACCOUNT_{0}_BATCH_JOB_QUEUE', (inputs.environment == 'prep' || inputs.environment == 'prod') && 'PROD' || 'NONPROD')] }} - steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.ref || null }} - + ref: ${{ inputs.ref || github.event.inputs.ref }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Configure AWS credentials + if: ${{ inputs.push || github.event_name == 'workflow_dispatch' }} uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.AWS_OIDC_ROLE }} aws-region: ${{ env.AWS_REGION }} - - - name: Submit Batch Job - id: submit-job - run: | - JOB_ID=$(aws batch submit-job \ - --job-name "db-migration-${{ inputs.environment }}-${{ github.run_id }}" \ - --job-queue "${{ env.BATCH_QUEUE }}" \ - --job-definition "db-migration-${{ inputs.environment }}" \ - --parameters environment="${{ inputs.environment }}" \ - --query 'jobId' --output text) - - echo "job-id=${JOB_ID}" >> $GITHUB_OUTPUT - - - name: Wait for job completion + - name: Login to ECR + if: ${{ inputs.push || github.event_name == 'workflow_dispatch' }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./infra/docker/liquibase + push: ${{ inputs.push || github.event_name == 'workflow_dispatch' }} + tags: | + ${{ env.REGISTRY }}/vol-app/liquibase:${{ inputs.version || github.event.inputs.version }} + ${{ env.REGISTRY }}/vol-app/liquibase:latest + cache-from: type=gha,scope=liquibase + cache-to: type=gha,mode=max,scope=liquibase + build-args: | + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + submit-batch-job: + needs: build + if: | + (inputs.push || (github.event_name == 'workflow_dispatch' && github.event.inputs.submit_job == 'true')) + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Submit AWS Batch job run: | - aws batch wait job-complete --jobs ${{ steps.submit-job.outputs.job-id }} - STATUS=$(aws batch describe-jobs --jobs ${{ steps.submit-job.outputs.job-id }} --query 'jobs[0].status' --output text) - if [ "$STATUS" != "SUCCEEDED" ]; then - echo "Job failed with status: ${STATUS}" - exit 1 - fi + aws batch submit-job \ + --job-name "liquibase-migration-${{ inputs.version || github.event.inputs.version }}" \ + --job-queue "${{ (inputs.account || github.event.inputs.account) == 'prod' && vars.ACCOUNT_PROD_BATCH_JOB_QUEUE || vars.ACCOUNT_NONPROD_BATCH_JOB_QUEUE }}" \ + --job-definition "${{ (inputs.account || github.event.inputs.account) == 'prod' && vars.ACCOUNT_PROD_BATCH_LIQUIBASE_JOB_DEFINITION || vars.ACCOUNT_NONPROD_BATCH_LIQUIBASE_JOB_DEFINITION }}" \ + --container-overrides "{ + \"image\": \"${{ env.REGISTRY }}/vol-app/liquibase:${{ inputs.version || github.event.inputs.version }}\", + \"environment\": [ + {\"name\": \"DRY_RUN\", \"value\": \"${{ github.event.inputs.dry_run || 'false' }}\"} + ] + }" diff --git a/infra/docker/liquibase/Dockerfile b/infra/docker/liquibase/Dockerfile index ed72eb9783..2c4cc4315a 100644 --- a/infra/docker/liquibase/Dockerfile +++ b/infra/docker/liquibase/Dockerfile @@ -1,19 +1,13 @@ FROM alpine/git AS repo +ARG GITHUB_TOKEN WORKDIR /app -RUN git clone https://github.com/dvsa/olcs-etl . +RUN git clone https://${GITHUB_TOKEN}@github.com/dvsa/olcs-etl . FROM liquibase/liquibase - -RUN apt-get update && apt-get install -y \ - aws-cli \ - jq \ - && rm -rf /var/lib/apt/lists/* - +USER root COPY --from=repo /app /liquibase/changelog - COPY entrypoint.sh /liquibase/ RUN chmod +x /liquibase/entrypoint.sh - ENV INSTALL_MYSQL=true - +USER liquibase ENTRYPOINT ["/liquibase/entrypoint.sh"] diff --git a/infra/docker/liquibase/entrypoint.sh b/infra/docker/liquibase/entrypoint.sh index bd9dbe1880..3c0281c019 100644 --- a/infra/docker/liquibase/entrypoint.sh +++ b/infra/docker/liquibase/entrypoint.sh @@ -1,22 +1,21 @@ #!/bin/bash set -e -cat > /liquibase/liquibase.properties << EOF -driver=com.mysql.cj.jdbc.Driver -url=jdbc:mysql://${DB_HOST}:${DB_PORT}/${DB_NAME} -username=${DB_USER} -password=${DB_PASSWORD} -classpath=/liquibase/changelog/mysql-connector-java-8.0.21/mysql-connector-java-8.0.21.jar -changeLogFile=changesets/OLCS.xml -logLevel=info -liquibase.hub.mode=off -EOF +cd /liquibase/changelog + +LIQUIBASE_OPTS="--driver=com.mysql.cj.jdbc.Driver \ + --classpath=/liquibase/changelog/mysql-connector-java-8.0.21/mysql-connector-java-8.0.21.jar \ + --url=jdbc:mysql://${DB_HOST}:${DB_PORT}/${DB_NAME} \ + --username=${DB_USER} \ + --password=${DB_PASSWORD} \ + --changelog-file=changesets/OLCS.xml \ + --log-level=info" if [[ "$1" == "--dry-run" ]]; then echo "Running in dry-run mode - showing pending changes:" - liquibase status --verbose - liquibase update-sql + liquibase ${LIQUIBASE_OPTS} status --verbose + liquibase ${LIQUIBASE_OPTS} update-sql else echo "Running migrations..." - liquibase update + liquibase ${LIQUIBASE_OPTS} update fi