[PRMP-594] - ODS improvements MI API (#13)

New DynamoDB tables have been created to hold weekly updated ODS information, and monthly updated ICB information.

The enrichment lambda has been modified to query the respective tables for enrichment data, and using the previous API method as a fallback if no data exists.

GitHub Actions has also been introduced as a replacement for GoCD.

.github/workflows/PR-terraform-plan.yml

@@ -0,0 +1,28 @@
+name: "PR-terraform-plan"
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'stacks/**'
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform_plan:
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        environment: [ dev, prod-plan ]
+        terraform_stack: [container-repositories, base-networking, ecs-cluster]
+    uses: ./.github/workflows/base-terraform-plan-and-apply.yml
+    with:
+      environment: ${{ matrix.environment }}
+      terraform_stack: ${{ matrix.terraform_stack }}
+    secrets: inherit
+
+

.github/workflows/base-gp-registrations-mi.yml

@@ -0,0 +1,171 @@
+name: base-gp-registrations-mi
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "Which Environment settings to use."
+        required: true
+        type: string
+        default: "dev"
+      is_deployment:
+        description: "Is workflow run on deployment"
+        type: boolean
+        default: false
+
+jobs:
+  terraform_process:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume:  ${{(inputs.is_deployment && secrets.AWS_ASSUME_ROLE) || secrets.AWS_ASSUME_ROLE_READ_ONLY}}
+          role-skip-session-tagging: true
+
+      - name: Publish Docker Image to Prod
+        id: push-image-to-prod
+        if: inputs.environment == 'prod' && inputs.is_deployment
+
+        run: |
+          aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ECR_DEV_REPOSITORY }}
+          IMAGE_TAG=$(aws ecr describe-images --registy-id ${{ secrets.AWS_DEV_ACCOUNT_ID }} --repository-name ${{ secrets.ECR_REPOSITORY_NAME }} --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags[0]')
+          source_repo=${{ secrets.AWS_ECR_DEV_REPOSITORY }}/${{ secrets.ECR_REPOSITORY_DEV_NAME }}:${IMAGE_TAG//\"}
+          destination_repo=${{ secrets.AWS_ECR_PROD_REPOSITORY}}/${{ secrets.ECR_REPOSITORY_NAME }}:${IMAGE_TAG//\"}
+          docker pull $source_repo
+          docker tag $source_repo $destination_repo
+          aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ECR_PROD_REPOSITORY }}
+          docker push $destination_repo
+          echo "image-tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: latest
+
+      - name: Terraform Format
+        id: fmt
+        run: terraform fmt -check
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Terraform Init
+        id: init
+        run: | 
+          terraform init -no-color -backend-config="key=${{ secrets.AWS_STATE_S3_KEY }}" \
+          -backend-config="bucket=${{ secrets.AWS_STATE_BUCKET }}" \
+          -backend-config="dynamodb_table=${{ secrets.AWS_STATE_LOCK_TABLE }}"
+        working-directory: ./stacks/gp-registrations-mi/terraform
+        shell: bash
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Build Lambdas
+        run: | 
+            ./tasks_github_actions.sh build-lambdas
+
+      - name: Set up Python
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Make virtual environment
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: |
+            python3 -m venv ./venv
+            ./venv/bin/pip3 install --upgrade pip requests
+
+      - name: Get ODS CSV Files
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: |
+          PYTHONPATH=$PYTHONPATH:. ./venv/bin/python3 stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py ${{ secrets.TRUD_API_KEY }} ${{ vars.TRUD_API_URL }}
+
+      - name: Setup Terraform variables
+        id: vars-prod
+        if: inputs.environment == 'prod' && inputs.is_deployment
+        run: |-
+          cat > pipeline.auto.tfvars <<EOF
+          gp_registrations_mi_image_tag = ${{ steps.push-image-to-prod.outputs.image-tag }}
+          environment = "${{ vars.AWS_ENVIRONMENT }}"
+          EOF
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Setup Terraform variables
+        id: vars-dev
+        if: inputs.environment != 'prod'
+        run: |-
+          IMAGE_TAG=$(aws ecr describe-images --repository-name ${{ secrets.ECR_REPOSITORY_NAME }} --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags[0]')
+          cat > pipeline.auto.tfvars <<EOF
+          gp_registrations_mi_image_tag = $IMAGE_TAG
+          environment = "${{ vars.AWS_ENVIRONMENT }}"
+          EOF
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Terraform Plan
+        id: plan
+        run: |
+          terraform plan -no-color -input=false -var-file="../vars/${{ vars.AWS_ENVIRONMENT }}.tfvars" -out "${{ vars.AWS_ENVIRONMENT }}.tfplan"
+          terraform show -no-color ${{ vars.AWS_ENVIRONMENT }}.tfplan > ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt
+          echo "summary=$(grep -E 'Plan: [0-9]+ to add, [0-9]+ to change, [0-9]+ to destroy\.|No changes\. Your infrastructure matches the configuration\.' ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt | sed 's/.*No changes\. Your infrastructure matches the configuration/Plan: no changes/g' | sed 's/.*Plan: //g' | sed 's/\..*//g')" >> $GITHUB_OUTPUT
+        working-directory: ./stacks/gp-registrations-mi/terraform
+        shell: bash
+
+      - name: Add PR comment
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request' && (success() || failure())
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // 1. Retrieve existing bot comments for the PR
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Report for gp-registrations-mi environment: ${{ inputs.environment }}')
+            });
+
+            // 2. Prepare format of the comment
+            const output = `### Report for gp-registrations-mi environment: ${{ inputs.environment }}
+
+            #### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+
+
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+
+
+            #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+
+
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            Plan results: ${{ steps.plan.outputs.summary }}`;
+
+            // 3. If we have a comment, update it, otherwise create a new one
+            if (botComment) {
+              github.rest.issues.deleteComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+              })
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            });
+
+      - name: Terraform Apply
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: terraform apply -auto-approve -input=false ${{ vars.AWS_ENVIRONMENT }}.tfplan
+        working-directory: ./stacks/gp-registrations-mi/terraform

.github/workflows/base-terraform-plan-and-apply.yml

@@ -0,0 +1,122 @@
+name: base-terraform-plan-and-apply
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "Which Environment settings to use"
+        required: true
+        type: string
+        default: "dev"
+      is_deployment:
+        description: "Is workflow run on deployment"
+        type: boolean
+        default: false
+      terraform_stack:
+        description: "Which terraform stack directory to run"
+        type: string
+        required: true
+
+jobs:
+  terraform_process:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    defaults:
+      run:
+        working-directory: ./stacks/${{ inputs.terraform_stack }}/terraform
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume:  ${{inputs.is_deployment && secrets.AWS_ASSUME_ROLE || secrets.AWS_ASSUME_ROLE_READ_ONLY}}
+          role-skip-session-tagging: true
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: latest
+
+      - name: Terraform Format
+        id: fmt
+        run: terraform fmt -check
+
+      - name: Terraform Init
+        id: init
+        run: | 
+          terraform init -no-color -backend-config="key=gp-registrations-mi/${{ inputs.terraform_stack }}/terraform.tfstate" \
+          -backend-config="bucket=${{ secrets.AWS_STATE_BUCKET }}" \
+          -backend-config="dynamodb_table=${{ secrets.AWS_STATE_LOCK_TABLE }}"
+        shell: bash
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - name: Setup Terraform variables
+        id: vars
+        run: |-
+          cat > pipeline.auto.tfvars <<EOF
+          environment = "${{ vars.AWS_ENVIRONMENT }}"
+          EOF
+
+      - name: Terraform Plan
+        id: plan
+        run: |
+          terraform plan -no-color -input=false -var-file="../vars/${{ vars.AWS_ENVIRONMENT }}.tfvars" -out "${{ vars.AWS_ENVIRONMENT }}.tfplan"
+          terraform show -no-color ${{ vars.AWS_ENVIRONMENT }}.tfplan > ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt
+          echo "summary=$(grep -E 'Plan: [0-9]+ to add, [0-9]+ to change, [0-9]+ to destroy\.|No changes\. Your infrastructure matches the configuration\.' ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt | sed 's/.*No changes\. Your infrastructure matches the configuration/Plan: no changes/g' | sed 's/.*Plan: //g' | sed 's/\..*//g')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Add PR comment
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request' && (success() || failure())
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // 1. Retrieve existing bot comments for the PR
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Report for ${{inputs.terraform_stack}} environment: ${{ inputs.environment }}')
+            });
+
+            // 2. Prepare format of the comment
+            const output = `### Report for ${{inputs.terraform_stack}} environment: ${{ inputs.environment }}
+
+            #### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+
+
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+
+
+            #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+
+
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            Plan results: ${{ steps.plan.outputs.summary }}`;
+
+            // 3. If we have a comment, update it, otherwise create a new one
+            if (botComment) {
+              github.rest.issues.deleteComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+              })
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            });
+
+      - name: Terraform Apply
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: terraform apply -auto-approve -input=false ${{ vars.AWS_ENVIRONMENT }}.tfplan

.github/workflows/deploy-gp-registrations-mi.yml

@@ -0,0 +1,25 @@
+name: deploy-gp-registrations-mi
+on:
+  workflow_dispatch:
+      inputs:
+        environment:
+          default: "dev"
+          description: "Which environment should this run against"
+          required: true
+          type: choice
+          options:
+            - dev
+            - prod
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform_process:
+    uses: ./.github/workflows/base-gp-registrations-mi.yml
+    with:
+      environment: ${{ inputs.environment }}
+      is_deployment: true
+    secrets: inherit

.github/workflows/deploy-terraform-by-stack.yml

@@ -0,0 +1,37 @@
+name: deploy-terraform-by-stack
+on:
+  workflow_dispatch:
+      inputs:
+        environment:
+          default: "dev"
+          description: "Which environment should this run against"
+          required: true
+          type: choice
+          options:
+            - dev
+            - prod
+        terraform_stack:
+          description: "Which terraform stack directory to run"
+          type: choice
+          required: true
+          options:
+            - container-repositories
+            - base-networking
+            - ecs-cluster
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform_process:
+    uses: ./.github/workflows/base-terraform-plan-and-apply.yml
+    with:
+      environment: ${{ inputs.environment }}
+      is_deployment: true
+      terraform_stack: ${{ inputs.terraform_stack }}
+    secrets: inherit
+
+
+