DAT-17834 Ephemeral AWS infra for liquibase-test-harness

.github/workflows/aws-weekly.yml

@@ -1,12 +1,11 @@
 # Terraform Action to test Cloud Databases with test-harness
 
 name: AWS Weekly Cloud Database Test Execution
-concurrency: aws-run
 on:
   # Triggers the workflow on a schedule for the main branch. Also allows for manual triggers
-  # Execute every sunday at 6:00 AM
+  # Execute every sunday at 4:00 AM
   schedule:
-    - cron: '0 6 * * 0'
+    - cron: '0 4 * * 0'
   workflow_dispatch:
     inputs:
       testClasses:
@@ -38,15 +37,24 @@ jobs:
         uses: actions/checkout@v4
 
   deploy-ephemeral-cloud-infra:
-    uses: liquibase/build-logic/.github/workflows/ephemeral-cloud-infra.yml@DAT-17834
+    uses: liquibase/build-logic/.github/workflows/ephemeral-cloud-infra.yml@main
     secrets: inherit
     with:
         deploy: true
-        aws_test_harness: true
+        aws_postgresql: ${{ contains(inputs.databases, 'postgresql:12') || contains(inputs.databases, 'postgresql:13') || contains(inputs.databases, 'postgresql:14')}}
+        aws_oracle: ${{ contains(inputs.databases, 'oracle') }}
+        aws_mariadb: ${{ contains(inputs.databases, 'mariadb') }}
+        aws_aurora_mysql: ${{ contains(inputs.databases, 'mysql:aurora') }}
+        aws_mssql: ${{ contains(inputs.databases, 'mssql') }}
+        aws_aurora_postgres: ${{ contains(inputs.databases, 'postgresql:aurora') }}
+        aws_mysql: ${{ contains(inputs.databases, 'mysql:aws') }}
 
   init-mysql:
     runs-on: ubuntu-latest
     needs: [deploy-ephemeral-cloud-infra]
+    permissions:
+      contents: read
+      id-token: write
     container:
       image: liquibase/liquibase:latest
       env:
@@ -69,52 +77,44 @@ jobs:
             role-to-assume: ${{ secrets.AWS_DEV_GITHUB_OIDC_ROLE_ARN_BUILD_LOGIC }}
             aws-region: us-east-1
 
+      - name: Configure Test
+        id: setup
+        uses: actions/[email protected]
+        with:
+            script: |
+                let splitValues = "${{ matrix.database }}".split(":")
+                core.setOutput("databasePlatform", splitValues[0]);
+                core.setOutput("databaseVersion", splitValues[1]);
+
       - name: Get AWS secrets
+        if: ${{ matrix.version == 'aws' && matrix.database == 'mysql'}}
         uses: aws-actions/aws-secretsmanager-get-secrets@v2
         with:
             secret-ids: |
-                TH_MYSQLURL_8_0, /testautomation/db_details/aws_mysql_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_AURORA_MYSQLURL, /testautomation/db_details/aurora_mysql_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_ORACLEURL_19, /testautomation/db_details/aws_oracle_19_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_PGRESURL_12, /testautomation/db_details/aws_postgresql_12_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_PGRESURL_13, /testautomation/db_details/aws_postgresql_13_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_PGRESURL_14, /testautomation/db_details/aws_postgresql_14_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_MARIADBURL_10_6, /testautomation/db_details/aws_mariadb_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_MSSQLURL, /testautomation/db_details/aws_mssql_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-                TH_AURORA_POSTGRESQLURL, /testautomation/db_details/aurora_postgresql_jdbc${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
-
+                TH_MYSQLURL_8_0, /testautomation/db_details/aws_mysql_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+      - name: Get AWS secrets
+        if: ${{ matrix.version == 'aurora' && matrix.database == 'mysql'}}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_AURORA_MYSQLURL, /testautomation/db_details/aurora_mysql_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
       #This additional init step is required because of mysql driver issue on GH actions
       - name: Install Dependencies
         run: lpm update && lpm add mysql
 
-      - name: Clean AWS MySQL Database
-        uses: liquibase-github-actions/[email protected]
-        if: ${{ matrix.version == 'aws' }}
-        with:
-          url: "${{ env.TH_MYSQLURL_8_0 }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
-
       - name: Init Database
-        if: ${{ matrix.version == 'aws' }}
-        run: liquibase --classpath="src/test/resources/init-changelogs/aws" --changeLogFile="${{ matrix.database }}.sql" --username="${{ secrets.TH_DB_ADMIN }}" --password="${{ secrets.TH_DB_PASSWD }}" --url="${{ env.TH_MYSQLURL_8_0 }}" update
-
-      - name: Clean Aurora MySQL Database
-        uses: liquibase-github-actions/[email protected]
-        if: ${{ matrix.version == 'aurora' }}
-        with:
-          url: "${{ env.TH_AURORA_MYSQLURL }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
+        if: ${{ matrix.version == 'aws' && matrix.database == 'mysql'}}
+        run: liquibase --classpath="src/test/resources/init-changelogs/aws" --changeLogFile="${{ matrix.database }}_cloud.sql" --username="${{ secrets.TH_DB_ADMIN }}" --password="${{ secrets.TH_DB_PASSWD }}" --url="${{ env.TH_MYSQLURL_8_0 }}" update
 
       - name: Init Database
-        if: ${{ matrix.version == 'aurora' }}
-        run: liquibase --classpath="src/test/resources/init-changelogs/aws" --changeLogFile="${{ matrix.database }}.sql" --username="${{ secrets.TH_DB_ADMIN }}" --password="${{ secrets.TH_DB_PASSWD }}" --url="${{ env.TH_AURORA_MYSQLURL }}" update
+        if: ${{ matrix.version == 'aurora' && matrix.database == 'mysql'}}
+        run: liquibase --classpath="src/test/resources/init-changelogs/aws" --changeLogFile="${{ matrix.database }}_cloud.sql" --username="${{ secrets.TH_DB_ADMIN }}" --password="${{ secrets.TH_DB_PASSWD }}" --url="${{ env.TH_AURORA_MYSQLURL }}" update
 
   test:
-    needs: [init-mysql, setup]
+    needs: [deploy-ephemeral-cloud-infra, init-mysql, setup]
+    permissions:
+        contents: read
+        id-token: write
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -133,14 +133,64 @@ jobs:
             let splitValues = "${{ matrix.database }}".split(":")
             core.setOutput("databasePlatform", splitValues[0]);
             core.setOutput("databaseVersion", splitValues[1]);
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+            role-to-assume: ${{ secrets.AWS_DEV_GITHUB_OIDC_ROLE_ARN_BUILD_LOGIC }}
+            aws-region: us-east-1
+
+      - name: Get AWS secrets
+        if: ${{ steps.setup.outputs.databasePlatform == 'mysql' && steps.setup.outputs.databaseVersion == 'aws' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_MYSQLURL_8_0, /testautomation/db_details/aws_mysql_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+
+      - name: Get AWS secrets
+        if: ${{ steps.setup.outputs.databasePlatform == 'mysql' && steps.setup.outputs.databaseVersion == 'aurora' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_AURORA_MYSQLURL, /testautomation/db_details/aurora_mysql_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
 
-      - uses: liquibase-github-actions/[email protected]
+      - name: Get AWS secrets
         if: ${{ steps.setup.outputs.databasePlatform == 'oracle' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
         with:
-          url: "${{ env.TH_ORACLEURL_19 }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
+            secret-ids: |
+                TH_ORACLEURL_19, /testautomation/db_details/aws_oracle_19_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+
+      - name: Get AWS secrets
+        if: ${{ steps.setup.outputs.databasePlatform == 'postgresql' && steps.setup.outputs.databaseVersion != 'aurora' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_PGRESURL_12, /testautomation/db_details/aws_postgresql_12_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+                TH_PGRESURL_13, /testautomation/db_details/aws_postgresql_13_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+                TH_PGRESURL_14, /testautomation/db_details/aws_postgresql_14_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+
+      - name: Get AWS secrets
+        if: ${{ steps.setup.outputs.databasePlatform == 'mariadb' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_MARIADBURL_10_6, /testautomation/db_details/aws_mariadb_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+
+      - name: Get AWS secrets
+        if: ${{ steps.setup.outputs.databasePlatform == 'mssql' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_MSSQLURL, /testautomation/db_details/aws_mssql_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+                TH_MSSQLURL_HOST, /testautomation/db_details/aws_mssql_host_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
+
+      - name: Get AWS secrets
+        if: ${{ steps.setup.outputs.databasePlatform == 'postgresql' && steps.setup.outputs.databaseVersion == 'aurora' }}
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+            secret-ids: |
+                TH_AURORA_POSTGRESQLURL, /testautomation/db_details/aurora_postgresql_jdbc_${{ needs.deploy-ephemeral-cloud-infra.outputs.resources_id }}
 
       - uses: liquibase/liquibase-github-action@v7
         if: ${{ steps.setup.outputs.databasePlatform == 'oracle' }}
@@ -152,14 +202,6 @@ jobs:
           password: "${{secrets.TH_DB_PASSWD}}"
           url: "${{ env.TH_ORACLEURL_19 }}"
 
-      - uses: liquibase-github-actions/[email protected]
-        if: ${{ steps.setup.outputs.databasePlatform == 'postgresql' && steps.setup.outputs.databaseVersion != 'aurora' }}
-        with:
-          url: "${{ env[format('TH_PGRESURL_{0}', steps.setup.outputs.databaseVersion)] }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
-
       - uses: liquibase/liquibase-github-action@v7
         if: ${{ steps.setup.outputs.databasePlatform == 'postgresql' && steps.setup.outputs.databaseVersion != 'aurora' }}
         with:
@@ -170,14 +212,6 @@ jobs:
           password: "${{secrets.TH_DB_PASSWD}}"
           url: "${{ env[format('TH_PGRESURL_{0}', steps.setup.outputs.databaseVersion)] }}"
 
-      - uses: liquibase-github-actions/[email protected]
-        if: ${{ steps.setup.outputs.databasePlatform == 'mariadb' }}
-        with:
-          url: "${{ env.TH_MARIADBURL_10_6 }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
-
       - uses: liquibase/liquibase-github-action@v7
         if: ${{ steps.setup.outputs.databasePlatform == 'mariadb' }}
         with:
@@ -188,13 +222,15 @@ jobs:
           password: "${{secrets.TH_DB_PASSWD}}"
           url: "${{ env.TH_MARIADBURL_10_6 }}"
 
-      - uses: liquibase-github-actions/[email protected]
+      - name: Install a SQL Server suite of tools
         if: ${{ steps.setup.outputs.databasePlatform == 'mssql' }}
+        uses: potatoqualitee/[email protected]
         with:
-          url: "${{ env.TH_MSSQLURL }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
+            install: sqlengine, sqlclient, sqlpackage
+
+      - name: Run sqlcmd
+        if: ${{ steps.setup.outputs.databasePlatform == 'mssql' }}
+        run: sqlcmd -S ${{ env.TH_MSSQLURL_HOST }} -U ${{secrets.TH_DB_ADMIN}} -P ${{secrets.TH_DB_PASSWD}} -Q "CREATE DATABASE lbcat"
 
       - uses: liquibase/liquibase-github-action@v7
         if: ${{ steps.setup.outputs.databasePlatform == 'mssql' }}
@@ -206,14 +242,6 @@ jobs:
           password: "${{secrets.TH_DB_PASSWD}}"
           url: "${{ env.TH_MSSQLURL }}"
 
-      - uses: liquibase-github-actions/[email protected]
-        if: ${{ steps.setup.outputs.databasePlatform == 'postgresql' && steps.setup.outputs.databaseVersion == 'aurora' }}
-        with:
-          url: "${{ env.TH_AURORA_POSTGRESQLURL }}"
-          username: "${{secrets.TH_DB_ADMIN}}"
-          password: "${{secrets.TH_DB_PASSWD}}"
-          licenseKey: "${{secrets.LICENSE_KEY}}"
-
       - uses: liquibase/liquibase-github-action@v7
         if: ${{ steps.setup.outputs.databasePlatform == 'postgresql' && steps.setup.outputs.databaseVersion == 'aurora' }}
         with:
@@ -283,9 +311,15 @@ jobs:
   destroy-ephemeral-cloud-infra:
     if: always()
     needs: [ deploy-ephemeral-cloud-infra, test ]
-    uses: liquibase/build-logic/.github/workflows/ephemeral-cloud-infra.yml@DAT-17834
+    uses: liquibase/build-logic/.github/workflows/ephemeral-cloud-infra.yml@main
     secrets: inherit
     with:
         destroy: true
         stack_id: ${{ needs.deploy-ephemeral-cloud-infra.outputs.stack_id }}
-        aws_test_harness: true
+        aws_postgresql: ${{ contains(inputs.databases, 'postgresql:12') || contains(inputs.databases, 'postgresql:13') || contains(inputs.databases, 'postgresql:14')}}
+        aws_oracle: ${{ contains(inputs.databases, 'oracle') }}
+        aws_mariadb: ${{ contains(inputs.databases, 'mariadb') }}
+        aws_aurora_mysql: ${{ contains(inputs.databases, 'mysql:aurora') }}
+        aws_mssql: ${{ contains(inputs.databases, 'mssql') }}
+        aws_aurora_postgres: ${{ contains(inputs.databases, 'postgresql:aurora') }}
+        aws_mysql: ${{ contains(inputs.databases, 'mysql:aws') }}

src/test/resources/harness-config-cloud.yml

@@ -72,6 +72,13 @@ databasesUnderTest:
     username: USERNAME
     password: PASSWORD
 
+  - name: postgresql
+    prefix: aurora
+    version: 13
+    url: DBENDPOINT
+    username: USERNAME
+    password: PASSWORD
+
   - name: postgresql
     prefix: aurora
     version: 16

src/test/resources/init-changelogs/aws/mysql_cloud.sql

@@ -0,0 +1,37 @@
+--liquibase formatted sql
+--changeset liquibase:1 runAlways:true
+DROP TABLE IF EXISTS `authors`;
+CREATE TABLE `authors` (
+                           `id` int(11) NOT NULL AUTO_INCREMENT,
+                           `first_name` varchar(50) COLLATE utf8_unicode_ci NOT NULL,
+                           `last_name` varchar(50) COLLATE utf8_unicode_ci NOT NULL,
+                           `email` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
+                           `birthdate` date NOT NULL,
+                           `added` timestamp NOT NULL DEFAULT current_timestamp(),
+                           PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+
+--changeset liquibase:2 runAlways:true
+INSERT INTO `authors` VALUES ('1','Eileen','Lubowitz','[email protected]','1991-03-04','2004-05-30 02:08:25'),
+                             ('2','Tamia','Mayert','[email protected]','2016-03-27','2014-03-21 02:52:00'),
+                             ('3','Cyril','Funk','[email protected]','1988-04-21','2011-06-24 18:17:48'),
+                             ('4','Nicolas','Buckridge','[email protected]','2017-02-03','2019-04-22 02:04:41'),
+                             ('5','Jayden','Walter','[email protected]','2010-02-27','1990-02-04 02:32:00');
+
+--changeset liquibase:3 runAlways:true
+DROP TABLE IF EXISTS `posts`;
+CREATE TABLE `posts` (
+                         `id` int(11) NOT NULL,
+                         `author_id` int(11) NOT NULL,
+                         `title` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
+                         `description` varchar(500) COLLATE utf8_unicode_ci NOT NULL,
+                         `content` text COLLATE utf8_unicode_ci NOT NULL,
+                         `inserted_date` date
+) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+
+--changeset liquibase:4 runAlways:true
+INSERT INTO `posts` VALUES ('1','1','temporibus','voluptatum','Fugit non et doloribus repudiandae.','2015-11-18'),
+                           ('2','2','ea','aut','Tempora molestias maiores provident molestiae sint possimus quasi.','1975-06-08'),
+                           ('3','3','illum','rerum','Delectus recusandae sit officiis dolor.','1975-02-25'),
+                           ('4','4','itaque','deleniti','Magni nam optio id recusandae.','2010-07-28'),
+                           ('5','5','ad','similique','Rerum tempore quis ut nesciunt qui excepturi est.','2006-10-09');