move tests w/ ohi validation timeout to separate workflow

.github/workflows/nonregression.yml

@@ -5,7 +5,7 @@ on:
   schedule:
     - cron: "0 9 * * 1-5"
   push:
-    branches: [main]
+    branches: [ main ]
 
 jobs:
   log-context:
@@ -51,29 +51,39 @@ jobs:
         uses: actions/github-script@v3
         with:
           script: |
-            const fs = require('fs');
+            const fs = require("fs");
             const fsp = fs.promises;
-            const path = require('path');
+            const path = require("path");
+
+            const { isOHIValidationTimeout } = require("${{ github.workspace }}/.github/workflows/scripts/ohiValidationTimeout");
 
             // readdir recursive directory search
             const { readdir } = fsp;
             async function getFiles(dir) {
-                const dirents = await readdir(dir, { withFileTypes: true });
-                const files = await Promise.all(dirents.map((dirent) => {
-                    const res = path.join(dir, dirent.name);
-                    return dirent.isDirectory() ? getFiles(res) : res;
-                }));
-                return Array.prototype.concat(...files);
+              const dirents = await readdir(dir, { withFileTypes: true });
+              const files = await Promise.all(
+                dirents.map((dirent) => {
+                  const res = path.join(dir, dirent.name);
+                  return dirent.isDirectory() ? getFiles(res) : res;
+                })
+              );
+              return Array.prototype.concat(...files);
             }
-            const definitionsDir = 'test/definitions'
+
+            const definitionsDir = "test/definitions";
             const testDefinitions = await getFiles(definitionsDir);
 
-            const outputTestFilesMap = testDefinitions.map(testDefinitionFile => {
-                return { testDefinitionFile, 'testDisplayName': testDefinitionFile.replace(`${definitionsDir}/`, '') }
-            })
+            const outputTestFilesMap = testDefinitions
+              .filter((testDefinitionFile) => !isOHIValidationTimeout(testDefinitionFile))
+              .map((testDefinitionFile) => {
+                return {
+                  testDefinitionFile,
+                  testDisplayName: testDefinitionFile.replace(`${definitionsDir}/`, ""),
+                };
+              });
             const output = {
-                "include": outputTestFilesMap
-            }
+              include: outputTestFilesMap,
+            };
             console.log(output);
             return output;
 

.github/workflows/ohi-validation-timeout.yml

@@ -0,0 +1,191 @@
+name: OHI Validation Timeout
+
+on: [ workflow_dispatch ]
+
+jobs:
+  log-context:
+    runs-on: ubuntu-latest
+    steps:
+      # Dump all contexts
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Dump job context
+        env:
+          JOB_CONTEXT: ${{ toJson(job) }}
+        run: echo "$JOB_CONTEXT"
+      - name: Dump steps context
+        env:
+          STEPS_CONTEXT: ${{ toJson(steps) }}
+        run: echo "$STEPS_CONTEXT"
+      - name: Dump runner context
+        env:
+          RUNNER_CONTEXT: ${{ toJson(runner) }}
+        run: echo "$RUNNER_CONTEXT"
+      - name: Dump strategy context
+        env:
+          STRATEGY_CONTEXT: ${{ toJson(strategy) }}
+        run: echo "$STRATEGY_CONTEXT"
+      - name: Dump matrix context
+        env:
+          MATRIX_CONTEXT: ${{ toJson(matrix) }}
+        run: echo "$MATRIX_CONTEXT"
+  get-test-definition-files:
+    name: Get Test Definition Files
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-test-definition-files.outputs.result }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Get Test Definition Files
+        id: get-test-definition-files
+        uses: actions/github-script@v3
+        with:
+          script: |
+            const fs = require("fs");
+            const fsp = fs.promises;
+            const path = require("path");
+
+            const { isOHIValidationTimeout } = require("${{ github.workspace }}/.github/workflows/scripts/ohiValidationTimeout");
+
+            // readdir recursive directory search
+            const { readdir } = fsp;
+            async function getFiles(dir) {
+              const dirents = await readdir(dir, { withFileTypes: true });
+              const files = await Promise.all(
+                dirents.map((dirent) => {
+                  const res = path.join(dir, dirent.name);
+                  return dirent.isDirectory() ? getFiles(res) : res;
+                })
+              );
+              return Array.prototype.concat(...files);
+            }
+            const definitionsDir = "test/definitions";
+            const testDefinitions = await getFiles(definitionsDir);
+
+            const outputTestFilesMap = testDefinitions
+              .filter(isOHIValidationTimeout)
+              .map((testDefinitionFile) => {
+                return {
+                  testDefinitionFile,
+                  testDisplayName: testDefinitionFile.replace(`${definitionsDir}/`, ""),
+                };
+              });
+            const output = {
+              include: outputTestFilesMap,
+            };
+            console.log(output);
+
+            return output;
+
+  test-deploy-recipe:
+    name: ${{ matrix.testDisplayName }}
+    needs: [get-test-definition-files]
+    if: ${{ fromJSON(needs.get-test-definition-files.outputs.matrix).include[0] }} # Avoids empty matrix validation error
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.get-test-definition-files.outputs.matrix) }}
+      fail-fast: false
+    env:
+      MATRIX: ${{ toJSON(matrix) }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Update Test Definition Files URLs
+        id: get-test-definition-files
+        env:
+          TEST_DEFINITION_FILE: ${{ matrix.testDefinitionFile }}
+        uses: actions/github-script@v3
+        with:
+          script: |
+            const fs = require('fs');
+            const fsp = fs.promises;
+            const path = require('path');
+
+            // before returning, we need to edit the deploy config files in-place so they 
+            // use the right URLs from the branch
+            async function getDeployConfigFile(file, outputDir) {
+              const data = await fsp.readFile(path.join(outputDir, file));
+              return JSON.parse(data);
+            }
+
+            // Get testDefinitonFile from MATRIX env var
+            const testDefinitionFile = process.env.TEST_DEFINITION_FILE;
+            console.log(`Detected Deploy Config: ${JSON.stringify(testDefinitionFile, null, 2)}`)
+
+            // Update URLs to use branch this PR is opened with
+            const data = await getDeployConfigFile(testDefinitionFile, process.env.GITHUB_WORKSPACE);
+
+            // Update github source URLs with branch name
+            let jsonContent = JSON.stringify(data, null, 2);
+            const branchName = process.env.GITHUB_HEAD_REF ? process.env.GITHUB_HEAD_REF : process.env.GITHUB_REF_NAME;
+            const replacementString = `$1$2-b ${branchName} $3$4`;
+            const sourceRepositoryRegex = /(.*)(\")(https:\/\/github.com\/newrelic\/open-install-library)(.*)/gi;
+            jsonContent = jsonContent.replace(sourceRepositoryRegex, replacementString);
+            console.log(`Detected Deploy Config: ${JSON.stringify(jsonContent, null, 2)}`)
+
+            // Update raw URLs with branch name
+            const replacementString2 = `$1${branchName}$3`;
+            const sourceRepositoryRegex2 = /(raw.githubusercontent.com\/newrelic\/open-install-library\/)(main)(\/newrelic\/recipes\/)*/gi;
+            jsonContent = jsonContent.replace(sourceRepositoryRegex2, replacementString2);
+            console.log(`Detected Deploy Config: ${JSON.stringify(jsonContent, null, 2)}`)
+
+            // Write file back to workspace
+            const outputPath = `${process.env.GITHUB_WORKSPACE}/${testDefinitionFile}`;
+            fs.writeFileSync(outputPath, jsonContent);
+
+            return testDefinitionFile;
+
+      - name: Write AWS Certificate to File
+        env:
+          AWS_PEM: ${{ secrets.GIT_DEPLOYER_CANADA_AWS_PEM }}
+        run: |
+          mkdir -p configs
+          rm -f configs/gitdeployerCanada.pem
+          echo "$AWS_PEM" > configs/gitdeployerCanada.pem
+          sudo chmod 400 configs/gitdeployerCanada.pem
+
+      - name: Write Test Definition File JSON to file
+        env:
+          USER_JSON: ${{ secrets.GIT_DEPLOYER_DOCKER_USER_CONFIG }}
+        run: |
+          echo "$USER_JSON" > configs/gitusdkr${{ github.run_id }}.json
+
+      - name: Pull Deployer image
+        run: |
+          docker pull ghcr.io/newrelic/deployer:latest
+          docker images ghcr.io/newrelic/deployer:latest
+
+      - name: Run deployer
+        id: deployerRun
+        run: |
+          set -e
+          testDefinitionFile=$(echo $MATRIX | jq -c -r '.testDefinitionFile')
+          echo $testDefinitionFile
+          docker run -i\
+            -v ${{ github.workspace }}/configs/:/mnt/deployer/configs/\
+            -v ${{ github.workspace }}/test/:/mnt/deployer/test/\
+            --entrypoint ruby ghcr.io/newrelic/deployer:latest main.rb -c configs/gitusdkr${{ github.run_id }}.json -d $testDefinitionFile -l debug
+          echo ::set-output name=exit_status::$?
+      - name: Teardown any previous deployment
+        if: always()
+        id: cleanupResources
+        continue-on-error: true
+        run: |
+          testDefinitionFile=$(echo $MATRIX | jq -c -r '.testDefinitionFile')
+          echo $testDefinitionFile
+          docker run \
+            -v ${{ github.workspace }}/configs/:/mnt/deployer/configs/\
+            -v ${{ github.workspace }}/test/:/mnt/deployer/test/\
+            --entrypoint ruby ghcr.io/newrelic/deployer:latest main.rb -c configs/gitusdkr${{ github.run_id }}.json -d $testDefinitionFile -t
+
+      - name: Report any error
+        if: steps.deployerRun.outputs.exit_status != 0
+        run: exit 1

.github/workflows/scripts/ohiValidationTimeout.js

@@ -0,0 +1,35 @@
+// These OHI are timing out in validation, causing installations to report as incomplete
+const ohiValidationTimeoutFiles = [
+  "test/definitions/ohi/linux/cassandra-debian.json",
+  "test/definitions/ohi/linux/consul-debian.json",
+  "test/definitions/ohi/linux/consul-rhel.json",
+  "test/definitions/ohi/linux/couchbase-debian.json",
+  "test/definitions/ohi/linux/couchbase-rhel.json",
+  "test/definitions/ohi/linux/elasticsearch-debian.json",
+  "test/definitions/ohi/linux/elasticsearch-rhel.json",
+  "test/definitions/ohi/linux/elasticsearch-suse.json",
+  "test/definitions/ohi/linux/haproxy-debian.json",
+  "test/definitions/ohi/linux/haproxy-rhel.json",
+  "test/definitions/ohi/linux/memcached-debian.json",
+  "test/definitions/ohi/linux/memcached-rhel.json",
+  "test/definitions/ohi/linux/mongodb-debian.json",
+  "test/definitions/ohi/linux/mysql-debian.json",
+  "test/definitions/ohi/linux/nagios-debian.json",
+  "test/definitions/ohi/linux/nagios-rhel.json",
+  "test/definitions/ohi/linux/nginx-debian.json",
+  "test/definitions/ohi/linux/nginx-linux2-ami.json",
+  "test/definitions/ohi/linux/postgres-debian.json",
+  "test/definitions/ohi/linux/postgres-rhel.json",
+  "test/definitions/ohi/linux/redis-debian.json",
+  "test/definitions/ohi/linux/varnish-debian.json",
+  "test/definitions/ohi/windows/ms-sql-server2019Standard.json",
+];
+
+const ohiLookup = ohiValidationTimeoutFiles.reduce(
+  (lookup, file) => lookup.set(file, true),
+  new Map()
+);
+
+const isOHIValidationTimeout = (file) => ohiLookup.get(file) !== undefined;
+
+module.exports = { isOHIValidationTimeout };