Add extra integration test for packaging

Also, update the options and inputs documentation.

.github/workflows/pr-checks.yml

@@ -101,6 +101,7 @@ jobs:
         fi
 
   # Packaging test that runs against a javascript database
+  # Specifying packs in the config file.
   test-packaging-javascript-config:
     needs: [check-js, check-node-modules]
     runs-on: ubuntu-latest
@@ -143,7 +144,8 @@ jobs:
           exit 1
         fi
 
-  # tests that we can run packages through actions inputs
+  # Packaging test that runs against a javascript database
+  # Specifying packs as an input.
   test-packaging-javascript-inputs:
     needs: [check-js, check-node-modules]
     runs-on: ubuntu-latest
@@ -187,6 +189,52 @@ jobs:
           exit 1
         fi
 
+  # Packaging test that runs against a javascript database
+  # Specifying packs in the config file and inputs.
+  test-packaging-javascript-config-and-inputs:
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        config-file: ".github/codeql/codeql-config-packaging3.yml"
+        packs: +dsp-testing/[email protected]
+        languages: javascript
+        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
+        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
+
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi
+
+
   # Identify the CodeQL tool versions to integration test against.
   check-codeql-versions:
     needs: [check-js, check-node-modules]

init/action.yml

@@ -24,11 +24,12 @@ inputs:
     required: false
   packs:
     description: >-
-      Comma-separated list of packs to run. Reference a pack in the format `scope/name[@version]`. If `version` is not
+      [Experimental] Comma-separated list of packs to run. Reference a pack in the format `scope/name[@version]`. If `version` is not
       specified, then the latest version of the pack is used. By default, this overrides the same setting in a
       configuration file; prefix with "+" to use both sets of packs.
 
-      This input is only available in single-language analyses.
+      This input is only available in single-language analyses. To use packs in multi-language
+      analyses, you must specify packs in the codeql-config.yml file.
     required: false
   external-repository-token:
     description: A token for fetching external config files and queries if they reside in a private repository.

src/config-utils.test.ts

@@ -1596,7 +1596,7 @@ test(
   {},
   " + ",
   [Language.cpp],
-  /Remove the '\+'/
+  /remove the '\+'/
 );
 
 test(

src/config-utils.ts

@@ -843,7 +843,7 @@ export async function getDefaultConfig(
     );
   }
 
-  const packs = parsePacksInput(packsInput, languages) ?? {};
+  const packs = parsePacksFromInput(packsInput, languages) ?? {};
 
   return {
     languages,
@@ -1075,7 +1075,7 @@ export function parsePacksFromConfig(
   return packs;
 }
 
-function parsePacksInput(
+function parsePacksFromInput(
   packsInput: string | undefined,
   languages: Language[]
 ): Packs | undefined {
@@ -1085,7 +1085,7 @@ function parsePacksInput(
 
   if (languages.length > 1) {
     throw new Error(
-      "Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by library."
+      "Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by language."
     );
   } else if (languages.length === 0) {
     throw new Error("No languages specified. Cannot process the packs input.");
@@ -1095,7 +1095,9 @@ function parsePacksInput(
   if (packsInput.startsWith("+")) {
     packsInput = packsInput.substring(1).trim();
     if (!packsInput) {
-      throw new Error("Remove the '+' from the packs input.");
+      throw new Error(
+        "A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."
+      );
     }
   }
 
@@ -1139,7 +1141,7 @@ export function parsePacks(
   languages: Language[],
   configFile: string
 ) {
-  const packsFromInput = parsePacksInput(rawPacksInput, languages);
+  const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
   const packsFomConfig = parsePacksFromConfig(
     rawPacksFromConfig,
     languages,

src/runner.ts

@@ -132,11 +132,12 @@ program
   )
   .option(
     "--packs <packs>",
-    `Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
+    `[Experimental] Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
     specified, then the latest version of the pack is used. By default, this overrides the same setting in a
     configuration file; prefix with "+" to use both sets of packs.
 
-    This option is only available in single-language analyses.`
+    This option is only available in single-language analyses. To use packs in multi-language
+    analyses, you must specify packs in the codeql-config.yml file.`
   )
   .option("--config-file <file>", "Path to config file.")
   .option(

tests/multi-language-repo/.github/codeql/codeql-config-packaging3.yml

@@ -0,0 +1,9 @@
+name: Pack testing in the CodeQL Action
+
+disable-default-queries: true
+packs:
+  javascript:
+    - dsp-testing/codeql-pack2 # latest
+paths-ignore:
+  - tests
+  - lib