Send the external repository token to the CLI

This commit does a few related things:

1. Bumps the minimum version for cli config parsing to 2.10.6
2. Ensures that if cli config parsing is enabled, then remove repos
   are _not_ downloaded by the action. It happens in the CLI.
3. Passes the `--external-repository-token-stdin` option to the CLI
   and passes the appropriate token via stdin if cli config parsing is
   enabled.

src/analyze.ts

@@ -262,6 +262,7 @@ export async function runQueries(
         logger.endGroup();
         logger.info(analysisSummary);
       } else {
+        // config was generated by the action, so must be interpreted by the action.
         logger.startGroup(`Running queries for ${language}`);
         const querySuitePaths: string[] = [];
         if (queries["builtin"].length > 0) {

src/codeql.test.ts

@@ -15,7 +15,7 @@ import { GitHubApiDetails } from "./api-client";
 import * as codeql from "./codeql";
 import { AugmentationProperties, Config } from "./config-utils";
 import * as defaults from "./defaults.json";
-import { Feature } from "./feature-flags";
+import { Feature, featureConfig } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
 import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
@@ -513,7 +513,7 @@ const injectedConfigMacro = test.macro({
       const codeqlObject = await codeql.getCodeQLForTesting();
       sinon
         .stub(codeqlObject, "getVersion")
-        .resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
+        .resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
 
       const thisStubConfig: Config = {
         ...stubConfig,
@@ -570,7 +570,7 @@ test(
   },
   {},
   {
-    packs: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
+    packs: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
   }
 );
 
@@ -591,7 +591,7 @@ test(
     packs: {
       javascript: [
         "codeql/something-else",
-        "codeql/javascript-experimental-atm-queries@~0.3.0",
+        "codeql/javascript-experimental-atm-queries@~0.4.0",
       ],
     },
   }
@@ -613,7 +613,7 @@ test(
   {
     packs: {
       cpp: ["codeql/something-else"],
-      javascript: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
+      javascript: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
     },
   }
 );
@@ -694,7 +694,7 @@ test(
     },
   },
   {
-    packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.3.0"],
+    packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.4.0"],
   }
 );
 
@@ -826,7 +826,7 @@ test("does not use injected config", async (t: ExecutionContext<unknown>) => {
     const codeqlObject = await codeql.getCodeQLForTesting();
     sinon
       .stub(codeqlObject, "getVersion")
-      .resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
+      .resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
 
     await codeqlObject.databaseInitCluster(
       stubConfig,

src/codeql.ts

@@ -9,7 +9,7 @@ import * as yaml from "js-yaml";
 import * as semver from "semver";
 import { v4 as uuidV4 } from "uuid";
 
-import { isRunningLocalAction } from "./actions-util";
+import { getOptionalInput, isRunningLocalAction } from "./actions-util";
 import * as api from "./api-client";
 import { Config } from "./config-utils";
 import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
@@ -252,7 +252,6 @@ const CODEQL_MINIMUM_VERSION = "2.6.3";
  */
 const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
 const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
-export const CODEQL_VERSION_CONFIG_FILES = "2.10.1";
 const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
 export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
 const CODEQL_VERSION_FILE_BASELINE_INFORMATION = "2.11.3";
@@ -885,24 +884,35 @@ async function getCodeQLForCmd(
         }
       }
 
+      // A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
+      // Only pass external repository token if a config file is
+      let externalRepositoryToken: string | undefined;
       const configLocation = await generateCodeScanningConfig(
         codeql,
         config,
         featureEnablement
       );
       if (configLocation) {
         extraArgs.push(`--codescanning-config=${configLocation}`);
+        externalRepositoryToken = getOptionalInput("external-repository-token");
+        if (externalRepositoryToken) {
+          extraArgs.push("--external-repository-token-stdin");
+        }
       }
 
-      await runTool(cmd, [
-        "database",
-        "init",
-        "--db-cluster",
-        config.dbLocation,
-        `--source-root=${sourceRoot}`,
-        ...extraArgs,
-        ...getExtraOptionsFromEnv(["database", "init"]),
-      ]);
+      await runTool(
+        cmd,
+        [
+          "database",
+          "init",
+          "--db-cluster",
+          config.dbLocation,
+          `--source-root=${sourceRoot}`,
+          ...extraArgs,
+          ...getExtraOptionsFromEnv(["database", "init"]),
+        ],
+        externalRepositoryToken
+      );
     },
     async runAutobuild(language: Language) {
       const cmdName =
@@ -1335,7 +1345,7 @@ export function getExtraOptions(
  */
 const maxErrorSize = 20_000;
 
-async function runTool(cmd: string, args: string[] = []) {
+async function runTool(cmd: string, args: string[] = [], stdin?: string) {
   let output = "";
   let error = "";
   const exitCode = await new toolrunner.ToolRunner(cmd, args, {
@@ -1354,6 +1364,7 @@ async function runTool(cmd: string, args: string[] = []) {
       },
     },
     ignoreReturnCode: true,
+    input: Buffer.from(stdin || ""),
   }).exec();
   if (exitCode !== 0)
     throw new CommandInvocationError(cmd, args, exitCode, error, output);