From 9dc4e91668552f76d0bf3a51429e0b2569657a85 Mon Sep 17 00:00:00 2001 From: dkirchan Date: Thu, 19 Oct 2023 09:16:42 +0300 Subject: [PATCH] Wrapped vault read to a retry --- .../pipeline.sh | 2 +- .../run_cypress/parallel_serverless.ts | 95 +++++++++++-------- 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh index dcde730013cf5..98822d33399d0 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh @@ -13,6 +13,6 @@ echo "--- Serverless Security Second Quality Gate" cd x-pack/test/security_solution_cypress set +e -QA_API_KEY=$(vault read -field=api-key secret/kibana-issues/dev/security-solution-qg-enc-key) +QA_API_KEY=$(retry 5 5 vault read -field=api-key secret/kibana-issues/dev/security-solution-qg-enc-key) CLOUD_QA_API_KEY=$QA_API_KEY yarn cypress:run:qa:serverless:parallel; status=$?; yarn junit:merge || :; exit $status \ No newline at end of file diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 53cd26da01b14..6e4daa3bf0e46 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -20,8 +20,20 @@ import fs from 'fs'; import { createFailError } from '@kbn/dev-cli-errors'; import axios from 'axios'; import { renderSummaryTable } from './print_run'; -import { isSkipped } from './utils'; +import { isSkipped, parseTestFileConfig, SecuritySolutionDescribeBlockFtrConfig } from './utils'; import path from 'path'; +import os from 'os'; + +type ProductType = { + product_line: string; + product_tier: string; +}; + +type CreateEnvironmentRequestBody = { + name: string; + region_id: string; + product_types: ProductType[]; +}; type Environment = { name: string; @@ -37,6 +49,9 @@ type Credentials = { password: string; }; +const DEFAULT_REGION = 'aws-eu-west-1'; +let log: ToolingLog; + /** * Retrieve test files using a glob pattern. * If process.env.RUN_ALL_TESTS is true, returns all matching files, otherwise, return files that should be run by this job based on process.env.BUILDKITE_PARALLEL_JOB_COUNT and process.env.BUILDKITE_PARALLEL_JOB @@ -69,13 +84,13 @@ const retrieveIntegrations = (integrationsPaths: string[]) => { const encode = (str: string): string => Buffer.from(str, 'binary').toString('base64'); const getApiKeyFromElasticCloudJsonFile = () => { - const userHomeDir = require('os').homedir(); + const userHomeDir = os.homedir(); try { const jsonString = fs.readFileSync(path.join(userHomeDir, '.elastic/cloud.json'), 'utf-8'); const jsonData = JSON.parse(jsonString); return jsonData.api_key.qa; } catch (e) { - console.log('API KEY could not be found in '); + log.info('API KEY could not be found in '); return null; } }; @@ -83,14 +98,16 @@ const getApiKeyFromElasticCloudJsonFile = () => { // Poller function that is polling every 20s, forever until function is resolved. async function poll( fn: () => Promise, - retries: number = Infinity, + retries: number = 200, interval: number = 20000 ): Promise { return Promise.resolve() .then(fn) - .catch(async function retry(err: any) { - if (retries-- > 0) + .catch(async function retry(err: Error): Promise { + if (retries-- > 0) { + retries -= 1; return new Promise((resolve) => setTimeout(resolve, interval)).then(fn).catch(retry); + } throw err; }); } @@ -101,24 +118,29 @@ async function createEnvironment( projectName: string, runnerId: string, apiKey: string, + ftrConfig: SecuritySolutionDescribeBlockFtrConfig, onEarlyExit: (msg: string) => void ): Promise { - console.log(`${runnerId}: Creating environment ${projectName}...`); + log.info(`${runnerId}: Creating environment ${projectName}...`); let environment = {} as Environment; + const body = { + name: projectName, + region_id: DEFAULT_REGION, + } as CreateEnvironmentRequestBody; + + const productTypes: ProductType[] = []; + ftrConfig?.productTypes?.forEach((t) => { + productTypes.push(t as ProductType); + }); + if (productTypes.length > 0) body.product_types = productTypes; + await axios - .post( - `${baseUrl}/api/v1/serverless/projects/security`, - { - name: `${projectName}`, - region_id: 'aws-eu-west-1', + .post(`${baseUrl}/api/v1/serverless/projects/security`, body, { + headers: { + Authorization: `ApiKey ${apiKey}`, + 'Content-Type': 'application/json', }, - { - headers: { - Authorization: `ApiKey ${apiKey}`, - 'Content-Type': 'application/json', - }, - } - ) + }) .then((response) => { environment.name = response.data.name; environment.id = response.data.id; @@ -149,7 +171,7 @@ async function deleteEnvironment( }, }) .then((response) => { - console.log(`${runnerId} : Environment ${projectName} was successfully deleted...`); + log.info(`${runnerId} : Environment ${projectName} was successfully deleted...`); }) .catch((error) => { onEarlyExit(`${error.code}:${error.data}`); @@ -163,11 +185,11 @@ async function resetCredentials( runnerId: string, apiKey: string ): Promise { - console.log(`${runnerId} : Reseting credentials`); + log.info(`${runnerId} : Reseting credentials`); let credentials = {} as Credentials; await poll(async () => { - return await axios + return axios .post( `${baseUrl}/api/v1/serverless/projects/security/${environmentId}/_reset-credentials`, {}, @@ -182,7 +204,7 @@ async function resetCredentials( credentials.username = response.data.username; }) .catch((error) => { - throw Error(`${error.code}:${error.data}`); + throw new Error(`${error.code}:${error.data}`); }); }); return credentials; @@ -198,15 +220,13 @@ async function waitForEsStatusGreen(esUrl: string, auth: string, runnerId: strin }, }) .then((response) => { - console.log(`${runnerId}: Elasticsearch is ready with status ${response.data.status}.`); + log.info(`${runnerId}: Elasticsearch is ready with status ${response.data.status}.`); }) .catch((error) => { - if (error.code == 'ENOTFOUND') { - console.log( - `${runnerId}: The elasticsearch url is not yet reachable. Retrying in 20s...` - ); + if (error.code === 'ENOTFOUND') { + log.info(`${runnerId}: The elasticsearch url is not yet reachable. Retrying in 20s...`); } - throw error; + throw new Error(`${runnerId} - ${error.code}:${error.data}`); }); }); } @@ -221,16 +241,16 @@ async function waitForKibanaAvailable(kbUrl: string, auth: string, runnerId: str }, }) .then((response) => { - if (response.data.status.overall.level != 'available') { - console.log(`${runnerId}: Kibana is not available. Retrying in 20s...`); + if (response.data.status.overall.level !== 'available') { + log.info(`${runnerId}: Kibana is not available. Retrying in 20s...`); throw new Error(`${runnerId}: Kibana is not available. Retrying in 20s...`); } }) .catch((error) => { - if (error.code == 'ENOTFOUND') { - console.log(`${runnerId}: The kibana url is not yet reachable. Retrying in 20s...`); + if (error.code === 'ENOTFOUND') { + log.info(`${runnerId}: The kibana url is not yet reachable. Retrying in 20s...`); } - throw error; + throw new Error(`${runnerId} - ${error.code}:${error.data}`); }); }); } @@ -238,7 +258,7 @@ async function waitForKibanaAvailable(kbUrl: string, auth: string, runnerId: str export const cli = () => { run( async () => { - const log = new ToolingLog({ + log = new ToolingLog({ level: 'info', writeTo: process.stdout, }); @@ -252,6 +272,7 @@ export const cli = () => { log.error( 'If running locally, ~/.elastic/cloud.json is attempted to be read which contains the api key.' ); + // eslint-disable-next-line no-process-exit return process.exit(0); } @@ -388,6 +409,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} }; const id = crypto.randomBytes(8).toString('hex'); const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`; + const specFileFTRConfig = parseTestFileConfig(filePath); // Creating environment for the test to run const environment = await createEnvironment( @@ -395,6 +417,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} PROJECT_NAME, id, API_KEY, + specFileFTRConfig, onEarlyExit ); @@ -430,9 +453,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} ---------------------------------------------- Cypress run ENV for file: ${filePath}: ---------------------------------------------- - ${JSON.stringify(cyCustomEnv, null, 2)} - ---------------------------------------------- `); }