Skip to content

Commit

Permalink
Wrapped vault read to a retry
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkirchan
dkirchan committed Oct 19, 2023
1 parent f3520d5 commit 9dc4e91
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 38 deletions.
2 changes: 1 addition & 1 deletion .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,6 @@ echo "--- Serverless Security Second Quality Gate"
cd x-pack/test/security_solution_cypress
set +e

QA_API_KEY=$(vault read -field=api-key secret/kibana-issues/dev/security-solution-qg-enc-key)
QA_API_KEY=$(retry 5 5 vault read -field=api-key secret/kibana-issues/dev/security-solution-qg-enc-key)

CLOUD_QA_API_KEY=$QA_API_KEY yarn cypress:run:qa:serverless:parallel; status=$?; yarn junit:merge || :; exit $status
95 changes: 58 additions & 37 deletions x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,20 @@ import fs from 'fs';
import { createFailError } from '@kbn/dev-cli-errors';
import axios from 'axios';
import { renderSummaryTable } from './print_run';
import { isSkipped } from './utils';
import { isSkipped, parseTestFileConfig, SecuritySolutionDescribeBlockFtrConfig } from './utils';
import path from 'path';
import os from 'os';

type ProductType = {
product_line: string;
product_tier: string;
};

type CreateEnvironmentRequestBody = {
name: string;
region_id: string;
product_types: ProductType[];
};

type Environment = {
name: string;
Expand All @@ -37,6 +49,9 @@ type Credentials = {
password: string;
};

const DEFAULT_REGION = 'aws-eu-west-1';
let log: ToolingLog;

/**
* Retrieve test files using a glob pattern.
* If process.env.RUN_ALL_TESTS is true, returns all matching files, otherwise, return files that should be run by this job based on process.env.BUILDKITE_PARALLEL_JOB_COUNT and process.env.BUILDKITE_PARALLEL_JOB
Expand Down Expand Up @@ -69,28 +84,30 @@ const retrieveIntegrations = (integrationsPaths: string[]) => {
const encode = (str: string): string => Buffer.from(str, 'binary').toString('base64');

const getApiKeyFromElasticCloudJsonFile = () => {
const userHomeDir = require('os').homedir();
const userHomeDir = os.homedir();
try {
const jsonString = fs.readFileSync(path.join(userHomeDir, '.elastic/cloud.json'), 'utf-8');
const jsonData = JSON.parse(jsonString);
return jsonData.api_key.qa;
} catch (e) {
console.log('API KEY could not be found in ');
log.info('API KEY could not be found in ');
return null;
}
};

// Poller function that is polling every 20s, forever until function is resolved.
async function poll<T>(
fn: () => Promise<T>,
retries: number = Infinity,
retries: number = 200,
interval: number = 20000
): Promise<T> {
return Promise.resolve()
.then(fn)
.catch(async function retry(err: any) {
if (retries-- > 0)
.catch(async function retry(err: Error): Promise<T> {
if (retries-- > 0) {
retries -= 1;
return new Promise((resolve) => setTimeout(resolve, interval)).then(fn).catch(retry);
}
throw err;
});
}
Expand All @@ -101,24 +118,29 @@ async function createEnvironment(
projectName: string,
runnerId: string,
apiKey: string,
ftrConfig: SecuritySolutionDescribeBlockFtrConfig,
onEarlyExit: (msg: string) => void
): Promise<Environment> {
console.log(`${runnerId}: Creating environment ${projectName}...`);
log.info(`${runnerId}: Creating environment ${projectName}...`);
let environment = {} as Environment;
const body = {
name: projectName,
region_id: DEFAULT_REGION,
} as CreateEnvironmentRequestBody;

const productTypes: ProductType[] = [];
ftrConfig?.productTypes?.forEach((t) => {
productTypes.push(t as ProductType);
});
if (productTypes.length > 0) body.product_types = productTypes;

await axios
.post(
`${baseUrl}/api/v1/serverless/projects/security`,
{
name: `${projectName}`,
region_id: 'aws-eu-west-1',
.post(`${baseUrl}/api/v1/serverless/projects/security`, body, {
headers: {
Authorization: `ApiKey ${apiKey}`,
'Content-Type': 'application/json',
},
{
headers: {
Authorization: `ApiKey ${apiKey}`,
'Content-Type': 'application/json',
},
}
)
})
.then((response) => {
environment.name = response.data.name;
environment.id = response.data.id;
Expand Down Expand Up @@ -149,7 +171,7 @@ async function deleteEnvironment(
},
})
.then((response) => {
console.log(`${runnerId} : Environment ${projectName} was successfully deleted...`);
log.info(`${runnerId} : Environment ${projectName} was successfully deleted...`);
})
.catch((error) => {
onEarlyExit(`${error.code}:${error.data}`);
Expand All @@ -163,11 +185,11 @@ async function resetCredentials(
runnerId: string,
apiKey: string
): Promise<Credentials> {
console.log(`${runnerId} : Reseting credentials`);
log.info(`${runnerId} : Reseting credentials`);
let credentials = {} as Credentials;

await poll(async () => {
return await axios
return axios
.post(
`${baseUrl}/api/v1/serverless/projects/security/${environmentId}/_reset-credentials`,
{},
Expand All @@ -182,7 +204,7 @@ async function resetCredentials(
credentials.username = response.data.username;
})
.catch((error) => {
throw Error(`${error.code}:${error.data}`);
throw new Error(`${error.code}:${error.data}`);
});
});
return credentials;
Expand All @@ -198,15 +220,13 @@ async function waitForEsStatusGreen(esUrl: string, auth: string, runnerId: strin
},
})
.then((response) => {
console.log(`${runnerId}: Elasticsearch is ready with status ${response.data.status}.`);
log.info(`${runnerId}: Elasticsearch is ready with status ${response.data.status}.`);
})
.catch((error) => {
if (error.code == 'ENOTFOUND') {
console.log(
`${runnerId}: The elasticsearch url is not yet reachable. Retrying in 20s...`
);
if (error.code === 'ENOTFOUND') {
log.info(`${runnerId}: The elasticsearch url is not yet reachable. Retrying in 20s...`);
}
throw error;
throw new Error(`${runnerId} - ${error.code}:${error.data}`);
});
});
}
Expand All @@ -221,24 +241,24 @@ async function waitForKibanaAvailable(kbUrl: string, auth: string, runnerId: str
},
})
.then((response) => {
if (response.data.status.overall.level != 'available') {
console.log(`${runnerId}: Kibana is not available. Retrying in 20s...`);
if (response.data.status.overall.level !== 'available') {
log.info(`${runnerId}: Kibana is not available. Retrying in 20s...`);
throw new Error(`${runnerId}: Kibana is not available. Retrying in 20s...`);
}
})
.catch((error) => {
if (error.code == 'ENOTFOUND') {
console.log(`${runnerId}: The kibana url is not yet reachable. Retrying in 20s...`);
if (error.code === 'ENOTFOUND') {
log.info(`${runnerId}: The kibana url is not yet reachable. Retrying in 20s...`);
}
throw error;
throw new Error(`${runnerId} - ${error.code}:${error.data}`);
});
});
}

export const cli = () => {
run(
async () => {
const log = new ToolingLog({
log = new ToolingLog({
level: 'info',
writeTo: process.stdout,
});
Expand All @@ -252,6 +272,7 @@ export const cli = () => {
log.error(
'If running locally, ~/.elastic/cloud.json is attempted to be read which contains the api key.'
);
// eslint-disable-next-line no-process-exit
return process.exit(0);
}

Expand Down Expand Up @@ -388,13 +409,15 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
};
const id = crypto.randomBytes(8).toString('hex');
const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`;
const specFileFTRConfig = parseTestFileConfig(filePath);

// Creating environment for the test to run
const environment = await createEnvironment(
BASE_ENV_URL,
PROJECT_NAME,
id,
API_KEY,
specFileFTRConfig,
onEarlyExit
);

Expand Down Expand Up @@ -430,9 +453,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
----------------------------------------------
Cypress run ENV for file: ${filePath}:
----------------------------------------------
${JSON.stringify(cyCustomEnv, null, 2)}
----------------------------------------------
`);
}
Expand Down

0 comments on commit 9dc4e91

Please sign in to comment.