diff --git a/.github/workflows/benchmark-nightly.yml b/.github/workflows/benchmark-nightly.yml index 2de86abaf740ea..d557a0c9ee6e34 100644 --- a/.github/workflows/benchmark-nightly.yml +++ b/.github/workflows/benchmark-nightly.yml @@ -24,6 +24,9 @@ env: ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }} K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }} + N8N_TAG: ${{ inputs.n8n_tag || 'nightly' }} + N8N_BENCHMARK_TAG: ${{ inputs.benchmark_tag || 'latest' }} + DEBUG: ${{ inputs.debug == 'true' && '--debug' || '' }} permissions: id-token: write @@ -62,12 +65,23 @@ jobs: run: pnpm destroy-cloud-env working-directory: packages/@n8n/benchmark - - name: Run the benchmark with debug logging - if: github.event.inputs.debug == 'true' - run: pnpm benchmark-in-cloud --n8nTag ${{ inputs.n8n_tag || 'nightly' }} --benchmarkTag ${{ inputs.benchmark_tag || 'latest' }} --debug + - name: Provision the environment + run: pnpm provision-cloud-env ${{ env.DEBUG }} working-directory: packages/@n8n/benchmark - name: Run the benchmark - if: github.event.inputs.debug != 'true' - run: pnpm benchmark-in-cloud --n8nTag ${{ inputs.n8n_tag || 'nightly' }} --benchmarkTag ${{ inputs.benchmark_tag || 'latest' }} + run: pnpm benchmark-in-cloud --n8nTag ${{ env.N8N_TAG }} --benchmarkTag ${{ env.N8N_BENCHMARK_TAG }} ${{ env.DEBUG }} + working-directory: packages/@n8n/benchmark + + # We need to login again because the access token expires + - name: Azure login + uses: azure/login@v2.1.1 + with: + client-id: ${{ env.ARM_CLIENT_ID }} + tenant-id: ${{ env.ARM_TENANT_ID }} + subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }} + + - name: Destroy the environment + if: always() + run: pnpm destroy-cloud-env ${{ env.DEBUG }} working-directory: packages/@n8n/benchmark diff --git a/packages/@n8n/benchmark/package.json b/packages/@n8n/benchmark/package.json index 2d9affef5dbc5e..d7a8fb8a5f231f 100644 --- a/packages/@n8n/benchmark/package.json +++ b/packages/@n8n/benchmark/package.json @@ -13,6 +13,7 @@ "benchmark": "zx scripts/run.mjs", "benchmark-in-cloud": "pnpm benchmark --env cloud", "benchmark-locally": "pnpm benchmark --env local", + "provision-cloud-env": "zx scripts/provisionCloudEnv.mjs", "destroy-cloud-env": "zx scripts/destroyCloudEnv.mjs", "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"" }, diff --git a/packages/@n8n/benchmark/scripts/clients/terraformClient.mjs b/packages/@n8n/benchmark/scripts/clients/terraformClient.mjs index b156998f92d7a9..522e35b6e98e41 100644 --- a/packages/@n8n/benchmark/scripts/clients/terraformClient.mjs +++ b/packages/@n8n/benchmark/scripts/clients/terraformClient.mjs @@ -17,6 +17,16 @@ export class TerraformClient { }); } + /** + * Provisions the environment + */ + async provisionEnvironment() { + console.log('Provisioning cloud environment...'); + + await this.$$`terraform init`; + await this.$$`terraform apply -input=false -auto-approve`; + } + /** * @typedef {Object} BenchmarkEnv * @property {string} vmName @@ -26,12 +36,7 @@ export class TerraformClient { * * @returns {Promise} */ - async provisionEnvironment() { - console.log('Provisioning cloud environment...'); - - await this.$$`terraform init`; - await this.$$`terraform apply -input=false -auto-approve`; - + async getTerraformOutputs() { const privateKeyName = await this.extractPrivateKey(); return { @@ -42,12 +47,11 @@ export class TerraformClient { }; } - async destroyEnvironment() { - if (!fs.existsSync(paths.terraformStateFile)) { - console.log('No cloud environment to destroy. Skipping...'); - return; - } + hasTerraformState() { + return fs.existsSync(paths.terraformStateFile); + } + async destroyEnvironment() { console.log('Destroying cloud environment...'); await this.$$`terraform destroy -input=false -auto-approve`; diff --git a/packages/@n8n/benchmark/scripts/destroyCloudEnv.mjs b/packages/@n8n/benchmark/scripts/destroyCloudEnv.mjs index 1ffc852aeaab8f..0e203efca353a1 100644 --- a/packages/@n8n/benchmark/scripts/destroyCloudEnv.mjs +++ b/packages/@n8n/benchmark/scripts/destroyCloudEnv.mjs @@ -1,52 +1,43 @@ #!/usr/bin/env zx /** - * Script that deletes all resources created by the benchmark environment - * and that are older than 2 hours. + * Script that deletes all resources created by the benchmark environment. * - * Even tho the environment is provisioned using terraform, the terraform - * state is not persisted. Hence we can't use terraform to delete the resources. - * We could store the state to a storage account, but then we wouldn't be able - * to spin up new envs on-demand. Hence this design. - * - * Usage: - * zx scripts/deleteCloudEnv.mjs + * This scripts tries to delete resources created by Terraform. If Terraform + * state file is not found, it will try to delete resources using Azure CLI. + * The terraform state is not persisted, so we want to support both cases. */ // @ts-check -import { $ } from 'zx'; +import { $, minimist } from 'zx'; +import { TerraformClient } from './clients/terraformClient.mjs'; -const EXPIRE_TIME_IN_H = 2; -const EXPIRE_TIME_IN_MS = EXPIRE_TIME_IN_H * 60 * 60 * 1000; const RESOURCE_GROUP_NAME = 'n8n-benchmarking'; +const args = minimist(process.argv.slice(3), { + boolean: ['debug'], +}); + +const isVerbose = !!args.debug; + async function main() { + const terraformClient = new TerraformClient({ isVerbose }); + + if (terraformClient.hasTerraformState()) { + await terraformClient.destroyEnvironment(); + } else { + await destroyUsingAz(); + } +} + +async function destroyUsingAz() { const resourcesResult = await $`az resource list --resource-group ${RESOURCE_GROUP_NAME} --query "[?tags.Id == 'N8nBenchmark'].{id:id, createdAt:tags.CreatedAt}" -o json`; const resources = JSON.parse(resourcesResult.stdout); - const now = Date.now(); - - const resourcesToDelete = resources - .filter((resource) => { - if (resource.createdAt === undefined) { - return true; - } - - const createdAt = new Date(resource.createdAt); - const resourceExpiredAt = createdAt.getTime() + EXPIRE_TIME_IN_MS; - - return now > resourceExpiredAt; - }) - .map((resource) => resource.id); + const resourcesToDelete = resources.map((resource) => resource.id); if (resourcesToDelete.length === 0) { - if (resources.length === 0) { - console.log('No resources found in the resource group.'); - } else { - console.log( - `Found ${resources.length} resources in the resource group, but none are older than ${EXPIRE_TIME_IN_H} hours.`, - ); - } + console.log('No resources found in the resource group.'); return; } @@ -87,4 +78,9 @@ async function deleteById(id) { } } -main(); +main().catch((error) => { + console.error('An error occurred destroying cloud env:'); + console.error(error); + + process.exit(1); +}); diff --git a/packages/@n8n/benchmark/scripts/provisionCloudEnv.mjs b/packages/@n8n/benchmark/scripts/provisionCloudEnv.mjs new file mode 100644 index 00000000000000..5f10e7c60c72fb --- /dev/null +++ b/packages/@n8n/benchmark/scripts/provisionCloudEnv.mjs @@ -0,0 +1,36 @@ +#!/usr/bin/env zx +/** + * Provisions the cloud benchmark environment + * + * NOTE: Must be run in the root of the package. + */ +// @ts-check +import { which, minimist } from 'zx'; +import { TerraformClient } from './clients/terraformClient.mjs'; + +const args = minimist(process.argv.slice(3), { + boolean: ['debug'], +}); + +const isVerbose = !!args.debug; + +export async function provision() { + await ensureDependencies(); + + const terraformClient = new TerraformClient({ + isVerbose, + }); + + await terraformClient.provisionEnvironment(); +} + +async function ensureDependencies() { + await which('terraform'); +} + +provision().catch((error) => { + console.error('An error occurred while provisioning cloud env:'); + console.error(error); + + process.exit(1); +}); diff --git a/packages/@n8n/benchmark/scripts/run.mjs b/packages/@n8n/benchmark/scripts/run.mjs index ece2e942d6f146..a276eee5fcc505 100755 --- a/packages/@n8n/benchmark/scripts/run.mjs +++ b/packages/@n8n/benchmark/scripts/run.mjs @@ -1,12 +1,9 @@ #!/usr/bin/env zx /** * Script to run benchmarks either on the cloud benchmark environment or locally. + * The cloud environment needs to be provisioned using Terraform before running the benchmarks. * * NOTE: Must be run in the root of the package. - * - * Usage: - * zx scripts/run.mjs - * */ // @ts-check import fs from 'fs'; diff --git a/packages/@n8n/benchmark/scripts/runInCloud.mjs b/packages/@n8n/benchmark/scripts/runInCloud.mjs index 8730807a6ed6c5..1c1b191ca9a0dd 100755 --- a/packages/@n8n/benchmark/scripts/runInCloud.mjs +++ b/packages/@n8n/benchmark/scripts/runInCloud.mjs @@ -39,16 +39,9 @@ export async function runInCloud(config) { isVerbose: config.isVerbose, }); - try { - const benchmarkEnv = await terraformClient.provisionEnvironment(); + const benchmarkEnv = await terraformClient.getTerraformOutputs(); - await runBenchmarksOnVm(config, benchmarkEnv); - } catch (error) { - console.error('An error occurred while running the benchmarks:'); - console.error(error); - } finally { - await terraformClient.destroyEnvironment(); - } + await runBenchmarksOnVm(config, benchmarkEnv); } async function ensureDependencies() { @@ -117,7 +110,15 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup } async function ensureVmIsReachable(sshClient) { - await sshClient.ssh('echo "VM is reachable"'); + try { + await sshClient.ssh('echo "VM is reachable"'); + } catch (error) { + console.error(`VM is not reachable: ${error.message}`); + console.error( + `Did you provision the cloud environment first with 'pnpm provision-cloud-env'? You can also run the benchmarks locally with 'pnpm run benchmark-locally'.`, + ); + process.exit(1); + } } /**