diff --git a/.evergreen.yml b/.evergreen.yml index 02a988b09..8b3fee7ba 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -6957,6 +6957,24 @@ functions: set -e .evergreen/run-evergreen-release.sh + run_perf_tests: + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + set -e + set -x + # fix rh-allow-sha1-signatures error, see run-e2e-tests.sh + export OPENSSL_CONF="" + npx -y mongodb-runner exec -t standalone --version=7.0.x-enterprise -- \ + sh -c 'MONGOSH_SMOKE_TEST_SERVER="$MONGODB_URI" ./dist/mongosh --perfTests > perf_results.json' + env: + DISTRO_ID: ${distro_id} + - command: perf.send + params: + file: src/perf_results.json + # Tasks will show up as the individual blocks in the Evergreen UI that can # pass or fail. # @@ -10426,6 +10444,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_darwin_x64 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: darwin + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: darwin-x64 + - func: run_perf_tests - name: e2e_tests_darwin_arm64_fips tags: ["e2e-test"] depends_on: @@ -10502,6 +10534,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_darwin_arm64 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: darwin_arm64 + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: darwin-arm64 + - func: run_perf_tests - name: e2e_tests_linux_x64_fips tags: ["e2e-test"] depends_on: @@ -10578,6 +10624,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_x64 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_x64_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-x64 + - func: run_perf_tests - name: e2e_tests_linux_x64_openssl11_fips tags: ["e2e-test"] depends_on: @@ -10654,6 +10714,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_x64_openssl11 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_x64_build_openssl11 + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-x64-openssl11 + - func: run_perf_tests - name: e2e_tests_linux_x64_openssl3_fips tags: ["e2e-test"] depends_on: @@ -10730,6 +10804,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_x64_openssl3 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_x64_build_openssl3 + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-x64-openssl3 + - func: run_perf_tests - name: e2e_tests_linux_arm64_fips tags: ["e2e-test"] depends_on: @@ -10806,6 +10894,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_arm64 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_arm64_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-arm64 + - func: run_perf_tests - name: e2e_tests_linux_arm64_openssl11_fips tags: ["e2e-test"] depends_on: @@ -10882,6 +10984,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_arm64_openssl11 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_arm64_build_openssl11 + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-arm64-openssl11 + - func: run_perf_tests - name: e2e_tests_linux_arm64_openssl3_fips tags: ["e2e-test"] depends_on: @@ -10958,6 +11074,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_arm64_openssl3 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_arm64_build_openssl3 + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-arm64-openssl3 + - func: run_perf_tests - name: e2e_tests_linux_ppc64le_fips tags: ["e2e-test"] depends_on: @@ -11034,6 +11164,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_ppc64le + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_ppc64le_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-ppc64le + - func: run_perf_tests - name: e2e_tests_linux_s390x_fips tags: ["e2e-test"] depends_on: @@ -11110,6 +11254,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_linux_s390x + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: linux_s390x_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: linux-s390x + - func: run_perf_tests - name: e2e_tests_win32_fips tags: ["e2e-test"] depends_on: @@ -11186,6 +11344,20 @@ tasks: mongosh_server_test_version: "6.0.x-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} + - name: perf_tests_win32 + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: win32_build + commands: + - func: checkout + - func: install + vars: + node_js_version: "20.11.1" + - func: download_compiled_artifact + vars: + executable_os_id: win32 + - func: run_perf_tests ### # EXECUTABLE CONNECTIVITY TESTS @@ -13516,7 +13688,7 @@ buildvariants: - name: package_and_upload_artifact_rpm_s390x - name: linux_x64_build display_name: "RHEL 7.0 x64 (build)" - run_on: rhel70-build + run_on: rhel70-xlarge expansions: executable_os_id: linux-x64 tags: ["nightly-driver"] @@ -13524,7 +13696,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_rhel8 display_name: "RHEL 8.0 x64 (build)" - run_on: rhel80-small + run_on: rhel80-build expansions: executable_os_id: linux-x64 tags: ["nightly-driver"] @@ -13532,7 +13704,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_openssl11 display_name: "RHEL 7.0 x64 (build, shared OpenSSL 1.1)" - run_on: rhel70-build + run_on: rhel70-xlarge expansions: executable_os_id: linux-x64-openssl11 mongosh_shared_openssl: openssl11 @@ -13541,7 +13713,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_openssl11_rhel8 display_name: "RHEL 8.0 x64 (build, shared OpenSSL 1.1)" - run_on: rhel80-small + run_on: rhel80-build expansions: executable_os_id: linux-x64-openssl11 mongosh_shared_openssl: openssl11 @@ -13550,7 +13722,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_openssl3 display_name: "RHEL 7.0 x64 (build, shared OpenSSL 3)" - run_on: rhel70-build + run_on: rhel70-xlarge expansions: executable_os_id: linux-x64-openssl3 mongosh_shared_openssl: openssl3 @@ -13559,7 +13731,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_openssl3_rhel8 display_name: "RHEL 8.0 x64 (build, shared OpenSSL 3)" - run_on: rhel80-small + run_on: rhel80-build expansions: executable_os_id: linux-x64-openssl3 mongosh_shared_openssl: openssl3 @@ -14100,3 +14272,9 @@ buildvariants: run_on: ubuntu2004-small tasks: - name: generate_license_and_vulnerability_report + + - name: perf_tests + display_name: "Performance Tests" + run_on: rhel90-dbx-perf-large + tasks: + - name: perf_tests_linux_x64 diff --git a/.evergreen/evergreen.yml.in b/.evergreen/evergreen.yml.in index f088c53f5..0f2e56a41 100644 --- a/.evergreen/evergreen.yml.in +++ b/.evergreen/evergreen.yml.in @@ -825,6 +825,24 @@ functions: set -e .evergreen/run-evergreen-release.sh + run_perf_tests: + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + set -e + set -x + # fix rh-allow-sha1-signatures error, see run-e2e-tests.sh + export OPENSSL_CONF="" + npx -y mongodb-runner exec -t standalone --version=7.0.x-enterprise -- \ + sh -c 'MONGOSH_SMOKE_TEST_SERVER="$MONGODB_URI" ./dist/mongosh --perfTests > perf_results.json' + env: + DISTRO_ID: ${distro_id} + - command: perf.send + params: + file: src/perf_results.json + # Tasks will show up as the individual blocks in the Evergreen UI that can # pass or fail. # @@ -996,7 +1014,22 @@ tasks: mongosh_server_test_version: "<% out(mVersion) %>-enterprise" mongosh_test_e2e_force_fips: "<% out(fipsVariant === 'fips' ? '1' : '') %>" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - <% } } } %> + <% } } %> + - name: perf_tests_<% out(executableOsId.replace(/-/g, '_')) %> + tags: ["perf-test"] + depends_on: + - name: compile_artifact + variant: <% out(compileBuildVariant) %> + commands: + - func: checkout + - func: install + vars: + node_js_version: "<% out(NODE_JS_VERSION_20) %>" + - func: download_compiled_artifact + vars: + executable_os_id: <% out(executableOsId) %> + - func: run_perf_tests + <% } %> ### # EXECUTABLE CONNECTIVITY TESTS @@ -1225,7 +1258,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_rhel8 display_name: "RHEL 8.0 x64 (build)" - run_on: rhel80-small + run_on: rhel80-build expansions: executable_os_id: linux-x64 tags: ["nightly-driver"] @@ -1242,7 +1275,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_openssl11_rhel8 display_name: "RHEL 8.0 x64 (build, shared OpenSSL 1.1)" - run_on: rhel80-small + run_on: rhel80-build expansions: executable_os_id: linux-x64-openssl11 mongosh_shared_openssl: openssl11 @@ -1260,7 +1293,7 @@ buildvariants: - name: compile_artifact - name: linux_x64_build_openssl3_rhel8 display_name: "RHEL 8.0 x64 (build, shared OpenSSL 3)" - run_on: rhel80-small + run_on: rhel80-build expansions: executable_os_id: linux-x64-openssl3 mongosh_shared_openssl: openssl3 @@ -1594,3 +1627,9 @@ buildvariants: run_on: ubuntu2004-small tasks: - name: generate_license_and_vulnerability_report + + - name: perf_tests + display_name: "Performance Tests" + run_on: rhel90-dbx-perf-large + tasks: + - name: perf_tests_linux_x64 diff --git a/package-lock.json b/package-lock.json index 51ebdf1b4..a69775475 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31051,6 +31051,7 @@ "ansi-escape-sequences": "^5.1.2", "askcharacter": "^1.0.0", "askpassword": "^1.2.4", + "escape-string-regexp": "^4.0.0", "is-recoverable-error": "^1.0.3", "js-yaml": "^4.1.0", "mongodb-connection-string-url": "^3.0.0", @@ -38249,6 +38250,7 @@ "askpassword": "^1.2.4", "chai-as-promised": "^7.1.1", "depcheck": "^1.4.3", + "escape-string-regexp": "^4.0.0", "eslint": "^7.25.0", "get-console-process-list": "^1.0.4", "is-recoverable-error": "^1.0.3", diff --git a/packages/arg-parser/src/cli-options.ts b/packages/arg-parser/src/cli-options.ts index 89ddd15a5..8f6f16325 100644 --- a/packages/arg-parser/src/cli-options.ts +++ b/packages/arg-parser/src/cli-options.ts @@ -20,6 +20,7 @@ export interface CliOptions { cryptSharedLibPath?: string; db?: string; eval?: string[]; + exposeAsyncRewriter?: boolean; // internal testing only gssapiServiceName?: string; sspiHostnameCanonicalization?: string; sspiRealmOverride?: string; diff --git a/packages/cli-repl/package.json b/packages/cli-repl/package.json index df67d72b9..9a1396ade 100644 --- a/packages/cli-repl/package.json +++ b/packages/cli-repl/package.json @@ -77,6 +77,7 @@ "ansi-escape-sequences": "^5.1.2", "askcharacter": "^1.0.0", "askpassword": "^1.2.4", + "escape-string-regexp": "^4.0.0", "is-recoverable-error": "^1.0.3", "js-yaml": "^4.1.0", "mongodb-connection-string-url": "^3.0.0", diff --git a/packages/cli-repl/src/arg-parser.ts b/packages/cli-repl/src/arg-parser.ts index 2e2c45e48..663e580ab 100644 --- a/packages/cli-repl/src/arg-parser.ts +++ b/packages/cli-repl/src/arg-parser.ts @@ -58,11 +58,13 @@ const OPTIONS = { 'apiDeprecationErrors', 'apiStrict', 'buildInfo', + 'exposeAsyncRewriter', 'help', 'ipv6', 'nodb', 'norc', 'oidcTrustedEndpoint', + 'perfTests', 'quiet', 'retryWrites', 'shell', @@ -155,6 +157,7 @@ function isConnectionSpecifier(arg?: string): boolean { */ export function parseCliArgs(args: string[]): CliOptions & { smokeTests: boolean; + perfTests: boolean; buildInfo: boolean; _argParseWarnings: string[]; } { @@ -163,6 +166,7 @@ export function parseCliArgs(args: string[]): CliOptions & { const parsed = parser(programArgs, OPTIONS) as unknown as CliOptions & { smokeTests: boolean; + perfTests: boolean; buildInfo: boolean; _argParseWarnings: string[]; _?: string[]; diff --git a/packages/cli-repl/src/mongosh-repl.ts b/packages/cli-repl/src/mongosh-repl.ts index baa8ce0e7..afa567677 100644 --- a/packages/cli-repl/src/mongosh-repl.ts +++ b/packages/cli-repl/src/mongosh-repl.ts @@ -55,6 +55,7 @@ export type MongoshCliOptions = ShellCliOptions & { * for async error tracking, or not. */ jsContext?: JSContext; + exposeAsyncRewriter?: boolean; }; /** @@ -186,7 +187,8 @@ class MongoshNodeRepl implements EvaluationListener { const shellEvaluator = new ShellEvaluator( instanceState, (value: any) => value, - markTime + markTime, + !!this.shellCliOptions.exposeAsyncRewriter ); instanceState.setEvaluationListener(this); await instanceState.fetchConnectionInfo(); diff --git a/packages/cli-repl/src/run.ts b/packages/cli-repl/src/run.ts index 89ab93d99..30ace6ece 100644 --- a/packages/cli-repl/src/run.ts +++ b/packages/cli-repl/src/run.ts @@ -131,26 +131,25 @@ async function main() { console.log(JSON.stringify(await buildInfo(), null, ' ')); return; } - if (options.smokeTests) { + if (options.smokeTests || options.perfTests) { const smokeTestServer = process.env.MONGOSH_SMOKE_TEST_SERVER; const cryptLibraryOpts = options.cryptSharedLibPath ? [`--cryptSharedLibPath=${options.cryptSharedLibPath}`] : []; if (process.execPath === process.argv[1]) { // This is the compiled binary. Use only the path to it. - await runSmokeTests( + await runSmokeTests({ smokeTestServer, - process.execPath, - ...cryptLibraryOpts - ); + args: [process.execPath, ...cryptLibraryOpts], + wantPerformanceTesting: !!options.perfTests, + }); } else { // This is not the compiled binary. Use node + this script. - await runSmokeTests( + await runSmokeTests({ smokeTestServer, - process.execPath, - process.argv[1], - ...cryptLibraryOpts - ); + args: [process.execPath, process.argv[1], ...cryptLibraryOpts], + wantPerformanceTesting: !!options.perfTests, + }); } return; } diff --git a/packages/cli-repl/src/smoke-tests.spec.ts b/packages/cli-repl/src/smoke-tests.spec.ts index 02f762a89..68402b625 100644 --- a/packages/cli-repl/src/smoke-tests.spec.ts +++ b/packages/cli-repl/src/smoke-tests.spec.ts @@ -13,17 +13,21 @@ describe('smoke tests', function () { cryptLibrary = await downloadCurrentCryptSharedLibrary(); }); - it('self-test passes', async function () { + it('self-test passes (perf testing disabled)', async function () { + this.timeout(120_000); // Use ts-node to run the .ts files directly so nyc can pick them up for // coverage. - await runSmokeTests( - await testServer.connectionString(), - process.execPath, - '-r', - 'ts-node/register', - path.resolve(__dirname, 'run.ts'), - '--cryptSharedLibPath', - cryptLibrary - ); + await runSmokeTests({ + smokeTestServer: await testServer.connectionString(), + args: [ + process.execPath, + '-r', + 'ts-node/register', + path.resolve(__dirname, 'run.ts'), + '--cryptSharedLibPath', + cryptLibrary, + ], + wantPerformanceTesting: false, + }); }); }); diff --git a/packages/cli-repl/src/smoke-tests.ts b/packages/cli-repl/src/smoke-tests.ts index ba7aa831b..afe24638b 100644 --- a/packages/cli-repl/src/smoke-tests.ts +++ b/packages/cli-repl/src/smoke-tests.ts @@ -1,8 +1,49 @@ import assert from 'assert'; +import { promises as fs } from 'fs'; import { once } from 'events'; import { redactURICredentials } from '@mongosh/history'; import fleSmokeTestScript from './smoke-tests-fle'; -import { buildInfo } from './build-info'; +import { baseBuildInfo, buildInfo } from './build-info'; +import escapeRegexp from 'escape-string-regexp'; +import path from 'path'; +import os from 'os'; + +export interface SelfTestOptions { + smokeTestServer: string | undefined; + args: string[]; // includes executable + args immediately after executable + wantPerformanceTesting: boolean; +} + +// https://pkg.go.dev/github.com/evergreen-ci/poplar#TestInfo +interface PerfTestResult { + info: { + test_name: string; + tags: string[]; + }; + created_at: string; + completed_at: string; + artifacts: unknown[]; + metrics: PerfTestMetric[]; +} + +interface PerfTestMetric { + name: string; + type: + | 'SUM' + | 'MEAN' + | 'MEDIAN' + | 'MAX' + | 'MIN' + | 'STANDARD_DEVIATION' + | 'THROUGHPUT' + | 'LATENCY' + | 'PERCENTILE_99TH' + | 'PERCENTILE_95TH' + | 'PERCENTILE_90TH' + | 'PERCENTILE_80TH' + | 'PERCENTILE_50TH'; + value: number; +} /** * Run smoke tests on an executable, e.g. @@ -13,13 +54,16 @@ import { buildInfo } from './build-info'; * @param executable The Node.js/mongosh executable to use * @param args Arguments to pass to the Node.js/mongosh executable */ -export async function runSmokeTests( - smokeTestServer: string | undefined, - executable: string, - ...args: string[] -): Promise { - console.log('MONGOSH_SMOKE_TEST_SERVER set?', !!smokeTestServer); - if (process.env.IS_CI) { +export async function runSmokeTests({ + smokeTestServer, + args: [executable, ...args], + wantPerformanceTesting, +}: SelfTestOptions): Promise { + if (!wantPerformanceTesting) { + console.log('MONGOSH_SMOKE_TEST_SERVER set?', !!smokeTestServer); + } + + if (process.env.IS_CI || wantPerformanceTesting) { assert( !!smokeTestServer, 'Make sure MONGOSH_SMOKE_TEST_SERVER is set in CI' @@ -29,31 +73,50 @@ export async function runSmokeTests( const expectFipsSupport = !!process.env.MONGOSH_SMOKE_TEST_OS_HAS_FIPS_SUPPORT && (await buildInfo()).sharedOpenssl; - console.log('FIPS support required to pass?', { expectFipsSupport }); + if (!wantPerformanceTesting) { + console.log('FIPS support required to pass?', { expectFipsSupport }); + } + const perfResults: PerfTestResult[] = []; - for (const { input, output, testArgs, includeStderr, exitCode } of [ + for (const { + name, + tags, + input, + output, + testArgs, + includeStderr, + exitCode, + perfTestIterations, + } of [ { + name: 'print_hello_world', input: 'print("He" + "llo" + " Wor" + "ld!")', output: /Hello World!/, includeStderr: false, testArgs: ['--nodb'], exitCode: 0, + perfTestIterations: 0, }, { + name: 'eval_nodb_error', input: '', output: /ReferenceError/, includeStderr: true, testArgs: ['--nodb', '--eval', 'foo.bar()'], exitCode: 1, + perfTestIterations: 0, }, { + name: 'eval_nodb_print', input: '', output: /Hello World!/, includeStderr: false, testArgs: ['--nodb', '--eval', 'print("He" + "llo" + " Wor" + "ld!")'], exitCode: 0, + perfTestIterations: 0, }, { + name: 'eval_nodb_print_plainvm', input: '', output: /Hello World!/, includeStderr: false, @@ -64,8 +127,11 @@ export async function runSmokeTests( '--jsContext=plain-vm', ], exitCode: 0, + perfTestIterations: 20, + tags: ['startup'], }, { + name: 'eval_nodb_print_repl', input: '', output: /Hello World!/, includeStderr: false, @@ -76,27 +142,54 @@ export async function runSmokeTests( '--jsContext=repl', ], exitCode: 0, + perfTestIterations: 20, + tags: ['startup'], }, { + name: 'mongosh_version', + input: '', + output: new RegExp(escapeRegexp(baseBuildInfo().version)), + includeStderr: false, + testArgs: ['--version'], + exitCode: 0, + perfTestIterations: 20, + tags: ['startup'], + }, + { + name: 'crypto_fips_md5', input: 'crypto.createHash("md5").update("hello").digest("hex")', output: expectFipsSupport ? /disabled for FIPS|digital envelope routines::unsupported/i : /disabled for FIPS|digital envelope routines::unsupported|Could not enable FIPS mode/i, includeStderr: true, testArgs: ['--tlsFIPSMode', '--nodb'], + perfTestIterations: 0, }, { + name: 'crypto_fips_sha256', input: 'crypto.createHash("sha256").update("hello").digest("hex")', output: expectFipsSupport ? /2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824/i : /2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824|digital envelope routines::unsupported|Could not enable FIPS mode/i, includeStderr: true, testArgs: ['--tlsFIPSMode', '--nodb'], + perfTestIterations: 0, + }, + { + name: 'async_rewrite', + input: + 'for (let i = 0; i < 100; i++) __asyncRewrite(String([].forEach)); print("done")', + output: /done/, + includeStderr: false, + testArgs: ['--exposeAsyncRewriter', '--nodb'], + perfTestIterations: 20, + tags: ['async_rewrite'], }, ].concat( smokeTestServer ? [ { + name: 'db_insert_retrieve', input: ` const dbname = "testdb_simplesmoke" + new Date().getTime(); use(dbname); @@ -109,27 +202,153 @@ export async function runSmokeTests( includeStderr: false, exitCode: 0, testArgs: [smokeTestServer], + perfTestIterations: 0, }, { + name: 'fle', input: fleSmokeTestScript, output: /Test succeeded|Test skipped/, includeStderr: false, exitCode: 0, testArgs: [smokeTestServer], + perfTestIterations: 0, + }, + { + name: 'db_eval_plainvm', + input: '', + output: /"ok": ?1\b/, + includeStderr: false, + exitCode: 0, + testArgs: [ + smokeTestServer, + '--eval', + 'db.hello()', + '--json=relaxed', + '--jsContext=plain-vm', + ], + perfTestIterations: 20, + tags: ['startup'], + }, + { + name: 'db_eval_repl', + input: '', + output: /"ok": ?1\b/, + includeStderr: false, + exitCode: 0, + testArgs: [ + smokeTestServer, + '--eval', + 'db.hello()', + '--json=relaxed', + '--jsContext=repl', + ], + perfTestIterations: 20, + tags: ['startup'], + }, + { + name: 'db_cursor_iteration_repl', + input: `let count = 0; for (const item of ${manyDocsCursor( + 12345 + )}) count++; print(count);`, + output: + /\b12345\b|Unrecognized pipeline stage name|is not allowed in user requests/, + includeStderr: true, + testArgs: [smokeTestServer, '--jsContext=repl'], + perfTestIterations: 20, + tags: ['db', 'cursor_iteration'], + }, + { + name: 'db_cursor_iteration_plainvm', + input: `let count = 0; for (const item of ${manyDocsCursor( + 200_000 + )}) count++; print(count);`, + output: + /\b200000\b|Unrecognized pipeline stage name|is not allowed in user requests/, + includeStderr: true, + testArgs: [ + smokeTestServer, + '--file=$INPUT_AS_FILE', + '--jsContext=plain-vm', + ], + perfTestIterations: 20, + tags: ['db', 'cursor_iteration'], + }, + { + name: 'db_repeat_command', + input: `let res;for (const item of [...Array(5000).keys()]) res = EJSON.stringify(db.hello()); print(res)`, + output: /"ok": ?1\b/, + includeStderr: false, + exitCode: 0, + testArgs: [ + smokeTestServer, + '--file=$INPUT_AS_FILE', + '--jsContext=plain-vm', + ], + perfTestIterations: 20, + tags: ['db'], }, ] : [] )) { - await runSmokeTest({ + const cleanup: (() => Promise)[] = []; + + let actualInput = input; + for (const [index, arg] of testArgs.entries()) { + if (arg.includes('$INPUT_AS_FILE')) { + const tmpfile = path.join( + os.tmpdir(), + `mongosh_smoke_test_${name}_${Date.now()}.js` + ); + await fs.writeFile(tmpfile, input, { mode: 0o600, flag: 'wx' }); + cleanup.unshift(async () => await fs.unlink(tmpfile)); + testArgs[index] = arg.replace('$INPUT_AS_FILE', tmpfile); + actualInput = ''; + } + } + + const smokeTestArgs = { + name, executable, args: [...args, ...testArgs], - input, + input: actualInput, output, includeStderr, exitCode, - }); + printSuccessResults: !wantPerformanceTesting, + }; + + try { + if (!wantPerformanceTesting) { + await runSmokeTest(smokeTestArgs); + } else { + const created_at = new Date().toISOString(); + if (!perfTestIterations) continue; + const durations: number[] = []; + for (let i = 0; i < perfTestIterations; i++) { + const { durationSeconds } = await runSmokeTest(smokeTestArgs); + durations.push(durationSeconds); + } + const completed_at = new Date().toISOString(); + perfResults.push({ + info: { + test_name: name, + tags: [...(tags ?? []), process.arch, process.platform], + }, + created_at, + completed_at, + artifacts: [], + metrics: buildMetrics(durations), + }); + } + } finally { + for (const cleaner of cleanup) await cleaner(); + } + } + if (wantPerformanceTesting) { + console.log(JSON.stringify(perfResults)); + } else { + console.log('all tests passed'); } - console.log('all tests passed'); } /** @@ -141,20 +360,25 @@ export async function runSmokeTests( * @param output Expected contents of stdout */ async function runSmokeTest({ + name, executable, args, input, output, exitCode, includeStderr, + printSuccessResults, }: { + name: string; executable: string; args: string[]; input: string; output: RegExp; exitCode?: number; includeStderr?: boolean; -}): Promise { + printSuccessResults?: boolean; +}): Promise<{ durationSeconds: number }> { + const startTime = process.hrtime.bigint(); // 'child_process' is not supported in startup snapshots yet. // eslint-disable-next-line const { spawn } = require('child_process') as typeof import('child_process'); @@ -175,7 +399,10 @@ async function runSmokeTest({ once(proc.stdout!, 'end'), proc.stderr && once(proc.stderr, 'end'), ]); + const durationSeconds = Number(process.hrtime.bigint() - startTime) / 1e9; const metadata = { + name, + durationSeconds, input, output, stdout, @@ -189,9 +416,63 @@ async function runSmokeTest({ if (exitCode !== undefined) { assert.strictEqual(actualExitCode, exitCode); } - console.error({ status: 'success', ...metadata }); + if (printSuccessResults !== false) + console.error({ status: 'success', ...metadata }); + return { + durationSeconds, + }; } catch (err: any) { console.error({ status: 'failure', ...metadata }); throw err; } } + +function manyDocsCursor(n: number): string { + return `db.aggregate([ + { $documents: [{}] }, + { $set: { + field: { $reduce: { // ~ 2**n documents + input: [...Array(${Math.ceil( + Math.log2(n) + )}).keys()], initialValue: [0], in: { $concatArrays: ['$$value', '$$value'] } + } } + } }, + { $unwind: '$field' }, + { $limit: ${n} } + ])`; +} + +function buildMetrics(durations: number[]): PerfTestMetric[] { + // These might not be the fastest or most numerically stable ways of computing these values, + // but this should be fine for our purposes + const mean = durations.reduce((a, b) => a + b, 0) / durations.length; + const stddev = Math.sqrt( + durations.map((d) => (d - mean) ** 2).reduce((a, b) => a + b, 0) / + (durations.length - 1) + ); + durations.sort(); + const median = durations[Math.floor(durations.length * 0.5)]; + const percentile95 = durations[Math.floor(durations.length * 0.95)]; + return [ + { + name: 'duration_sec_mean', + type: 'MEAN', + value: mean, + }, + { + name: 'duration_sec_stddev', + type: 'STANDARD_DEVIATION', + value: stddev, + }, + { + name: 'duration_sec_median', + type: 'MEDIAN', + value: median, + }, + { + name: 'duration_sec_p95', + type: 'PERCENTILE_95TH', + value: percentile95, + }, + ]; +} diff --git a/packages/shell-evaluator/src/shell-evaluator.ts b/packages/shell-evaluator/src/shell-evaluator.ts index 34279435a..4bb479c98 100644 --- a/packages/shell-evaluator/src/shell-evaluator.ts +++ b/packages/shell-evaluator/src/shell-evaluator.ts @@ -41,16 +41,19 @@ class ShellEvaluator { private hasAppliedAsyncWriterRuntimeSupport = true; private asyncWriter: AsyncWriter; private markTime?: (category: TimingCategory, label: string) => void; + private exposeAsyncRewriter: boolean; constructor( instanceState: ShellInstanceState, resultHandler: ResultHandler = toShellResult as any, - markTime?: (category: TimingCategory, label: string) => void + markTime?: (category: TimingCategory, label: string) => void, + exposeAsyncRewriter?: boolean ) { this.instanceState = instanceState; this.resultHandler = resultHandler; this.asyncWriter = new AsyncWriter(); this.hasAppliedAsyncWriterRuntimeSupport = false; + this.exposeAsyncRewriter = !!exposeAsyncRewriter; this.markTime = markTime; } @@ -89,6 +92,10 @@ class ShellEvaluator { return shellApi[cmd](...argv); } + if (this.exposeAsyncRewriter) { + (context as any).__asyncRewrite = () => this.asyncWriter.process(input); + } + this.markTime?.(TimingCategories.AsyncRewrite, 'start async rewrite'); let rewrittenInput = this.asyncWriter.process(input); this.markTime?.(TimingCategories.AsyncRewrite, 'done async rewrite');