diff --git a/.gitattributes b/.gitattributes index bd2acce6f6..cd1d488e62 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,4 +15,5 @@ # Github Linguist configuration (https://github.com/github/linguist) yarn.lock linguist-generated *.snap linguist-generated +packages/@jsii/benchmarks/fixtures/** linguist-vendored docs/** linguist-documentation diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index bb2a9b7e40..5b7af8dc56 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -73,7 +73,7 @@ jobs: git config user.email "aws-cdk+automation@amazon.com" - name: Prepare Commit run: |- - rsync --delete --exclude=.git --recursive ${{ runner.temp }}/site/ ./ + rsync --delete --exclude=.git --exclude=dev --recursive ${{ runner.temp }}/site/ ./ touch .nojekyll git add . git diff --cached --exit-code >/dev/null || ( diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f032b82124..4a005ed748 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -365,3 +365,51 @@ jobs: && echo "Untracked files: ${untracked:-}" \ && test -z "${untracked}" shell: bash + benchmark: + name: Run benchmark suite + runs-on: ubuntu-latest + permissions: + contents: write + needs: build + steps: + # Check out the code + - name: Download Artifact + uses: actions/download-artifact@v3 + with: + name: built-tree + - name: Extract Artifact + run: |- + echo "::group::Untar Archive" + tar zxvf built-tree.tgz + echo "::endgroup" + + rm built-tree.tgz + - name: Set up Node + uses: actions/setup-node@v3 + with: + cache: yarn + node-version: '14' + - name: Install Dependencies + run: yarn install --frozen-lockfile + - name: Run Benchmark + working-directory: packages/@jsii/benchmarks + run: yarn bench --output ${{ runner.temp }}/bench-output.json + - name: Compare Benchmark Results + if: github.event_name == 'pull_request' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: jsii Benchmark Regression + tool: 'customSmallerIsBetter' + output-file-path: ${{ runner.temp }}/bench-output.json + comment-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-alert: true + - name: Upload Benchmark Results + if: github.event_name == 'push' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: jsii Benchmark + tool: 'customSmallerIsBetter' + output-file-path: ${{ runner.temp }}/bench-output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true diff --git a/packages/@jsii/benchmarks/.eslintrc.yaml b/packages/@jsii/benchmarks/.eslintrc.yaml new file mode 100644 index 0000000000..18ccbb678d --- /dev/null +++ b/packages/@jsii/benchmarks/.eslintrc.yaml @@ -0,0 +1,9 @@ +--- +extends: ../../../eslint-config.yaml +ignorePatterns: + - fixtures + +rules: + 'import/no-extraneous-dependencies': + - error + - devDependencies: ['**/scripts/**'] diff --git a/packages/@jsii/benchmarks/.gitignore b/packages/@jsii/benchmarks/.gitignore new file mode 100644 index 0000000000..6c6558b43c --- /dev/null +++ b/packages/@jsii/benchmarks/.gitignore @@ -0,0 +1,3 @@ +output.txt +*.d.ts +*.js diff --git a/packages/@jsii/benchmarks/README.md b/packages/@jsii/benchmarks/README.md new file mode 100644 index 0000000000..9dbd300c1f --- /dev/null +++ b/packages/@jsii/benchmarks/README.md @@ -0,0 +1,25 @@ +# jsii Benchmarks + +This package is meant to collect benchmarks for `jsii`, `jsii-pacmak`, and any other jsii packages sourced in TS. It +contains a basic benchmark runner in [`benchmark.ts`](lib/benchmark.ts) that uses the `perf_hooks` module in order to +time synchronous functions. + +## Usage + +There is a small CLI app wrapping calls to the benchmarks defined. To call the benchmarks: + +``` +yarn benchmark +``` + +To output benchmark run results to a json file, pass the `--output` option + +``` +yarn benchmark --output my-file.json +``` + +## Output Format + +The output format is JSON and is used by the +[continous benchmark action](https://github.com/benchmark-action/github-action-benchmark) which tracks the results of +benchmarks over time. diff --git a/packages/@jsii/benchmarks/bin/benchmark.ts b/packages/@jsii/benchmarks/bin/benchmark.ts new file mode 100644 index 0000000000..0d555540d7 --- /dev/null +++ b/packages/@jsii/benchmarks/bin/benchmark.ts @@ -0,0 +1,86 @@ +import * as fs from 'fs-extra'; +import * as yargs from 'yargs'; + +import { benchmarks } from '../lib'; +import { Benchmark } from '../lib/benchmark'; + +/** + * Format of benchmark output used by continous benchmarking action. + * See [documentation](https://github.com/benchmark-action/github-action-benchmark/blob/master/README.md) for details + */ +interface ResultsJson { + /** + * The name of the benchmark + */ + name: string; + + /** + * The unit of measure, usually seconds + */ + unit: string; + + /** + * The result of the measurement, usually an average over x iterations + */ + value: number; + + /** + * The variance of all runs + */ + range: number; + + /** + * Extra information about the benchmark, displayed in a tooltip + */ + extra: string; +} + +(async () => { + /* eslint-disable-next-line @typescript-eslint/await-thenable */ + const argv = await yargs + .command('$0', 'Runs jsii benchmark tests and displays results', (argv) => + argv.option('output', { + type: 'string', + desc: 'location of benchmark results json file, does not output to file if not specified.', + }), + ) + .help().argv; + + // Run list of benchmarks in sequence + const resultsJson: ResultsJson[] = await benchmarks.reduce( + async ( + accum: Promise, + benchmark: Benchmark, + ): Promise => { + const prev = await accum; + const result = await benchmark.run(); + const extra = `${result.name} averaged ${result.average} milliseconds over ${result.iterations.length} runs`; + console.log(extra); + return [ + ...prev, + { + name: result.name, + unit: 'milliseconds', + value: result.average, + range: result.variance, + extra, + }, + ]; + }, + Promise.resolve([]), + ); + + if (argv.output) { + await fs.writeJson(argv.output, resultsJson, { spaces: 2 }); + console.log(`results written to ${argv.output}`); + } + + return resultsJson; +})() + .then((results) => { + console.log(`successfully completed ${results.length} benchmarks`); + }) + .catch((e) => { + console.error(`Error: ${e.stack}`); + process.exitCode = -1; + }); diff --git a/packages/@jsii/benchmarks/fixtures/aws-cdk-lib@v2-21-1.tgz b/packages/@jsii/benchmarks/fixtures/aws-cdk-lib@v2-21-1.tgz new file mode 100644 index 0000000000..4722f5ff82 Binary files /dev/null and b/packages/@jsii/benchmarks/fixtures/aws-cdk-lib@v2-21-1.tgz differ diff --git a/packages/@jsii/benchmarks/lib/benchmark.ts b/packages/@jsii/benchmarks/lib/benchmark.ts new file mode 100644 index 0000000000..5fc5b4b314 --- /dev/null +++ b/packages/@jsii/benchmarks/lib/benchmark.ts @@ -0,0 +1,158 @@ +import { performance, PerformanceObserver, PerformanceEntry } from 'perf_hooks'; + +/** + * Result of a benchmark run + */ +interface Result { + /** + * The name of the benchmark + */ + readonly name: string; + + /** + * The average duration across all iterations + */ + readonly average: number; + + /** + * Maximum duration across all iteraions + */ + readonly max: number; + + /** + * Minimum duration across all iterations + */ + readonly min: number; + + /** + * max - min + */ + readonly variance: number; + + /** + * Results of individual runs + */ + readonly iterations: readonly PerformanceEntry[]; +} + +/** + * A simple benchmark for measuring synchronous functions. Uses the `perf_hooks` + * module to measure how long a subject takes to execute and averages the result + * over all runs. Runs `setup`, `beforeEach`, `afterEach`, and `teardown` + * lifecycle hooks before, between, and after runs. These functions, and the + * subject function, have access to an optionally defined `context` object that + * can be returned from the `setup` function. This allows referencing shared + * state across benchmark runs and lifecycle hooks to do things like setup, + * teardown, stubbing, etc. + */ +export class Benchmark { + /** + * How many times to run the subject + */ + #iterations = 5; + + /** + * Results of individual runs + */ + #results: PerformanceEntry[] = []; + + public constructor(private readonly name: string) {} + #setup: () => C | Promise = () => ({} as C); + #subject: (ctx: C) => void = () => undefined; + #beforeEach: (ctx: C) => void = () => undefined; + #afterEach: (ctx: C) => void = () => undefined; + #teardown: (ctx: C) => void = () => undefined; + + /** + * Create a setup function to be run once before the benchmark, optionally + * return a context object to be used across runs and lifecycle functions. + */ + public setup(fn: () => T | Promise) { + this.#setup = fn; + return this as unknown as Benchmark; + } + + /** + * Create a teardown function to be run once after all benchmark runs. Use to + * clean up your mess. + */ + public teardown(fn: (ctx: C) => void) { + this.#teardown = fn; + return this; + } + + /** + * Create a beforeEach function to be run before each iteration. Use to reset + * state the subject may have changed. + */ + public beforeEach(fn: (ctx: C) => void) { + this.#beforeEach = fn; + return this; + } + + /** + * Create an afterEach function to be run after each iteration. Use to reset + * state the subject may have changed. + */ + public afterEach(fn: (ctx: C) => void) { + this.#afterEach = fn; + return this; + } + + /** + * Setup the subject to be measured. + */ + public subject(fn: (ctx: C) => void) { + this.#subject = fn; + return this; + } + + /** + * Set the number of iterations to be run. + */ + public iterations(i: number) { + this.#iterations = i; + return this; + } + + /** + * Run and measure the benchmark + */ + public async run(): Promise { + const c = await this.#setup?.(); + return new Promise((ok) => { + const wrapped = performance.timerify(this.#subject); + const obs = new PerformanceObserver((list, observer) => { + this.#results = list.getEntries(); + performance.clearMarks(); + observer.disconnect(); + const durations = this.#results.map((i) => i.duration); + const max = Math.max(...durations); + const min = Math.min(...durations); + const variance = max - min; + + return ok({ + name: this.name, + average: + durations.reduce((accum, duration) => accum + duration, 0) / + durations.length, + max, + min, + variance, + iterations: this.#results, + }); + }); + obs.observe({ entryTypes: ['function'] }); + + try { + for (let i = 0; i < this.#iterations; i++) { + this.#beforeEach(c); + wrapped(c); + this.#afterEach(c); + } + } finally { + this.#teardown(c); + } + }); + } +} diff --git a/packages/@jsii/benchmarks/lib/constants.ts b/packages/@jsii/benchmarks/lib/constants.ts new file mode 100644 index 0000000000..9e9eaf87ea --- /dev/null +++ b/packages/@jsii/benchmarks/lib/constants.ts @@ -0,0 +1,9 @@ +import * as path from 'path'; + +export const fixturesDir = path.resolve(__dirname, '..', 'fixtures'); + +export const cdkTagv2_21_1 = 'v2.21.1'; +export const cdkv2_21_1 = path.resolve( + fixturesDir, + `aws-cdk-lib@${cdkTagv2_21_1.replace(/\./g, '-')}.tgz`, +); diff --git a/packages/@jsii/benchmarks/lib/index.ts b/packages/@jsii/benchmarks/lib/index.ts new file mode 100644 index 0000000000..bcf5319021 --- /dev/null +++ b/packages/@jsii/benchmarks/lib/index.ts @@ -0,0 +1,46 @@ +import * as cp from 'child_process'; +import * as fs from 'fs-extra'; +import { Compiler } from 'jsii/lib/compiler'; +import { loadProjectInfo } from 'jsii/lib/project-info'; +import * as os from 'os'; +import * as path from 'path'; + +import { Benchmark } from './benchmark'; +import { cdkv2_21_1, cdkTagv2_21_1 } from './constants'; +import { streamUntar } from './util'; + +// Always run against the same version of CDK source +const cdk = new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1}`) + .setup(async () => { + const sourceDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'), + ); + await streamUntar(cdkv2_21_1, { cwd: sourceDir }); + cp.execSync('npm ci', { cwd: sourceDir }); + + // Working directory for benchmark + const workingDir = fs.mkdtempSync( + path.join(os.tmpdir(), `jsii-cdk-bench@${cdkTagv2_21_1}`), + ); + + return { + workingDir, + sourceDir, + } as const; + }) + .beforeEach(({ workingDir, sourceDir }) => { + fs.removeSync(workingDir); + fs.copySync(sourceDir, workingDir); + }) + .subject(({ workingDir }) => { + const { projectInfo } = loadProjectInfo(workingDir); + const compiler = new Compiler({ projectInfo }); + + compiler.emit(); + }) + .teardown(({ workingDir, sourceDir }) => { + fs.removeSync(workingDir); + fs.removeSync(sourceDir); + }); + +export const benchmarks = [cdk]; diff --git a/packages/@jsii/benchmarks/lib/util.ts b/packages/@jsii/benchmarks/lib/util.ts new file mode 100644 index 0000000000..6a121dee03 --- /dev/null +++ b/packages/@jsii/benchmarks/lib/util.ts @@ -0,0 +1,11 @@ +import * as fs from 'fs-extra'; +import * as tar from 'tar'; + +export async function streamUntar(file: string, config: tar.ExtractOptions) { + const stream = fs.createReadStream(file).pipe(tar.x(config)); + + return new Promise((ok, ko) => { + stream.on('end', ok); + stream.on('error', (error: Error) => ko(error)); + }); +} diff --git a/packages/@jsii/benchmarks/package.json b/packages/@jsii/benchmarks/package.json new file mode 100644 index 0000000000..6f4ad3d3c3 --- /dev/null +++ b/packages/@jsii/benchmarks/package.json @@ -0,0 +1,35 @@ +{ + "name": "@jsii/benchmarks", + "version": "0.0.0", + "private": true, + "description": "the jsii benchmark suite", + "main": "index.js", + "dependencies": { + "fs-extra": "^10.0.1", + "jsii": "^0.0.0", + "tar": "^6.1.11", + "yargs": "^16.2.0" + }, + "devDependencies": { + "@types/glob": "^7.2.0", + "eslint": "^8.13.0", + "glob": "^8.0.1", + "jsii-calc": "^3.20.120", + "prettier": "^2.6.2", + "typescript": "~3.9.10" + }, + "scripts": { + "build": "tsc --build && npm run lint", + "watch": "tsc -w", + "bench": "node bin/benchmark.js", + "snapshot": "node scripts/snapshot-package.js", + "lint": "eslint . --ext .ts --ignore-path=.gitignore", + "lint:fix": "yarn lint --fix" + }, + "bin": { + "benchmark": "bin/benchmark.js" + }, + "keywords": [], + "author": "Amazon Web Services, Inc.", + "license": "Apache-2.0" +} diff --git a/packages/@jsii/benchmarks/scripts/snapshot-package.ts b/packages/@jsii/benchmarks/scripts/snapshot-package.ts new file mode 100644 index 0000000000..44b828698e --- /dev/null +++ b/packages/@jsii/benchmarks/scripts/snapshot-package.ts @@ -0,0 +1,95 @@ +import * as cp from 'child_process'; +import * as fs from 'fs-extra'; +import * as glob from 'glob'; +import * as os from 'os'; +import * as path from 'path'; +import * as tar from 'tar'; + +import { cdkTagv2_21_1, cdkv2_21_1 } from '../lib/constants'; + +function snapshotAwsCdk(tag: string, file: string) { + // Directory of aws-cdk repository + const repoDir = fs.mkdtempSync( + path.join(os.tmpdir(), `jsii-cdk-bench@${tag}`), + ); + // Directory for snapshot of aws-cdk-lib source + const intermediate = fs.mkdtempSync( + path.join(os.tmpdir(), `jsii-cdk-bench-inter@${tag}`), + ); + + // Clone aws/aws-cdk + cp.execSync( + `git clone --depth=1 -b ${tag} https://github.com/aws/aws-cdk.git .`, + { + cwd: repoDir, + }, + ); + + // Install/link dependencies + cp.execSync('yarn install --frozen-lockfile', { cwd: repoDir }); + + // build aws-cdk-lib and dependencies + cp.execSync( + `npx lerna run --scope aws-cdk-lib --include-dependencies build`, + { cwd: repoDir }, + ); + + // Copy built package to intermediate directory + fs.copySync(path.resolve(repoDir, 'packages', 'aws-cdk-lib'), intermediate); + + // Remove build artifacts so we can rebuild + const artifacts = glob.sync( + path.join(intermediate, '**/*@(.js|.js.map|.d.ts|.tsbuildinfo)'), + ); + artifacts.forEach(fs.removeSync); + + // Remove node_modules from monorepo setup + fs.removeSync(path.resolve(intermediate, 'node_modules')); + + // Remove @aws-cdk/* deps from package.json so we can npm install to get hoisted dependencies + // into local node_modules + const packageJsonPath = path.resolve(intermediate, 'package.json'); + const { devDependencies, ...pkgJson } = fs.readJsonSync(packageJsonPath); + const newDevDependencies = Object.entries(devDependencies).reduce( + (accum, [pkg, version]) => { + if (pkg.startsWith('@aws-cdk/')) return accum; + + return { + ...accum, + [pkg]: version, + }; + }, + {}, + ); + + fs.writeFileSync( + packageJsonPath, + JSON.stringify( + { ...pkgJson, devDependencies: newDevDependencies }, + undefined, + 2, + ), + ); + + // Run npm install to get package-lock.json for reproducible dependency tree + cp.execSync(`npm install`, { cwd: intermediate }); + fs.removeSync(path.resolve(intermediate, 'node_modules')); + tar.c( + { + file, + cwd: intermediate, + sync: true, + gzip: true, + }, + ['.'], + ); + + fs.removeSync(intermediate); + fs.removeSync(repoDir); +} + +function main() { + snapshotAwsCdk(cdkTagv2_21_1, cdkv2_21_1); +} + +main(); diff --git a/packages/@jsii/benchmarks/tsconfig.json b/packages/@jsii/benchmarks/tsconfig.json new file mode 100644 index 0000000000..b190a5dff6 --- /dev/null +++ b/packages/@jsii/benchmarks/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../../tsconfig-base", + "include": ["**/*.ts"], + "exclude": ["fixtures"] +} diff --git a/yarn.lock b/yarn.lock index 4f8c9bbb8a..55affa47be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1546,6 +1546,14 @@ dependencies: "@types/node" "*" +"@types/glob@^7.2.0": + version "7.2.0" + resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" + integrity sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA== + dependencies: + "@types/minimatch" "*" + "@types/node" "*" + "@types/graceful-fs@^4.1.2": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" @@ -1598,7 +1606,7 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= -"@types/minimatch@^3.0.3": +"@types/minimatch@*", "@types/minimatch@^3.0.3": version "3.0.5" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== @@ -2325,6 +2333,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -3958,6 +3973,18 @@ glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.1.7, gl once "^1.3.0" path-is-absolute "^1.0.0" +glob@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.1.tgz#00308f5c035aa0b2a447cd37ead267ddff1577d3" + integrity sha512-cF7FYZZ47YzmCu7dDy50xSRRfO3ErRfrXuLZcNIuyiJEco0XSrGtuilG19L5xp3NcwTx7Gn+X6Tv3fmsUPTbow== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + globals@^11.1.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" @@ -5467,6 +5494,13 @@ minimatch@^3.0.4, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b" + integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g== + dependencies: + brace-expansion "^2.0.1" + minimist-options@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"