From 2bbb308c70551be5d0d37c8c3118cb38a015adec Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Thu, 26 Sep 2024 13:14:35 +0200 Subject: [PATCH 1/6] Turbopack build: Add devlow-bench --- bench/heavy-npm-deps/package.json | 4 + scripts/devlow-bench.mjs | 216 +++++++++++++++++++++++++++++- 2 files changed, 219 insertions(+), 1 deletion(-) diff --git a/bench/heavy-npm-deps/package.json b/bench/heavy-npm-deps/package.json index cf0a1e154eda4..c61a01510dcd6 100644 --- a/bench/heavy-npm-deps/package.json +++ b/bench/heavy-npm-deps/package.json @@ -5,6 +5,10 @@ "scripts": { "dev-turbopack": "next dev --turbo", "dev-webpack": "next dev", + "build-turbopack": "TURBOPACK=1 TURBOPACK_BUILD=1 next build", + "build-webpack": "next build", + "start-turbopack": "TURBOPACK=1 next start", + "start-webpack": "next start", "build-application": "next build", "start-application": "next start" }, diff --git a/scripts/devlow-bench.mjs b/scripts/devlow-bench.mjs index be149c0c71613..378b8b7619da1 100644 --- a/scripts/devlow-bench.mjs +++ b/scripts/devlow-bench.mjs @@ -25,6 +25,212 @@ const GIT_BRANCH = return cmd.output })()) +const nextBuildWorkflow = + (benchmarkName, pages) => + async ({ turbopack, page }) => { + const pageConfig = + typeof pages[page] === 'string' ? { url: pages[page] } : pages[page] + const cleanupTasks = [] + try { + const env = { + PATH: process.env.PATH, + NODE: process.env.NODE, + HOSTNAME: process.env.HOSTNAME, + PWD: process.env.PWD, + // Disable otel initialization to prevent pending / hanging request to otel collector + OTEL_SDK_DISABLED: 'true', + NEXT_PUBLIC_OTEL_SENTRY: 'true', + NEXT_PUBLIC_OTEL_DEV_DISABLED: 'true', + NEXT_TRACE_UPLOAD_DISABLED: 'true', + // Enable next.js test mode to get HMR events + __NEXT_TEST_MODE: '1', + } + + const benchmarkDir = resolve(REPO_ROOT, 'bench', benchmarkName) + + // cleanup .next directory to remove persistent cache + await retry(() => + rm(join(benchmarkDir, '.next'), { recursive: true, force: true }) + ) + + await measureTime('cleanup', { + scenario: benchmarkName, + props: { turbopack: null, page: null }, + }) + + const buildArgs = [turbopack ? 'build-turbopack' : 'build-webpack'] + let buildShell = command('pnpm', buildArgs, { + cwd: benchmarkDir, + env, + }) + await buildShell.ok() + + await measureTime('build', { + scenario: benchmarkName, + props: { turbopack: null, page: null }, + }) + + // startup browser + let session = await newBrowserSession({}) + const closeSession = async () => { + if (session) { + await session.close() + session = null + } + } + cleanupTasks.push(closeSession) + await measureTime('browser startup', { + props: { turbopack: null, page: null }, + }) + + // run command to start dev server + const startArgs = [turbopack ? 'start-turbopack' : 'start-webpack'] + let shell = command('pnpm', startArgs, { + cwd: benchmarkDir, + env, + }) + const killShell = async () => { + if (shell) { + await shell.kill() + shell = null + } + } + cleanupTasks.push(killShell) + + // wait for server to be ready + const START_SERVER_REGEXP = /Local:\s+(?.+)\n/ + const { + groups: { url }, + } = await shell.waitForOutput(START_SERVER_REGEXP) + await measureTime('server startup', { props: { page: null } }) + await shell.reportMemUsage('mem usage after startup', { + props: { page: null }, + }) + + // open page + const pageInstance = await session.hardNavigation( + 'open page', + url + pageConfig.url + ) + await shell.reportMemUsage('mem usage after open page') + + let status = 0 + try { + if ( + await pageInstance.evaluate( + '!next.appDir && __NEXT_DATA__.page === "/404"' + ) + ) { + status = 2 + } + } catch (e) { + status = 2 + } + + try { + if ( + !(await pageInstance.evaluate( + 'next.appDir || __NEXT_DATA__.page && !__NEXT_DATA__.err' + )) + ) { + status = 1 + } + } catch (e) { + status = 1 + } + + await reportMeasurement('page status', status, 'status code') + + // reload page + await session.reload('reload page') + + await reportMeasurement( + 'console output', + shell.output.split(/\n/).length, + 'lines' + ) + + if (turbopack) { + // close dev server and browser + await killShell() + await closeSession() + } else { + // wait for persistent cache to be written + const waitPromise = new Promise((resolve) => { + setTimeout(resolve, 5000) + }) + const cacheLocation = join( + benchmarkDir, + '.next', + 'cache', + 'webpack', + 'client-development' + ) + await Promise.race([ + waitForFile(join(cacheLocation, 'index.pack')), + waitForFile(join(cacheLocation, 'index.pack.gz')), + ]) + await measureTime('cache created') + await waitPromise + await measureTime('waiting') + + // close dev server and browser + await killShell() + await closeSession() + } + + buildShell = command('pnpm', buildArgs, { + cwd: benchmarkDir, + env, + }) + await buildShell.ok() + + await measureTime('build with cache', { + scenario: benchmarkName, + props: { turbopack: null, page: null }, + }) + + // startup new browser + session = await newBrowserSession({}) + await measureTime('browser startup', { + props: { turbopack: null, page: null }, + }) + + // run command to start dev server + shell = command('pnpm', startArgs, { + cwd: benchmarkDir, + env, + }) + + // wait for server to be ready + const { + groups: { url: url2 }, + } = await shell.waitForOutput(START_SERVER_REGEXP) + await shell.reportMemUsage('mem usage after startup with cache') + + // open page + await session.hardNavigation( + 'open page with cache', + url2 + pageConfig.url + ) + + await reportMeasurement( + 'console output with cache', + shell.output.split(/\n/).length, + 'lines' + ) + await shell.reportMemUsage('mem usage after open page with cache') + } catch (e) { + console.log('CAUGHT', e) + throw e + } finally { + // This must run in order + // eslint-disable-next-line no-await-in-loop + for (const task of cleanupTasks.reverse()) await task() + await measureTime('shutdown') + } + } + const nextDevWorkflow = (benchmarkName, pages) => async ({ turbopack, page }) => { @@ -62,7 +268,6 @@ const nextDevWorkflow = NODE: process.env.NODE, HOSTNAME: process.env.HOSTNAME, PWD: process.env.PWD, - NODE_ENV: 'development', // Disable otel initialization to prevent pending / hanging request to otel collector OTEL_SDK_DISABLED: 'true', NEXT_PUBLIC_OTEL_SENTRY: 'true', @@ -361,6 +566,15 @@ describe( nextDevWorkflow('heavy-npm-deps', pages) ) +describe( + 'heavy-npm-deps build test', + { + turbopack: true, + page: Object.keys(pages), + }, + nextBuildWorkflow('heavy-npm-deps', pages) +) + async function retry(fn) { let lastError for (let i = 100; i < 2000; i += 100) { From 2771005f391f5348acf521f9f00630f8f042990b Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Thu, 26 Sep 2024 13:21:04 +0200 Subject: [PATCH 2/6] Add scenario to CI --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 46c28490a56db..88aa144f1f144 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -187,6 +187,7 @@ jobs: - '--turbopack=true' selector: - '--scenario=heavy-npm-deps --page=homepage' + - '--scenario="heavy-npm-deps build" --page=homepage' uses: ./.github/workflows/build_reusable.yml with: afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }} From 7cb7b28e643095a16bfbdad82f7e5c837746ac9d Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Thu, 26 Sep 2024 13:39:13 +0200 Subject: [PATCH 3/6] Pass through props --- scripts/devlow-bench.mjs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/devlow-bench.mjs b/scripts/devlow-bench.mjs index 378b8b7619da1..ac6194cdbc515 100644 --- a/scripts/devlow-bench.mjs +++ b/scripts/devlow-bench.mjs @@ -55,7 +55,7 @@ const nextBuildWorkflow = await measureTime('cleanup', { scenario: benchmarkName, - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) const buildArgs = [turbopack ? 'build-turbopack' : 'build-webpack'] @@ -67,7 +67,7 @@ const nextBuildWorkflow = await measureTime('build', { scenario: benchmarkName, - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) // startup browser @@ -80,7 +80,7 @@ const nextBuildWorkflow = } cleanupTasks.push(closeSession) await measureTime('browser startup', { - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) // run command to start dev server @@ -102,9 +102,9 @@ const nextBuildWorkflow = const { groups: { url }, } = await shell.waitForOutput(START_SERVER_REGEXP) - await measureTime('server startup', { props: { page: null } }) + await measureTime('server startup', { props: { turbopack, page } }) await shell.reportMemUsage('mem usage after startup', { - props: { page: null }, + props: { turbopack, page }, }) // open page @@ -187,13 +187,13 @@ const nextBuildWorkflow = await measureTime('build with cache', { scenario: benchmarkName, - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) // startup new browser session = await newBrowserSession({}) await measureTime('browser startup', { - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) // run command to start dev server @@ -247,7 +247,7 @@ const nextDevWorkflow = await measureTime('cleanup', { scenario: benchmarkName, - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) // startup browser @@ -260,7 +260,7 @@ const nextDevWorkflow = } cleanupTasks.push(closeSession) await measureTime('browser startup', { - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) const env = { @@ -296,9 +296,9 @@ const nextDevWorkflow = const { groups: { url }, } = await shell.waitForOutput(START_SERVER_REGEXP) - await measureTime('server startup', { props: { page: null } }) + await measureTime('server startup', { props: { turbopack, page } }) await shell.reportMemUsage('mem usage after startup', { - props: { page: null }, + props: { turbopack, page }, }) // open page @@ -509,7 +509,7 @@ const nextDevWorkflow = // startup new browser session = await newBrowserSession({}) await measureTime('browser startup', { - props: { turbopack: null, page: null }, + props: { turbopack, page }, }) // run command to start dev server From e0c8864190c3de0ee9bdc7443f97932a52f37ad1 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Thu, 26 Sep 2024 13:46:06 +0200 Subject: [PATCH 4/6] Add property to filter dev vs build --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 88aa144f1f144..525763981e87f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -186,8 +186,8 @@ jobs: - '--turbopack=false' - '--turbopack=true' selector: - - '--scenario=heavy-npm-deps --page=homepage' - - '--scenario="heavy-npm-deps build" --page=homepage' + - '--scenario=heavy-npm-deps --page=homepage --mode=dev' + - '--scenario="heavy-npm-deps build" --page=homepage --mode=build' uses: ./.github/workflows/build_reusable.yml with: afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }} From 44f83f9c80c7a28d74ae0a93ec7497185789bf61 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Thu, 26 Sep 2024 15:55:06 +0200 Subject: [PATCH 5/6] Fix artifact name --- .github/workflows/build_and_test.yml | 2 +- scripts/devlow-bench.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 525763981e87f..64c2f176703ad 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -187,7 +187,7 @@ jobs: - '--turbopack=true' selector: - '--scenario=heavy-npm-deps --page=homepage --mode=dev' - - '--scenario="heavy-npm-deps build" --page=homepage --mode=build' + - '--scenario=heavy-npm-deps-build --page=homepage --mode=build' uses: ./.github/workflows/build_reusable.yml with: afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }} diff --git a/scripts/devlow-bench.mjs b/scripts/devlow-bench.mjs index ac6194cdbc515..f7d31c5da45af 100644 --- a/scripts/devlow-bench.mjs +++ b/scripts/devlow-bench.mjs @@ -567,7 +567,7 @@ describe( ) describe( - 'heavy-npm-deps build test', + 'heavy-npm-deps-build test', { turbopack: true, page: Object.keys(pages), From 3a91245ae2046a9f04dbc009bab2937ad7086a6a Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Thu, 26 Sep 2024 16:50:35 +0200 Subject: [PATCH 6/6] Fix test run --- .github/workflows/build_and_test.yml | 4 ++-- scripts/devlow-bench.mjs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 64c2f176703ad..1abf5e7cd50e2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -186,8 +186,8 @@ jobs: - '--turbopack=false' - '--turbopack=true' selector: - - '--scenario=heavy-npm-deps --page=homepage --mode=dev' - - '--scenario=heavy-npm-deps-build --page=homepage --mode=build' + - '--scenario=heavy-npm-deps-dev --page=homepage' + - '--scenario=heavy-npm-deps-build --page=homepage' uses: ./.github/workflows/build_reusable.yml with: afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }} diff --git a/scripts/devlow-bench.mjs b/scripts/devlow-bench.mjs index f7d31c5da45af..221225c756cef 100644 --- a/scripts/devlow-bench.mjs +++ b/scripts/devlow-bench.mjs @@ -558,18 +558,20 @@ const pages = { } describe( - 'heavy-npm-deps dev test', + 'heavy-npm-deps-dev', { turbopack: true, + mode: 'dev', page: Object.keys(pages), }, nextDevWorkflow('heavy-npm-deps', pages) ) describe( - 'heavy-npm-deps-build test', + 'heavy-npm-deps-build', { turbopack: true, + mode: 'build', page: Object.keys(pages), }, nextBuildWorkflow('heavy-npm-deps', pages)