From 432a71f842d7db4a22676f8256d640817319e136 Mon Sep 17 00:00:00 2001 From: temariq Date: Wed, 19 Apr 2023 21:35:55 +0300 Subject: [PATCH 1/5] feat(main-suite): add npm suite --- spec/main-suite/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/main-suite/package.json b/spec/main-suite/package.json index 6c47d5e..2c5c588 100644 --- a/spec/main-suite/package.json +++ b/spec/main-suite/package.json @@ -22,6 +22,7 @@ "integration:prepare": "./start.sh", "integration:check": "NODE_OPTIONS=--experimental-vm-modules jest test", "integration:all": "pnpm run integration:unchanged && pnpm run integration:degradation", + "integration:npm": "perftool -o test-result/old.json --logLevel=verbose && SLOW=1 perftool -o test-result/new.json --logLevel=verbose && npm run integration:compare", "integration:degradation": "perftool -o test-result/old.json --logLevel=verbose && SLOW=1 perftool -o test-result/new.json --logLevel=verbose && pnpm run integration:compare -o test-result/degradation.json", "integration:unchanged": "perftool -o test-result/old.json --logLevel=verbose && perftool -o test-result/new.json --logLevel=verbose && pnpm run integration:compare -o test-result/unchanged.json", "integration:compare": "perftool-compare test-result/new.json test-result/old.json --logLevel=verbose" From 69d8839ee96acf76922907d7403b25f9164aa8e1 Mon Sep 17 00:00:00 2001 From: temariq Date: Fri, 21 Apr 2023 14:54:21 +0300 Subject: [PATCH 2/5] ci: separate integration tests --- .github/workflows/checks.yml | 29 --------------------- .github/workflows/integration.yml | 43 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b13c9c2..d0fbab4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -53,32 +53,3 @@ jobs: - name: Type Check run: npx lerna run typecheck - - integration: - strategy: - matrix: - node: [ 16, 18 ] - react: [ 16, 17, 18 ] - name: Integration testing (Node ${{ matrix.node }}, React ${{ matrix.react }}) - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }} - steps: - - uses: actions/checkout@v3 - - uses: pnpm/action-setup@v2 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - cache: 'pnpm' - - name: Setup packages - run: pnpm install - - - name: Run tests - env: - REACT_VERSION: ${{ matrix.react }} - run: | - cd spec/main-suite - npm run integration:prepare - npm run integration:all - npm run integration:check diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..1fc043e --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,43 @@ +name: Integration testing + +on: + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + basic: + strategy: + matrix: + node: [ 16, 18 ] + react: [ 16, 17, 18 ] + name: Basic (Node ${{ matrix.node }}, React ${{ matrix.react }}) + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + cache: 'pnpm' + - name: Setup packages + run: pnpm install + + - name: Run tests + env: + REACT_VERSION: ${{ matrix.react }} + run: | + cd spec/main-suite + npm run integration:prepare + npm run integration:all + npm run integration:check From 21b36f258c18ea1e28dee73b5470a3cf3a778a66 Mon Sep 17 00:00:00 2001 From: temariq Date: Fri, 21 Apr 2023 14:56:15 +0300 Subject: [PATCH 3/5] test: add compatibility test for npm, add bad ts-node in project test --- .github/workflows/integration.yml | 63 +++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1fc043e..239b5db 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -41,3 +41,66 @@ jobs: npm run integration:prepare npm run integration:all npm run integration:check + + npm: + name: Project using NPM + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: Setup packages + run: pnpm install + + - name: Prepare + run: | + cd spec/main-suite + npm run integration:prepare + + - name: Install via NPM + run: | + cd spec/main-suite + rm -r node_modules + npm install + + - name: Run perftool + run: | + cd spec/main-suite + npm run integration:npm + + bad_ts-node: + name: Project with different TS-node version + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: Setup packages + run: pnpm install + + - name: Prepare + run: | + cd spec/main-suite + pnpm run integration:prepare + + - name: Install bad ts-node version + run: | + cd spec/main-suite + pnpm add ts-node@9.1.1 + + - name: Run perftool + run: | + cd spec/main-suite + pnpm run integration:unchanged From 2c94f4594dff1999ada51e7b94aa679d5a46ddc7 Mon Sep 17 00:00:00 2001 From: temariq Date: Fri, 21 Apr 2023 15:42:04 +0300 Subject: [PATCH 4/5] build: update pnpm version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2b9c892..1cd42f1 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "conventional-commits" ] }, - "packageManager": "pnpm@8.2.0", + "packageManager": "pnpm@8.3.1", "engines": { "node": ">=18", "pnpm": ">=8" From 196230a58d3c2cb24124719067c66da7c2f48bce Mon Sep 17 00:00:00 2001 From: temariq Date: Sat, 22 Apr 2023 12:47:46 +0300 Subject: [PATCH 5/5] fix(perftool): fix react 16,17 compatibility --- packages/perftool/scripts/start-compare.sh | 3 +- packages/perftool/scripts/start.sh | 3 +- packages/perftool/src/client/index.ts | 10 ++++- .../perftool/src/client/measurement/runner.ts | 40 ++++++++++++++----- packages/perftool/src/clientEntry.ts | 6 ++- packages/perftool/src/config/webpack.ts | 9 ++++- .../perftool/src/utils/__spec__/react.test.ts | 18 +++++---- packages/perftool/src/utils/react.ts | 13 ++++-- 8 files changed, 74 insertions(+), 28 deletions(-) diff --git a/packages/perftool/scripts/start-compare.sh b/packages/perftool/scripts/start-compare.sh index 3f26557..b588eb2 100755 --- a/packages/perftool/scripts/start-compare.sh +++ b/packages/perftool/scripts/start-compare.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -set -ex +set -e case "$(uname -s)" in Darwin*) @@ -21,7 +21,6 @@ OPTS="--loader $TS_NODE_ESM_PATH --experimental-specifier-resolution=node" if [ -z "$PERFTOOL_DEBUG" ]; then OPTS="$OPTS --no-warnings" - set -x fi NODE_OPTIONS=$OPTS TS_NODE_PROJECT="$PROJECT_DIR/tsconfig.json" TS_NODE_FILES=true node "$PROJECT_DIR/src/compare/index.ts" "$@" diff --git a/packages/perftool/scripts/start.sh b/packages/perftool/scripts/start.sh index 343236e..3667a76 100755 --- a/packages/perftool/scripts/start.sh +++ b/packages/perftool/scripts/start.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -set -ex +set -e case "$(uname -s)" in Darwin*) @@ -21,7 +21,6 @@ OPTS="--loader $TS_NODE_ESM_PATH --experimental-specifier-resolution=node" if [ -z "$PERFTOOL_DEBUG" ]; then OPTS="$OPTS --no-warnings" - set -x fi NODE_OPTIONS=$OPTS TS_NODE_PROJECT="$PROJECT_DIR/tsconfig.json" TS_NODE_FILES=true node "$PROJECT_DIR/src/index.ts" "$@" diff --git a/packages/perftool/src/client/index.ts b/packages/perftool/src/client/index.ts index 1f35226..b8f022a 100644 --- a/packages/perftool/src/client/index.ts +++ b/packages/perftool/src/client/index.ts @@ -1,4 +1,5 @@ import type { Config } from '../config/common'; +import { debug } from '../utils/logger'; import type { Task } from './measurement/types'; import { runTask, Subject } from './measurement/runner'; @@ -10,22 +11,29 @@ type CreatePerfToolClientParams[]> = { config: Config; }; -// TODO debug logging export async function createPerfToolClient[]>({ subjects, tasks, config, }: CreatePerfToolClientParams) { + debug('Perftool client created'); + debug('Available subjects: ', subjects); + debug('Available tasks: ', tasks); + debug('Config: ', config); + const tests = await resolveTests({ tasks, subjects }); const resultPromises = []; + debug(`Running ${tests.length} tests`); for (const { task, subject } of tests) { const resultPromise = runTask({ task, subject, config }); resultPromises.push(resultPromise); } + debug('Waiting for all tests to complete...'); const results = await Promise.all(resultPromises); + debug('All tests complete, calling window.finish'); await window.finish(results); } diff --git a/packages/perftool/src/client/measurement/runner.ts b/packages/perftool/src/client/measurement/runner.ts index f37ec46..f1b9476 100644 --- a/packages/perftool/src/client/measurement/runner.ts +++ b/packages/perftool/src/client/measurement/runner.ts @@ -4,6 +4,7 @@ import type { Config } from '../../config/common'; import { getTaskConfig } from '../../config/task'; import BaseError from '../../utils/baseError'; import { defer } from '../../utils/deferred'; +import { debug } from '../../utils/logger'; import { Task } from './types'; @@ -43,18 +44,39 @@ export function runTask>({ const meta = { taskId: task.id, subjectId: subject.id }; const config = getTaskConfig(task, globalConfig); const container = createContainer(); + let isComplete = false; + + debug('Running test\n', `TaskId: ${meta.taskId}\n`, `SubjectId: ${meta.subjectId}`); + debug('Task config: ', config); return Promise.race([ task .run({ Subject: subject.Component, config, container }) - .then((result) => ({ - ...meta, - result, - })) - .catch((error: Error) => ({ - ...meta, - error: error.toString(), - })), - defer(globalConfig.taskWaitTimeout, () => ({ ...meta, error: new TimeoutError().toString() })), + .then((result) => { + debug(`Test ${meta.taskId}, ${meta.subjectId} complete. Result:`, result); + + return { + ...meta, + result, + }; + }) + .catch((error: Error) => { + debug(`Test ${meta.taskId}, ${meta.subjectId} failed. Error:`, error); + + return { + ...meta, + error: error.toString(), + }; + }) + .finally(() => { + isComplete = true; + }), + defer(globalConfig.taskWaitTimeout, () => { + if (!isComplete) { + debug(`Test ${meta.taskId}, ${meta.subjectId} timed out`); + } + + return { ...meta, error: new TimeoutError().toString() }; + }), ]); } diff --git a/packages/perftool/src/clientEntry.ts b/packages/perftool/src/clientEntry.ts index cedff7d..89518f9 100644 --- a/packages/perftool/src/clientEntry.ts +++ b/packages/perftool/src/clientEntry.ts @@ -5,6 +5,7 @@ import { getAllTasks } from './config/task'; import { createPerfToolClient } from './client'; import { Subject } from './client/measurement/runner'; import { subject as staticTaskSubject } from './stabilizers/staticTask'; +import { setLogLevel } from './utils/logger'; // @@ -12,14 +13,17 @@ const config = ((v) => v)( // ) as unknown as Config; +setLogLevel(config.logLevel); + const allTestSubjects: Subject[] = [ staticTaskSubject, // ]; +// TODO tasks in client config are serialized, const allTasks = getAllTasks(config); -createPerfToolClient({ +await createPerfToolClient({ config, tasks: allTasks, subjects: allTestSubjects, diff --git a/packages/perftool/src/config/webpack.ts b/packages/perftool/src/config/webpack.ts index 4b66f61..f97a210 100644 --- a/packages/perftool/src/config/webpack.ts +++ b/packages/perftool/src/config/webpack.ts @@ -1,19 +1,26 @@ import HtmlWebpackPlugin from 'html-webpack-plugin'; import webpack, { Configuration as WebpackConfig } from 'webpack'; import { createRequire } from 'node:module'; +import path from 'path'; +import { fileURLToPath } from 'url'; import { debug } from '../utils/logger'; import { Config } from './common'; const require = createRequire(import.meta.url); +const dirname = path.dirname(fileURLToPath(import.meta.url)); +const ownNodeModules = path.relative(process.cwd(), path.resolve(dirname, '../../node_modules')); const defaultConfig: WebpackConfig = { mode: 'production', externals: ['fsevents'], resolve: { extensions: ['.ts', '.tsx', '.js', '.jsx', '.mjs'], - modules: ['node_modules', 'node_modules/@salutejs/perftool/node_modules'], + modules: ['node_modules', ownNodeModules], + fallback: { + 'react-dom/client': false, + }, }, experiments: { topLevelAwait: true, diff --git a/packages/perftool/src/utils/__spec__/react.test.ts b/packages/perftool/src/utils/__spec__/react.test.ts index 9277fda..d5c924b 100644 --- a/packages/perftool/src/utils/__spec__/react.test.ts +++ b/packages/perftool/src/utils/__spec__/react.test.ts @@ -1,8 +1,6 @@ import { jest } from '@jest/globals'; import type React from 'react'; -const originalRequire = global.require; - describe('utils/react/isReact18AndNewer', () => { afterEach(() => { jest.restoreAllMocks(); @@ -25,11 +23,14 @@ describe('utils/react/isReact18AndNewer', () => { }); describe('utils/render', () => { + beforeEach(() => { + process.env.PERFTOOL_CLIENT_RUNTIME = '1'; + }); + afterEach(() => { jest.restoreAllMocks(); jest.resetModules(); process.env.PERFTOOL_CLIENT_RUNTIME = undefined; - global.require = originalRequire; }); it('should call ReactDOM.render if react version is under 18', async () => { @@ -38,7 +39,7 @@ describe('utils/render', () => { jest.unstable_mockModule('react', () => ({ version: '17.3.0' })); jest.unstable_mockModule('react-dom', () => ({ - default: { render: (renderMock = jest.fn((_, __, resolve: any) => (res = resolve)())) }, + render: (renderMock = jest.fn((_, __, resolve: any) => (res = resolve)())), })); const { render } = await import('../react'); @@ -58,7 +59,6 @@ describe('utils/render', () => { jest.unstable_mockModule('react-dom/client', () => ({ createRoot: (createRootMock = jest.fn(() => ({ render: (rootRenderMock = jest.fn()) }))), })); - process.env.PERFTOOL_CLIENT_RUNTIME = '1'; const { render } = await import('../react'); const fakeElement = {} as React.ReactElement; @@ -74,11 +74,14 @@ describe('utils/render', () => { }); describe('utils/hydrate', () => { + beforeEach(() => { + process.env.PERFTOOL_CLIENT_RUNTIME = '1'; + }); + afterEach(() => { jest.restoreAllMocks(); jest.resetModules(); process.env.PERFTOOL_CLIENT_RUNTIME = undefined; - global.require = originalRequire; }); it('should call ReactDOM.hydrate if react version is under 18', async () => { @@ -86,7 +89,7 @@ describe('utils/hydrate', () => { let res = () => undefined; jest.unstable_mockModule('react', () => ({ version: '17.3.0' })); jest.unstable_mockModule('react-dom', () => ({ - default: { hydrate: (hydrateMock = jest.fn((_, __, resolve: any) => (res = resolve)())) }, + hydrate: (hydrateMock = jest.fn((_, __, resolve: any) => (res = resolve)())), })); const { hydrate } = await import('../react'); @@ -105,7 +108,6 @@ describe('utils/hydrate', () => { jest.unstable_mockModule('react-dom/client', () => ({ hydrateRoot: (hydrateRootMock = jest.fn()), })); - process.env.PERFTOOL_CLIENT_RUNTIME = '1'; const { hydrate } = await import('../react'); const fakeElement = {} as React.ReactElement; diff --git a/packages/perftool/src/utils/react.ts b/packages/perftool/src/utils/react.ts index 496a1be..1fe300e 100644 --- a/packages/perftool/src/utils/react.ts +++ b/packages/perftool/src/utils/react.ts @@ -1,15 +1,20 @@ import { version } from 'react'; -import ReactDOM from 'react-dom'; +import type * as ReactDOMType from 'react-dom'; import type * as ReactDOMClientType from 'react-dom/client'; export function isReact18AndNewer() { return Number(version.split('.')[0]) >= 18; } -// Import (React 18 specific) react-dom/client conditionally to avoid build-time errors when using older versions +// Import version specific react-dom modules conditionally to avoid build-time errors let ReactDOMClient: typeof ReactDOMClientType = {} as any; -if (isReact18AndNewer() && process.env.PERFTOOL_CLIENT_RUNTIME) { - ReactDOMClient = await import('react-dom/client'); +let ReactDOM: typeof ReactDOMType = {} as any; +if (process.env.PERFTOOL_CLIENT_RUNTIME) { + if (isReact18AndNewer()) { + ReactDOMClient = await import(`react-dom/client`); + } else { + ReactDOM = await import(`react-dom`); + } } export async function render(element: React.ReactElement, container: HTMLElement): Promise {