Build: Add benchmark task

.circleci/config.yml

@@ -151,88 +151,6 @@ jobs:
             - code/renderers
             - code/presets
             - .verdaccio-cache
-  cra-bench:
-    executor:
-      class: large
-      name: sb_playwright
-    working_directory: /tmp/storybook
-    steps:
-      - git-shallow-clone/checkout_advanced:
-          clone_options: '--depth 1 --verbose'
-      - attach_workspace:
-          at: .
-      - run:
-          name: Running local registry
-          command: |
-            cd code
-            yarn local-registry --open
-          background: true
-      - run:
-          name: Wait for registry
-          command: |
-            cd code
-            yarn wait-on http://localhost:6001
-      - run:
-          name: set up cra repro, skip tests
-          command: |
-            cd code
-            SANDBOX_ROOT=../bench yarn task --task sandbox --template cra/default-ts --skip-template-stories --start-from=never  --no-link
-      - run:
-          name: Run @storybook/bench on repro
-          command: |
-            cd bench/cra-default-ts
-            rm -rf node_modules
-            mkdir node_modules
-            ../../code/node_modules/.bin/sb-bench 'yarn install' --label cra
-      - report-workflow-on-failure
-      - run:
-          name: prep artifacts
-          when: always
-          command: tar cvzf /tmp/cra-default-ts-bench.tar.gz bench/cra-default-ts
-      - store_artifacts:
-          path: /tmp/cra-default-ts-bench.tar.gz
-          destination: cra-default-ts-bench.tar.gz
-  react-vite-bench:
-    executor:
-      class: large
-      name: sb_playwright
-    working_directory: /tmp/storybook
-    steps:
-      - git-shallow-clone/checkout_advanced:
-          clone_options: '--depth 1 --verbose'
-      - attach_workspace:
-          at: .
-      - run:
-          name: Running local registry
-          command: |
-            cd code
-            yarn local-registry --open
-          background: true
-      - run:
-          name: Wait for registry
-          command: |
-            cd code
-            yarn wait-on http://localhost:6001
-      - run:
-          name: set up react-vite repro, skip tests
-          command: |
-            cd code
-            SANDBOX_ROOT=../bench yarn task --task sandbox --template react-vite/default-ts --skip-template-stories --start-from=never  --no-link
-      - run:
-          name: Run @storybook/bench on repro
-          command: |
-            cd bench/react-vite-default-ts
-            rm -rf node_modules
-            mkdir node_modules
-            ../../code/node_modules/.bin/sb-bench 'yarn install' --label react-vite
-      - report-workflow-on-failure
-      - run:
-          name: prep artifacts
-          when: always
-          command: tar cvzf /tmp/react-vite-default-ts-bench.tar.gz bench/react-vite-default-ts
-      - store_artifacts:
-          path: /tmp/react-vite-default-ts-bench.tar.gz
-          destination: react-vite-default-ts-bench.tar.gz
   lint:
     executor:
       class: large
@@ -525,6 +443,36 @@ jobs:
       - store_artifacts: # this is where playwright puts more complex stuff
           path: code/playwright-results/
           destination: playwright
+  bench:
+    parameters:
+      parallelism:
+        type: integer
+        default: 2
+    executor:
+      class: medium
+      name: sb_playwright
+    parallelism: << parameters.parallelism >>
+    steps:
+      - git-shallow-clone/checkout_advanced:
+          clone_options: '--depth 1 --verbose'
+      - attach_workspace:
+          at: .
+      - run:
+          # NOTE: this assumes that the bench-dev key represents BOTH tasks since they are run at the same time.
+          # Update if we ever run these separately!
+          name: Running Dev Bench
+          command: yarn task --task bench-dev --template $(yarn get-template << pipeline.parameters.workflow >> bench-dev) --no-link --start-from=never --junit
+      - run:
+          name: Running Prod Bench
+          command: yarn task --task bench-build --template $(yarn get-template << pipeline.parameters.workflow >> bench-dev) --no-link --start-from=never --junit
+      - run:
+          name: Uploading results
+          command: yarn upload-bench $(yarn get-template << pipeline.parameters.workflow >> bench-dev)
+      - report-workflow-on-failure:
+          template: $(yarn get-template << pipeline.parameters.workflow >> bench-dev)
+      - store_artifacts: # this is where playwright puts more complex stuff
+          path: code/bench-results/
+          destination: bench-results
 
 workflows:
   ci:
@@ -596,18 +544,12 @@ workflows:
       - coverage:
           requires:
             - unit-tests
-      - cra-bench:
-          requires:
-            - build
-      - react-vite-bench:
-          requires:
-            - build
       - create-sandboxes:
-          parallelism: 9
+          parallelism: 11
           requires:
             - build
       - build-sandboxes:
-          parallelism: 9
+          parallelism: 11
           requires:
             - create-sandboxes
       - chromatic-sandboxes:
@@ -625,6 +567,10 @@ workflows:
           parallelism: 9
           requires:
             - build-sandboxes
+      - bench:
+          parallelism: 2
+          requires:
+            - build-sandboxes
       # TODO: reenable once we find out the source of flakyness
       # - test-runner-dev:
       #     requires:
@@ -653,34 +599,32 @@ workflows:
       - coverage:
           requires:
             - unit-tests
-      - cra-bench:
-          requires:
-            - build
-      - react-vite-bench:
-          requires:
-            - build
       - create-sandboxes:
-          parallelism: 18
+          parallelism: 20
           requires:
             - build
       - build-sandboxes:
-          parallelism: 18
+          parallelism: 20
           requires:
             - create-sandboxes
       - chromatic-sandboxes:
-          parallelism: 18
+          parallelism: 20
           requires:
             - build-sandboxes
       - e2e-production:
-          parallelism: 18
+          parallelism: 20
           requires:
             - build-sandboxes
       - e2e-dev:
           parallelism: 4
           requires:
             - create-sandboxes
       - test-runner-production:
-          parallelism: 18
+          parallelism: 20
+          requires:
+            - build-sandboxes
+      - bench:
+          parallelism: 2
           requires:
             - build-sandboxes
       # TODO: reenable once we find out the source of flakyness

.gitignore

@@ -49,3 +49,4 @@ code/test-results/
 code/playwright-results/
 code/playwright-report/
 code/playwright/.cache/
+code/bench-results/