From f607049e3e87ae92202fb91fa7cc27c0eba3e651 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 13 Sep 2024 13:25:51 -0400 Subject: [PATCH 01/20] Adding codecov uploading to github and buildkite with monorepo flags --- .buildkite/pipeline.yml | 12 ++++++++++++ .github/workflows/ci.yml | 6 ++++++ packages/config/src/jest/config.js | 3 +++ 3 files changed, 21 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 993722c91..79871e523 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -77,6 +77,9 @@ steps: - exit_status: '*' limit: 2 plugins: + - joscha/codecov#v3.1.0: + args: + - '-F browser' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" paths: ['.yarn/cache/'] @@ -95,6 +98,9 @@ steps: - yarn run -T core test - yarn turbo run --filter=core-integration-tests lint plugins: + - joscha/codecov#v3.1.0: + args: + - '-F core' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" paths: ['.yarn/cache/'] @@ -128,6 +134,9 @@ steps: - echo "+++ Run Tests" - yarn turbo run --filter='./packages/consent/*' test plugins: + - joscha/codecov#v3.1.0: + args: + - '-F consent' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" @@ -147,6 +156,9 @@ steps: - echo "+++ Run Tests" - yarn turbo run --filter='./packages/signals/*' test plugins: + - joscha/codecov#v3.1.0: + args: + - '-F signals' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" paths: ['.yarn/cache/'] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e27e2ab49..355f18a59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,12 @@ jobs: - run: yarn turbo run --filter='./packages/node*' lint - run: yarn turbo run --filter='./packages/node*' test - run: yarn turbo run --filter='./packages/node-integration-tests' test:perf-and-durability + - name: Upload coverage + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: node + slug: segmentio/analytics-next/node analytics-node-cf-workers: name: 'analytics-node QA (Cloudflare Workers)' runs-on: ubuntu-latest diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index f40d0fb65..89ace042b 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,6 +22,9 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ + collectCoverage: process.env.CI === true, + coverageDirectory: path.join(dirname, 'coverage'), + coverageReporters: ['json'], moduleNameMapper: moduleMap, preset: 'ts-jest', modulePathIgnorePatterns: [ From 8a50072eae1d1f698a1d157c06691ccf2d8152f4 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 13 Sep 2024 16:00:35 -0400 Subject: [PATCH 02/20] Trying command line arguments --- .buildkite/pipeline.yml | 8 ++++---- .github/workflows/ci.yml | 2 +- packages/config/src/jest/config.js | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 79871e523..7b8c1899f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -71,7 +71,7 @@ steps: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn install --immutable - PLAYWRIGHT_BROWSERS_PATH=$HOME/pw-browsers npx playwright install - echo "+++ Run Browser integration tests :pray:" - - PLAYWRIGHT_BROWSERS_PATH=$HOME/pw-browsers yarn turbo run --filter=browser-integration-tests test + - PLAYWRIGHT_BROWSERS_PATH=$HOME/pw-browsers yarn turbo run --filter=browser-integration-tests test --coverage retry: automatic: - exit_status: '*' @@ -95,7 +95,7 @@ steps: - yarn run -T core . build - echo "+++ Run tests" - yarn run -T core lint - - yarn run -T core test + - yarn run -T core test --coverage - yarn turbo run --filter=core-integration-tests lint plugins: - joscha/codecov#v3.1.0: @@ -132,7 +132,7 @@ steps: - echo "+++ Run Lint" - yarn turbo run --filter='./packages/consent/*' lint - echo "+++ Run Tests" - - yarn turbo run --filter='./packages/consent/*' test + - yarn turbo run --filter='./packages/consent/*' test --coverage plugins: - joscha/codecov#v3.1.0: args: @@ -154,7 +154,7 @@ steps: - echo "+++ Run Lint" - yarn turbo run --filter='./packages/signals/*' lint - echo "+++ Run Tests" - - yarn turbo run --filter='./packages/signals/*' test + - yarn turbo run --filter='./packages/signals/*' test --coverage plugins: - joscha/codecov#v3.1.0: args: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 355f18a59..cfc6d050e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: restore-keys: | ${{ runner.os }}-turbo- - run: yarn turbo run --filter='./packages/node*' lint - - run: yarn turbo run --filter='./packages/node*' test + - run: yarn turbo run --filter='./packages/node*' test --coverage - run: yarn turbo run --filter='./packages/node-integration-tests' test:perf-and-durability - name: Upload coverage uses: codecov/codecov-action@v4 diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index 89ace042b..73fc38f11 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,9 +22,9 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - collectCoverage: process.env.CI === true, + /*collectCoverage: process.env.CI === true, coverageDirectory: path.join(dirname, 'coverage'), - coverageReporters: ['json'], + coverageReporters: ['json'],*/ moduleNameMapper: moduleMap, preset: 'ts-jest', modulePathIgnorePatterns: [ From 59da6f455feec94916249d6872ca32b4bb8299b3 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 13 Sep 2024 16:16:12 -0400 Subject: [PATCH 03/20] Another attempt --- .buildkite/pipeline.yml | 8 ++++---- .github/workflows/ci.yml | 2 +- packages/config/src/jest/config.js | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7b8c1899f..79871e523 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -71,7 +71,7 @@ steps: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn install --immutable - PLAYWRIGHT_BROWSERS_PATH=$HOME/pw-browsers npx playwright install - echo "+++ Run Browser integration tests :pray:" - - PLAYWRIGHT_BROWSERS_PATH=$HOME/pw-browsers yarn turbo run --filter=browser-integration-tests test --coverage + - PLAYWRIGHT_BROWSERS_PATH=$HOME/pw-browsers yarn turbo run --filter=browser-integration-tests test retry: automatic: - exit_status: '*' @@ -95,7 +95,7 @@ steps: - yarn run -T core . build - echo "+++ Run tests" - yarn run -T core lint - - yarn run -T core test --coverage + - yarn run -T core test - yarn turbo run --filter=core-integration-tests lint plugins: - joscha/codecov#v3.1.0: @@ -132,7 +132,7 @@ steps: - echo "+++ Run Lint" - yarn turbo run --filter='./packages/consent/*' lint - echo "+++ Run Tests" - - yarn turbo run --filter='./packages/consent/*' test --coverage + - yarn turbo run --filter='./packages/consent/*' test plugins: - joscha/codecov#v3.1.0: args: @@ -154,7 +154,7 @@ steps: - echo "+++ Run Lint" - yarn turbo run --filter='./packages/signals/*' lint - echo "+++ Run Tests" - - yarn turbo run --filter='./packages/signals/*' test --coverage + - yarn turbo run --filter='./packages/signals/*' test plugins: - joscha/codecov#v3.1.0: args: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc6d050e..355f18a59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: restore-keys: | ${{ runner.os }}-turbo- - run: yarn turbo run --filter='./packages/node*' lint - - run: yarn turbo run --filter='./packages/node*' test --coverage + - run: yarn turbo run --filter='./packages/node*' test - run: yarn turbo run --filter='./packages/node-integration-tests' test:perf-and-durability - name: Upload coverage uses: codecov/codecov-action@v4 diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index 73fc38f11..5569591dc 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,9 +22,8 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - /*collectCoverage: process.env.CI === true, - coverageDirectory: path.join(dirname, 'coverage'), - coverageReporters: ['json'],*/ + collectCoverage: process.env.CI === true, + coverageReporters: ['text'], moduleNameMapper: moduleMap, preset: 'ts-jest', modulePathIgnorePatterns: [ From 9ba5baa19e18fd24416122b5712680c579ca0d32 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 13 Sep 2024 17:29:32 -0400 Subject: [PATCH 04/20] Using segment codecov uploader possibly? --- .buildkite/pipeline.yml | 22 ++++++++++------------ packages/config/src/jest/config.js | 6 ++++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 79871e523..809e71b70 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -12,6 +12,8 @@ steps: - echo "CI -> $CI" - label: '[Monorepo] QA' key: monorepo + env: + SEGMENT_CODECOV_FLAGS: 'scripts' agents: queue: v1 commands: @@ -64,6 +66,8 @@ steps: - label: '[Browser] Integration Tests :perfection:' key: browser-integ + env: + SEGMENT_CODECOV_FLAGS: 'browser' agents: queue: v1 commands: @@ -77,14 +81,13 @@ steps: - exit_status: '*' limit: 2 plugins: - - joscha/codecov#v3.1.0: - args: - - '-F browser' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" paths: ['.yarn/cache/'] - label: '[Core] Lint + Test' + env: + SEGMENT_CODECOV_FLAGS: 'core' agents: queue: v1 commands: @@ -98,9 +101,6 @@ steps: - yarn run -T core test - yarn turbo run --filter=core-integration-tests lint plugins: - - joscha/codecov#v3.1.0: - args: - - '-F core' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" paths: ['.yarn/cache/'] @@ -121,6 +121,8 @@ steps: paths: ['.yarn/cache/'] - label: '[Consent] Lint + Test' + env: + SEGMENT_CODECOV_FLAGS: 'consent' agents: queue: v1 commands: @@ -134,13 +136,12 @@ steps: - echo "+++ Run Tests" - yarn turbo run --filter='./packages/consent/*' test plugins: - - joscha/codecov#v3.1.0: - args: - - '-F consent' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" - label: '[Signals] Lint + Test' + env: + SEGMENT_CODECOV_FLAGS: 'signals' agents: queue: v1 commands: @@ -156,9 +157,6 @@ steps: - echo "+++ Run Tests" - yarn turbo run --filter='./packages/signals/*' test plugins: - - joscha/codecov#v3.1.0: - args: - - '-F signals' - ssh://git@github.com/segmentio/cache-buildkite-plugin#v2.0.0: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" paths: ['.yarn/cache/'] diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index 5569591dc..887baf784 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,8 +22,10 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - collectCoverage: process.env.CI === true, - coverageReporters: ['text'], + ...(process.env.CI === 'true' && isRootConfig + ? { collectCoverage: true } + : {}), + coverageReporters: ['json'], moduleNameMapper: moduleMap, preset: 'ts-jest', modulePathIgnorePatterns: [ From a599898f05416d003d9d7dec9172c4b17f0bac53 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 16 Sep 2024 11:48:59 -0400 Subject: [PATCH 05/20] Excepting backwards compatibility test from coverage due to jest errors --- packages/config/src/jest/config.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index 887baf784..b4b99c193 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,10 +22,13 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - ...(process.env.CI === 'true' && isRootConfig - ? { collectCoverage: true } + ...(process.env.CI !== 'true' && isRootConfig + ? { + collectCoverage: true, + coverageReporters: ['json'], + coveragePathIgnorePatterns: ['*backwards-compatibility*'], + } : {}), - coverageReporters: ['json'], moduleNameMapper: moduleMap, preset: 'ts-jest', modulePathIgnorePatterns: [ From 75f302dde96a47dad57d6bff88eb8014eff54ec6 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 16 Sep 2024 12:00:14 -0400 Subject: [PATCH 06/20] Fixing logic from local testing --- packages/config/src/jest/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index b4b99c193..1b76b18dd 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,7 +22,7 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - ...(process.env.CI !== 'true' && isRootConfig + ...(process.env.CI === 'true' && isRootConfig ? { collectCoverage: true, coverageReporters: ['json'], From 866d24ecaed12d4364ae712d8b8dcb79ac5e68c8 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Wed, 13 Nov 2024 20:29:37 -0500 Subject: [PATCH 07/20] Specifying directory per package and using buildkite for upload --- .buildkite/pipeline.yml | 5 +++++ .github/workflows/ci.yml | 6 ------ packages/config/src/jest/config.js | 1 - 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 809e71b70..0d72f0d59 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,6 +14,7 @@ steps: key: monorepo env: SEGMENT_CODECOV_FLAGS: 'scripts' + SEGMENT_CODECOV_ARGS: '--dir=packages/scripts/coverage' agents: queue: v1 commands: @@ -68,6 +69,7 @@ steps: key: browser-integ env: SEGMENT_CODECOV_FLAGS: 'browser' + SEGMENT_CODECOV_ARGS: '--dir=packages/browser/coverage' agents: queue: v1 commands: @@ -88,6 +90,7 @@ steps: - label: '[Core] Lint + Test' env: SEGMENT_CODECOV_FLAGS: 'core' + SEGMENT_CODECOV_ARGS: '--dir=packages/core/coverage' agents: queue: v1 commands: @@ -123,6 +126,7 @@ steps: - label: '[Consent] Lint + Test' env: SEGMENT_CODECOV_FLAGS: 'consent' + SEGMENT_CODECOV_ARGS: '--dir=packages/consent/coverage' agents: queue: v1 commands: @@ -142,6 +146,7 @@ steps: - label: '[Signals] Lint + Test' env: SEGMENT_CODECOV_FLAGS: 'signals' + SEGMENT_CODECOV_ARGS: '--dir=packages/signals/coverage' agents: queue: v1 commands: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 355f18a59..e27e2ab49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,12 +28,6 @@ jobs: - run: yarn turbo run --filter='./packages/node*' lint - run: yarn turbo run --filter='./packages/node*' test - run: yarn turbo run --filter='./packages/node-integration-tests' test:perf-and-durability - - name: Upload coverage - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: node - slug: segmentio/analytics-next/node analytics-node-cf-workers: name: 'analytics-node QA (Cloudflare Workers)' runs-on: ubuntu-latest diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index 1b76b18dd..9c067101d 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -26,7 +26,6 @@ const createJestTSConfig = ( ? { collectCoverage: true, coverageReporters: ['json'], - coveragePathIgnorePatterns: ['*backwards-compatibility*'], } : {}), moduleNameMapper: moduleMap, From f8f033a8056af9755bcc2ed27fe96ae85fb09e23 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 14 Nov 2024 17:55:11 -0500 Subject: [PATCH 08/20] More codecov fixes --- packages/config/src/jest/config.js | 9 ++++++++- scripts/clean.sh | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index 9c067101d..d2910b9b0 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,10 +22,17 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - ...(process.env.CI === 'true' && isRootConfig + ...(process.env.CI === 'true' ? { collectCoverage: true, coverageReporters: ['json'], + collectCoverageFrom: [ + 'src/**/*.{js,jsx,ts,tsx}', + '!src/**/*.test.{js,jsx,ts,tsx}', + // Exclude test files "!src/**/index.{js,ts}", + // Exclude index files + ], + coverageDirectory: '/coverage', } : {}), moduleNameMapper: moduleMap, diff --git a/scripts/clean.sh b/scripts/clean.sh index b1c8477a8..c6c312ad7 100644 --- a/scripts/clean.sh +++ b/scripts/clean.sh @@ -1,7 +1,7 @@ #!/bin/sh # Clear build artifacts and build cache -find . \( -name ".turbo" -o -name "dist" -o -name ".next" -o -name "tsconfig.tsbuildinfo" \) ! -path "*/node_modules/*" -print0 | xargs -0 rm -rf +find . \( -name ".turbo" -o -name "dist" -o -name ".next" -o -name "tsconfig.tsbuildinfo" -o -name "coverage" \) ! -path "*/node_modules/*" -print0 | xargs -0 rm -rf rm -rf node_modules/.cache echo "Build files and cache deleted." From ddf268c7a4739d621e76bbe0f3305c91a2c8e639 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 14 Nov 2024 18:21:02 -0500 Subject: [PATCH 09/20] Changing how coverage is initialized --- .buildkite/pipeline.yml | 1 + packages/config/src/jest/config.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0d72f0d59..3359a34d8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,4 +1,5 @@ env: + COVERAGE: true SEGMENT_CONTEXTS: 'snyk,npm,aws-credentials,ecr,npm-publish' SEGMENT_BUILDKITE_IMAGE: 'analytics-next-ci-agent' steps: diff --git a/packages/config/src/jest/config.js b/packages/config/src/jest/config.js index d2910b9b0..00c53331c 100644 --- a/packages/config/src/jest/config.js +++ b/packages/config/src/jest/config.js @@ -22,7 +22,7 @@ const createJestTSConfig = ( * No need to manually run yarn build all the time. * This resolve packages for ts-jest so typescript compilation happens in-memory. */ - ...(process.env.CI === 'true' + ...(process.env.COVERAGE === 'true' ? { collectCoverage: true, coverageReporters: ['json'], From 9daabe54c0fbf053512d5b02306ba117c3d3358f Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 14 Nov 2024 18:36:18 -0500 Subject: [PATCH 10/20] Adding an intiial codecov config --- codecov.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..f5be7fd90 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,7 @@ +coverage: + status: + project: + default: + target: auto + threshold: 0% + base: auto From b4f5751e7b2df0e71d6e67902f964e9716ded1fc Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 19 Nov 2024 13:46:48 -0500 Subject: [PATCH 11/20] Updating *cough* old browser coverage thresholds --- packages/browser/jest.config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/browser/jest.config.js b/packages/browser/jest.config.js index b3f10efe1..2e212a7fb 100644 --- a/packages/browser/jest.config.js +++ b/packages/browser/jest.config.js @@ -6,10 +6,10 @@ module.exports = createJestTSConfig(__dirname, { testEnvironment: 'jsdom', coverageThreshold: { global: { - branches: 80.91, - functions: 87.25, - lines: 91.03, - statements: 87.25, + branches: 74, + functions: 80, + lines: 87, + statements: 82, }, }, }) From eded328aff0cffaa6464c021622d7a03716ab724 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 19 Nov 2024 13:52:46 -0500 Subject: [PATCH 12/20] removing custom buildkite image --- .buildkite/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3359a34d8..a0745efde 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,7 +1,6 @@ env: COVERAGE: true SEGMENT_CONTEXTS: 'snyk,npm,aws-credentials,ecr,npm-publish' - SEGMENT_BUILDKITE_IMAGE: 'analytics-next-ci-agent' steps: - label: Log Environment agents: From 3b9f86cf3eacedfdafa6672fdc14b7575b1771ce Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 19 Nov 2024 13:57:33 -0500 Subject: [PATCH 13/20] Adding new standard image. --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a0745efde..97ad36664 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,7 @@ env: COVERAGE: true SEGMENT_CONTEXTS: 'snyk,npm,aws-credentials,ecr,npm-publish' + SEGMENT_BUILDKITE_IMAGE: 'buildkite-agent-node20' steps: - label: Log Environment agents: From e2e908079c7aa61367fbe526137ef22d334d015f Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:06:56 -0600 Subject: [PATCH 14/20] wip --- .buildkite/Dockerfile.agent | 54 ++----------------------------------- .buildkite/pipeline.yml | 18 +------------ 2 files changed, 3 insertions(+), 69 deletions(-) diff --git a/.buildkite/Dockerfile.agent b/.buildkite/Dockerfile.agent index 8b5a6b52e..d73820c82 100644 --- a/.buildkite/Dockerfile.agent +++ b/.buildkite/Dockerfile.agent @@ -1,55 +1,5 @@ -FROM 528451384384.dkr.ecr.us-west-2.amazonaws.com/buildkite-agent-node16:node-16.16 +FROM 528451384384.dkr.ecr.us-west-2.amazonaws.com/buildkite-agent-node16:latest -# 2. Install WebKit dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libwoff1 \ - libopus0 \ - libwebp6 \ - libwebpdemux2 \ - libenchant1c2a \ - libgudev-1.0-0 \ - libsecret-1-0 \ - libhyphen0 \ - libgdk-pixbuf2.0-0 \ - libegl1 \ - libnotify4 \ - libxslt1.1 \ - libevent-2.1-6 \ - libgles2 \ - libvpx5 \ - libxcomposite1 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libepoxy0 \ - libgtk-3-0 \ - libharfbuzz-icu0 - -# 3. Install gstreamer and plugins to support video playback in WebKit. -RUN apt-get update && apt-get install -y --no-install-recommends \ - libgstreamer-gl1.0-0 \ - libgstreamer-plugins-bad1.0-0 \ - gstreamer1.0-plugins-good \ - gstreamer1.0-libav - -# 4. Install Chromium dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libnss3 \ - libxss1 \ - libasound2 \ - fonts-noto-color-emoji \ - libxtst6 - -# 5. Install Firefox dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libdbus-glib-1-2 \ - libxt6 - -# 6. Install ffmpeg to bring in audio and video codecs necessary for playing videos in Firefox. -RUN apt-get update && apt-get install -y --no-install-recommends \ - ffmpeg - -# 7. (Optional) Install XVFB if there's a need to run browsers in headful mode -RUN apt-get update && apt-get install -y --no-install-recommends \ - xvfb +RUN sudo npx playwright install-deps ENTRYPOINT [] diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 97ad36664..993722c91 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,7 +1,6 @@ env: - COVERAGE: true SEGMENT_CONTEXTS: 'snyk,npm,aws-credentials,ecr,npm-publish' - SEGMENT_BUILDKITE_IMAGE: 'buildkite-agent-node20' + SEGMENT_BUILDKITE_IMAGE: 'analytics-next-ci-agent' steps: - label: Log Environment agents: @@ -13,9 +12,6 @@ steps: - echo "CI -> $CI" - label: '[Monorepo] QA' key: monorepo - env: - SEGMENT_CODECOV_FLAGS: 'scripts' - SEGMENT_CODECOV_ARGS: '--dir=packages/scripts/coverage' agents: queue: v1 commands: @@ -68,9 +64,6 @@ steps: - label: '[Browser] Integration Tests :perfection:' key: browser-integ - env: - SEGMENT_CODECOV_FLAGS: 'browser' - SEGMENT_CODECOV_ARGS: '--dir=packages/browser/coverage' agents: queue: v1 commands: @@ -89,9 +82,6 @@ steps: paths: ['.yarn/cache/'] - label: '[Core] Lint + Test' - env: - SEGMENT_CODECOV_FLAGS: 'core' - SEGMENT_CODECOV_ARGS: '--dir=packages/core/coverage' agents: queue: v1 commands: @@ -125,9 +115,6 @@ steps: paths: ['.yarn/cache/'] - label: '[Consent] Lint + Test' - env: - SEGMENT_CODECOV_FLAGS: 'consent' - SEGMENT_CODECOV_ARGS: '--dir=packages/consent/coverage' agents: queue: v1 commands: @@ -145,9 +132,6 @@ steps: key: "v1.1-cache-dev-{{ checksum 'yarn.lock' }}" - label: '[Signals] Lint + Test' - env: - SEGMENT_CODECOV_FLAGS: 'signals' - SEGMENT_CODECOV_ARGS: '--dir=packages/signals/coverage' agents: queue: v1 commands: From bbe57f99dab91e25e1dc5f7d4893031f3cf753de Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:09:02 -0600 Subject: [PATCH 15/20] update readme --- .buildkite/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/Readme.md b/.buildkite/Readme.md index c38a7468c..054518bdb 100644 --- a/.buildkite/Readme.md +++ b/.buildkite/Readme.md @@ -5,7 +5,7 @@ Builds the base image that is used by analytics-next in CI. ```bash -$ robo docker.login-privileged +$ robo-tooling.docker.login $ make agent ``` From fdbe66675741f3ec117c4340a64802add9698bfa Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:13:05 -0600 Subject: [PATCH 16/20] wip --- .buildkite/Dockerfile.agent | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/Dockerfile.agent b/.buildkite/Dockerfile.agent index d73820c82..7ef44e24b 100644 --- a/.buildkite/Dockerfile.agent +++ b/.buildkite/Dockerfile.agent @@ -1,5 +1,5 @@ -FROM 528451384384.dkr.ecr.us-west-2.amazonaws.com/buildkite-agent-node16:latest +FROM 528451384384.dkr.ecr.us-west-2.amazonaws.com/buildkite-agent-node20 -RUN sudo npx playwright install-deps +RUN npx playwright install-deps ENTRYPOINT [] From d98f01846aee0a5ca98e63c6ac9f52f0e10f8467 Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:30:58 -0600 Subject: [PATCH 17/20] wip --- .buildkite/Readme.md | 12 ++---- .github/workflows/ci.yml | 2 +- .github/workflows/create-github-release.yml | 4 +- .github/workflows/release-creator.yml | 4 +- .nvmrc | 2 +- package.json | 2 +- packages/browser/package.json | 2 +- packages/node/package.json | 2 +- .../signals/signals/src/core/client/redact.ts | 6 ++- scripts/package.json | 2 +- yarn.lock | 38 +++++++++++++++++-- 11 files changed, 54 insertions(+), 22 deletions(-) diff --git a/.buildkite/Readme.md b/.buildkite/Readme.md index 054518bdb..e3cbdb22b 100644 --- a/.buildkite/Readme.md +++ b/.buildkite/Readme.md @@ -1,14 +1,10 @@ # Buildkite setup -## Dockerfile.agent - -Builds the base image that is used by analytics-next in CI. - +## How to update the buildkite docker agent +1. Make your changes to `Dockerfile.agent`. +2. Push the changes to ecr +(will need `Ops Write` permission). ```bash $ robo-tooling.docker.login $ make agent ``` - -## .pipeline - -Full buildkite configuration. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e27e2ab49..0835259b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - uses: browser-actions/setup-chrome@v1 - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 20 cache: 'yarn' - run: yarn install --immutable - name: Turbo cache diff --git a/.github/workflows/create-github-release.yml b/.github/workflows/create-github-release.yml index 5a6957951..185d40363 100644 --- a/.github/workflows/create-github-release.yml +++ b/.github/workflows/create-github-release.yml @@ -11,10 +11,10 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v3 - - name: Setup Node.js 16.x + - name: Setup Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 20.x cache: "yarn" - name: Install Dependencies run: HUSKY=0 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn install --immutable diff --git a/.github/workflows/release-creator.yml b/.github/workflows/release-creator.yml index c2e43a733..69d10066a 100644 --- a/.github/workflows/release-creator.yml +++ b/.github/workflows/release-creator.yml @@ -21,10 +21,10 @@ jobs: # This makes Actions fetch all Git history so that Changesets can generate changelogs with the correct commits fetch-depth: 0 - - name: Setup Node.js 16.x + - name: Setup Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 20.x cache: "yarn" - name: Install Dependencies diff --git a/.nvmrc b/.nvmrc index d4b25d088..f3f52b42d 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -16.16 +20.9.0 diff --git a/package.json b/package.json index 43c93ae6d..246980765 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "scripts" ], "engines": { - "node": "^16.16.0" + "node": "^20" }, "scripts": { "test": "jest", diff --git a/packages/browser/package.json b/packages/browser/package.json index 330c3391e..89244a7a4 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -74,7 +74,7 @@ "@types/js-cookie": "3.0.1", "@types/jsdom": "^16.2.14", "@types/mime": "^2.0.3", - "@types/node": "^16", + "@types/node": "^20", "@types/serve-handler": "^6.1.0", "aws-sdk": "^2.814.0", "circular-dependency-plugin": "^5.2.2", diff --git a/packages/node/package.json b/packages/node/package.json index 7124262e9..67f15ca16 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -45,7 +45,7 @@ }, "devDependencies": { "@internal/config": "0.0.0", - "@types/node": "^16", + "@types/node": "^18", "axios": "^1.6.2" }, "packageManager": "yarn@3.4.1" diff --git a/packages/signals/signals/src/core/client/redact.ts b/packages/signals/signals/src/core/client/redact.ts index 3c28c2ada..6f7a7eb61 100644 --- a/packages/signals/signals/src/core/client/redact.ts +++ b/packages/signals/signals/src/core/client/redact.ts @@ -3,7 +3,11 @@ import { Signal } from '@segment/analytics-signals-runtime' export const redactSignalData = (signalArg: Signal): Signal => { const signal = structuredClone(signalArg) if (signal.type === 'interaction') { - if ('target' in signal.data) { + if ( + 'target' in signal.data && + signal.data.target && + typeof signal.data.target === 'object' + ) { if ('value' in signal.data.target) { signal.data.target.value = redactJsonValues(signal.data.target.value) } diff --git a/scripts/package.json b/scripts/package.json index 9cf25a45d..c68ec6d17 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -15,7 +15,7 @@ "packageManager": "yarn@3.4.1", "devDependencies": { "@node-kit/yarn-workspace-root": "^3.2.0", - "@types/node": "^16", + "@types/node": "^20", "ts-node": "^10.8.0" } } diff --git a/yarn.lock b/yarn.lock index e4d322be6..ade0dc543 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2984,7 +2984,7 @@ __metadata: resolution: "@internal/scripts@workspace:scripts" dependencies: "@node-kit/yarn-workspace-root": ^3.2.0 - "@types/node": ^16 + "@types/node": ^20 ts-node: ^10.8.0 languageName: unknown linkType: soft @@ -4273,7 +4273,7 @@ __metadata: "@types/js-cookie": 3.0.1 "@types/jsdom": ^16.2.14 "@types/mime": ^2.0.3 - "@types/node": ^16 + "@types/node": ^20 "@types/serve-handler": ^6.1.0 aws-sdk: ^2.814.0 circular-dependency-plugin: ^5.2.2 @@ -4315,7 +4315,7 @@ __metadata: "@lukeed/uuid": ^2.0.0 "@segment/analytics-core": 1.8.0 "@segment/analytics-generic-utils": 1.2.0 - "@types/node": ^16 + "@types/node": ^18 axios: ^1.6.2 buffer: ^6.0.3 jose: ^5.1.0 @@ -6463,6 +6463,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^18": + version: 18.19.64 + resolution: "@types/node@npm:18.19.64" + dependencies: + undici-types: ~5.26.4 + checksum: e7680215b03c9bee8a33947f03d06048e8e460f23b1b7b29c45350cf437faa5f8fcb7d8c3eb8dfec8427923e7a184df42bc710c1b6252b4852e3ed7064c6228f + languageName: node + linkType: hard + "@types/node@npm:^18.11.15": version: 18.17.5 resolution: "@types/node@npm:18.17.5" @@ -6470,6 +6479,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20": + version: 20.17.6 + resolution: "@types/node@npm:20.17.6" + dependencies: + undici-types: ~6.19.2 + checksum: d51dbb9881c94d0310b32b5fd8013e3261595c61bc888fa27258469c93c3dc0b3c4d20a9f28f3f5f79562f6737e28e7f3dd04940dc8b4d966d34aaf318f7f69b + languageName: node + linkType: hard + "@types/node@npm:^20.1.0, @types/node@npm:^20.1.1": version: 20.3.1 resolution: "@types/node@npm:20.3.1" @@ -22393,6 +22411,20 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~5.26.4": + version: 5.26.5 + resolution: "undici-types@npm:5.26.5" + checksum: 3192ef6f3fd5df652f2dc1cd782b49d6ff14dc98e5dced492aa8a8c65425227da5da6aafe22523c67f035a272c599bb89cfe803c1db6311e44bed3042fc25487 + languageName: node + linkType: hard + +"undici-types@npm:~6.19.2": + version: 6.19.8 + resolution: "undici-types@npm:6.19.8" + checksum: de51f1b447d22571cf155dfe14ff6d12c5bdaec237c765085b439c38ca8518fc360e88c70f99469162bf2e14188a7b0bcb06e1ed2dc031042b984b0bb9544017 + languageName: node + linkType: hard + "undici@npm:^5.22.1": version: 5.28.4 resolution: "undici@npm:5.28.4" From 75f2accf83af021c7679eec667175011d8437fbe Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:40:40 -0600 Subject: [PATCH 18/20] wip --- .buildkite/Readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/Readme.md b/.buildkite/Readme.md index e3cbdb22b..db662d8a1 100644 --- a/.buildkite/Readme.md +++ b/.buildkite/Readme.md @@ -5,6 +5,7 @@ 2. Push the changes to ecr (will need `Ops Write` permission). ```bash -$ robo-tooling.docker.login +$ cd .buildkite +$ robo-tooling.docker.login-privileged $ make agent ``` From 06ba479232bd65798968972e864be111ba9ea15e Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:40:50 -0600 Subject: [PATCH 19/20] wip --- .buildkite/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/Readme.md b/.buildkite/Readme.md index db662d8a1..da5d625b6 100644 --- a/.buildkite/Readme.md +++ b/.buildkite/Readme.md @@ -1,4 +1,4 @@ -# Buildkite setup +# Buildkite ## How to update the buildkite docker agent 1. Make your changes to `Dockerfile.agent`. From 69bfaa469a1e37869e409f24c18acefd61c68e1e Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:06:56 -0600 Subject: [PATCH 20/20] downgrade consent integration tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0835259b0..a7db4a5a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - uses: browser-actions/setup-chrome@v1 - uses: actions/setup-node@v3 with: - node-version: 20 + node-version: 16 # UPDATE TO NODE 20!!! cache: 'yarn' - run: yarn install --immutable - name: Turbo cache