chore: Remove retries for testcafe tests but increase timeout. Wait f…

…or lag in parallel (#1380)

* Bump typescript version to 5.5 and also bump related libraries

* Remove retries but increase timeout

* Wrap in Selector

* Add console log saying how long it took to get results

* Add "after" query param

* Add max allowed api errors

* Stop on first fail

* Make testSessionId more readable, change ingestion endpoint

* Don't use uuidv7

* Pass env vars as dependencies

* Improve test namr

* Fix typo

* Typo

* only assert that custom-event was seen

* try using a different distinct_id

* undo changes, look for all events

* use successfunction to look for custom-event in response

* Move asserts to a later step in the CI so that ingestion lag can be in parallel

* Tidy up deps

Move ts-node to dev deps
Upgrade express due to dep warning

* Tidy up how results file are written and read

* Use jest override

* Fix complaining about timeout and interval types

* WIP

* Override testcafe's typescript compiler

* Revert check script to js

* Sanitise file names

* Correctly named script

* Some more debuggability

* Add a script to make it easier to run locally

* Add some debugging

* Fix check file

* Revert changes to wait duration

* Move waits around

* Add assertion for whether the snippet has loaded

* Revert the assertion event count

* Re-add missing asserts

* Remove some unused stuff

* Be more specific about test files

* Fix url

* Disable request batching

* Rewrite the retry loop to do an attempt immediately

* Assert on the number of captures

* Fix event count assert

* Add captures map

* Test

* Fix captures

* Fix first assert

* Fix other asserts

* Fix deadline

* Increase timeout to 30 minutes

* Use optional catch binding syntax

* Add an explainer comment

* Remove unnecessary hackiness

* Slightly improve the hackiness even more

---------

Co-authored-by: Phani Raj <[email protected]>

.eslintrc.js

@@ -7,6 +7,7 @@ const rules = {
     '@typescript-eslint/no-this-alias': 'off',
     '@typescript-eslint/no-explicit-any': 'off',
     '@typescript-eslint/no-unused-vars': ['error'],
+    '@typescript-eslint/no-unused-expressions': 'off',
     'no-prototype-builtins': 'off',
     'no-empty': 'off',
     'no-console': 'error',
@@ -101,6 +102,12 @@ module.exports = {
                 Cypress: true,
             },
         },
+        {
+            files: 'testcafe/**/*',
+            globals: {
+                __dirname: true,
+            },
+        },
     ],
     root: true,
 }

.github/workflows/testcafe.yml

@@ -54,4 +54,11 @@ jobs:
         run: pnpm build-rollup
 
       - name: Run ${{ matrix.name }} test
-        run: node scripts/run-testcafe-with-retries.mjs --browser "${{ matrix.browser }}" --attempts 3
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+          RUN_ID: ${{ github.run_id }}
+          BROWSER: ${{ matrix.browser }}
+        run: pnpm testcafe ${{ matrix.browser }} --stop-on-first-fail
+
+      - name: Check ${{ matrix.name }} events
+        run: pnpm check-testcafe-results

.testcaferc.js

@@ -1,3 +1,8 @@
 module.exports = {
-    src: './testcafe',
+    src: './testcafe/*.spec.js',
+    compilerOptions: {
+        typescript: {
+            customCompilerModulePath: require.resolve('typescript'),
+        },
+    },
 }

package.json

@@ -23,7 +23,9 @@
         "typecheck": "tsc --noEmit --project tsconfig.json",
         "cypress": "cypress open",
         "prepare": "husky install",
-        "deprecate-old-versions": "node scripts/deprecate-old-versions.mjs"
+        "deprecate-old-versions": "node scripts/deprecate-old-versions.mjs",
+        "check-testcafe-results": "ts-node testcafe/check-testcafe-results.js",
+        "run-testcafe-localhost": "node scripts/run-testcafe-localhost.mjs"
     },
     "main": "dist/main.js",
     "module": "dist/module.js",
@@ -59,37 +61,39 @@
         "@testing-library/preact": "^3.2.4",
         "@types/eslint": "^8.44.6",
         "@types/jest": "^29.5.1",
+        "@types/node": "^22.5.0",
         "@types/react-dom": "^18.0.10",
         "@types/sinon": "^17.0.1",
         "@types/uuid": "^9.0.1",
         "@types/web": "^0.0.154",
-        "@typescript-eslint/eslint-plugin": "^6.19.0",
-        "@typescript-eslint/parser": "^6.19.0",
+        "@typescript-eslint/eslint-plugin": "^8.2.0",
+        "@typescript-eslint/parser": "^8.2.0",
         "babel-eslint": "10.1.0",
         "babel-jest": "^26.6.3",
         "compare-versions": "^6.1.0",
         "cypress": "13.6.3",
         "cypress-localstorage-commands": "^2.2.6",
         "date-fns": "^3.6.0",
-        "eslint": "8.56.0",
+        "eslint": "8.57.0",
         "eslint-config-posthog-js": "link:eslint-rules",
         "eslint-config-prettier": "^8.5.0",
         "eslint-plugin-compat": "^4.1.4",
-        "eslint-plugin-jest": "^27.2.3",
+        "eslint-plugin-jest": "^28.8.0",
         "eslint-plugin-no-only-tests": "^3.1.0",
         "eslint-plugin-posthog-js": "link:eslint-rules",
         "eslint-plugin-prettier": "^4.2.1",
         "eslint-plugin-react": "^7.30.1",
         "eslint-plugin-react-hooks": "^4.6.0",
-        "express": "^4.18.2",
+        "expect": "^29.7.0",
+        "express": "^4.19.2",
         "fast-check": "^2.17.0",
         "husky": "^8.0.1",
         "jest": "^27.5.1",
         "jsdom": "16.5.0",
         "jsdom-global": "3.0.2",
         "lint-staged": "^10.2.11",
         "localStorage": "1.0.4",
-        "msw": "^1.2.1",
+        "msw": "^1.3.3",
         "node-fetch": "^2.6.11",
         "posthog-js": "link:",
         "preact-render-to-string": "^6.3.1",
@@ -103,7 +107,8 @@
         "testcafe": "1.19.0",
         "testcafe-browser-provider-browserstack": "1.14.0",
         "tslib": "^2.5.0",
-        "typescript": "^4.9.5",
+        "ts-node": "^10.9.2",
+        "typescript": "^5.5.4",
         "yargs": "^17.7.2"
     },
     "lint-staged": {