diff --git a/package.json b/package.json
index d219aa7853..71947fb242 100644
--- a/package.json
+++ b/package.json
@@ -5110,7 +5110,7 @@
     "check:eslint": "eslint src --ext ts",
     "pretest": "yarn run test-compile",
     "test": "node ./out/test/runTest.js",
-    "test:web": "vscode-test-web --browserType=chromium --extensionDevelopmentPath=. --extensionTestsPath=dist/web/test/suite/index.js --permission clipboard-read --permission clipboard-write",
+    "test:web": "./scripts/run-web-test.sh",
     "pretest:web": "yarn run compile-web",
     "compile-web": "webpack --config ./build/web-extension.webpack.config.js",
     "watch-web": "webpack --watch --config ./build/web-extension.webpack.config.js",
diff --git a/scripts/run-web-test.sh b/scripts/run-web-test.sh
new file mode 100755
index 0000000000..e78967ac9c
--- /dev/null
+++ b/scripts/run-web-test.sh
@@ -0,0 +1,25 @@
+#!/bin/bash -eu
+ARGS=$@
+ROOT="$(dirname "$(dirname "$(realpath "$0")")")"
+
+# Remove the "*" activation event from package.json
+# so that the extension is not activated when running tests.
+# Ref: https://github.com/microsoft/vscode-test-web/issues/96#issuecomment-1702658443
+setup() {
+  mv ${ROOT}/package.json ${ROOT}/package.json.bk
+  jq '.activationEvents |= map(select(. != "*"))' ${ROOT}/package.json.bk > ${ROOT}/package.json
+}
+
+tearDown() {
+  mv ${ROOT}/package.json.bk ${ROOT}/package.json
+}
+
+setup
+yarn vscode-test-web \
+  --browserType=chromium \
+  --extensionDevelopmentPath=. \
+  --extensionTestsPath=dist/web/test/suite/index.js \
+  --permission clipboard-read \
+  --permission clipboard-write \
+  $ARGS
+tearDown