From cc1161a52a4b23f779fc00a34007d132193c5157 Mon Sep 17 00:00:00 2001 From: Mark Stacey Date: Tue, 1 Dec 2020 17:54:56 -0330 Subject: [PATCH] Add metrics e2e test (#9784) An e2e test has been added that uses the new mock Segment server to verify that the three initial page metric events are sent correctly. Using the mock Segment server requires a special build with this mock Segment server hostname embedded, so a distinct job for building and running this test was required. As such, it was left out of the `run-all.sh` script. --- .circleci/config.yml | 79 +++++++++ package.json | 3 + test/e2e/fixtures/metrics-enabled/state.json | 168 +++++++++++++++++++ test/e2e/helpers.js | 36 +++- test/e2e/metrics.spec.js | 57 +++++++ 5 files changed, 341 insertions(+), 2 deletions(-) create mode 100644 test/e2e/fixtures/metrics-enabled/state.json create mode 100644 test/e2e/metrics.spec.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c43e17b6559..cd6efe2a3742 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,6 +18,9 @@ workflows: - prep-build-test: requires: - prep-deps + - prep-build-test-metrics: + requires: + - prep-deps - prep-build-storybook: requires: - prep-deps @@ -34,6 +37,12 @@ workflows: - test-e2e-firefox: requires: - prep-build-test + - test-e2e-chrome-metrics: + requires: + - prep-build-test-metrics + - test-e2e-firefox-metrics: + requires: + - prep-build-test-metrics - test-unit: requires: - prep-deps @@ -58,6 +67,8 @@ workflows: - test-mozilla-lint - test-e2e-chrome - test-e2e-firefox + - test-e2e-chrome-metrics + - test-e2e-firefox-metrics - benchmark: requires: - prep-build-test @@ -162,6 +173,27 @@ jobs: paths: - dist-test + prep-build-test-metrics: + docker: + - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88 + resource_class: medium+ + environment: + NODE_OPTIONS: --max_old_space_size=2048 + steps: + - checkout + - attach_workspace: + at: . + - run: + name: Build extension for testing metrics + command: yarn build:test:metrics + - run: + name: Move test build to 'dist-test-metrics' to avoid conflict with production build + command: mv ./dist ./dist-test-metrics + - persist_to_workspace: + root: . + paths: + - dist-test-metrics + prep-build-storybook: docker: - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88 @@ -245,6 +277,28 @@ jobs: path: test-artifacts destination: test-artifacts + test-e2e-chrome-metrics: + docker: + - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88 + steps: + - checkout + - attach_workspace: + at: . + - run: + name: Move test build to dist + command: mv ./dist-test-metrics ./dist + - run: + name: test:e2e:chrome:metrics + command: | + if .circleci/scripts/test-run-e2e + then + yarn test:e2e:chrome:metrics + fi + no_output_timeout: 20m + - store_artifacts: + path: test-artifacts + destination: test-artifacts + test-e2e-firefox: docker: - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88 @@ -270,6 +324,31 @@ jobs: path: test-artifacts destination: test-artifacts + test-e2e-firefox-metrics: + docker: + - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88 + steps: + - checkout + - run: + name: Install Firefox + command: ./.circleci/scripts/firefox-install + - attach_workspace: + at: . + - run: + name: Move test build to dist + command: mv ./dist-test-metrics ./dist + - run: + name: test:e2e:firefox:metrics + command: | + if .circleci/scripts/test-run-e2e + then + yarn test:e2e:firefox:metrics + fi + no_output_timeout: 20m + - store_artifacts: + path: test-artifacts + destination: test-artifacts + benchmark: docker: - image: circleci/node@sha256:e16740707de2ebed45c05d507f33ef204902349c7356d720610b5ec6a35d3d88 diff --git a/package.json b/package.json index 01a7ffb908c7..dab32252ba36 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "benchmark:chrome": "SELENIUM_BROWSER=chrome node test/e2e/benchmark.js", "benchmark:firefox": "SELENIUM_BROWSER=firefox node test/e2e/benchmark.js", "build:test": "yarn build test", + "build:test:metrics": "SEGMENT_HOST='http://localhost:9090' SEGMENT_WRITE_KEY='FAKE' SEGMENT_LEGACY_WRITE_KEY='FAKE' yarn build test", "test": "yarn test:unit && yarn lint", "dapp": "node development/static-server.js node_modules/@metamask/test-dapp/dist --port 8080", "dapp-chain": "GANACHE_ARGS='-b 2' concurrently -k -n ganache,dapp -p '[{time}][{name}]' 'yarn ganache:start' 'sleep 5 && yarn dapp'", @@ -22,7 +23,9 @@ "test:unit:strict": "mocha --exit --require test/env.js --require test/setup.js --recursive \"test/unit/**/permissions/*.js\"", "test:unit:path": "mocha --exit --require test/env.js --require test/setup.js --recursive", "test:e2e:chrome": "SELENIUM_BROWSER=chrome test/e2e/run-all.sh", + "test:e2e:chrome:metrics": "SELENIUM_BROWSER=chrome mocha test/e2e/metrics.spec.js", "test:e2e:firefox": "SELENIUM_BROWSER=firefox test/e2e/run-all.sh", + "test:e2e:firefox:metrics": "SELENIUM_BROWSER=firefox mocha test/e2e/metrics.spec.js", "test:coverage": "nyc --silent --check-coverage yarn test:unit:strict && nyc --silent --no-clean yarn test:unit:lax && nyc report --reporter=text --reporter=html", "test:coverage:strict": "nyc --check-coverage yarn test:unit:strict", "test:coverage:path": "nyc --check-coverage yarn test:unit:path", diff --git a/test/e2e/fixtures/metrics-enabled/state.json b/test/e2e/fixtures/metrics-enabled/state.json new file mode 100644 index 000000000000..4b0e52e70d38 --- /dev/null +++ b/test/e2e/fixtures/metrics-enabled/state.json @@ -0,0 +1,168 @@ +{ + "data": { + "AppStateController": { + "connectedStatusPopoverHasBeenShown": false, + "swapsWelcomeMessageHasBeenShown": true + }, + "CachedBalancesController": { + "cachedBalances": { + "4": {}, + "1337": { + "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": "0x15af1d78b58c40000" + } + } + }, + "CurrencyController": { + "conversionDate": 1594348502.519, + "conversionRate": 240.09, + "currentCurrency": "usd", + "nativeCurrency": "ETH" + }, + "IncomingTransactionsController": { + "incomingTransactions": {}, + "incomingTxLastFetchedBlocksByNetwork": { + "goerli": null, + "kovan": null, + "mainnet": null, + "rinkeby": 5570536, + "localhost": 98 + } + }, + "KeyringController": { + "vault": "{\"data\":\"s6TpYjlUNsn7ifhEFTkuDGBUM1GyOlPrim7JSjtfIxgTt8/6MiXgiR/CtFfR4dWW2xhq85/NGIBYEeWrZThGdKGarBzeIqBfLFhw9n509jprzJ0zc2Rf+9HVFGLw+xxC4xPxgCS0IIWeAJQ+XtGcHmn0UZXriXm8Ja4kdlow6SWinB7sr/WM3R0+frYs4WgllkwggDf2/Tv6VHygvLnhtzp6hIJFyTjh+l/KnyJTyZW1TkZhDaNDzX3SCOHT\",\"iv\":\"FbeHDAW5afeWNORfNJBR0Q==\",\"salt\":\"TxZ+WbCW6891C9LK/hbMAoUsSEW1E8pyGLVBU6x5KR8=\"}" + }, + "NetworkController": { + "provider": { + "nickname": "Localhost 8545", + "rpcUrl": "http://localhost:8545", + "chainId": "0x539", + "ticker": "ETH", + "type": "rpc" + }, + "network": "1337" + }, + "OnboardingController": { + "onboardingTabs": {}, + "seedPhraseBackedUp": false + }, + "PermissionsMetadata": { + "permissionsLog": [ + { + "id": 1764280960, + "method": "eth_requestAccounts", + "methodType": "restricted", + "origin": "http://127.0.0.1:8080", + "request": { + "method": "eth_requestAccounts", + "jsonrpc": "2.0", + "id": 1764280960, + "origin": "http://127.0.0.1:8080", + "tabId": 2 + }, + "requestTime": 1594348329232, + "response": { + "id": 1764280960, + "jsonrpc": "2.0", + "result": ["0x5cfe73b6021e818b776b421b1c4db2474086a7e1"] + }, + "responseTime": 1594348332276, + "success": true + } + ], + "permissionsHistory": { + "http://127.0.0.1:8080": { + "eth_accounts": { + "accounts": { + "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": 1594348332276 + }, + "lastApproved": 1594348332276 + } + } + }, + "domainMetadata": { + "http://127.0.0.1:8080": { + "name": "E2E Test Dapp", + "icon": "http://127.0.0.1:8080/metamask-fox.svg", + "lastUpdated": 1594348323811, + "host": "127.0.0.1:8080" + } + } + }, + "PreferencesController": { + "frequentRpcListDetail": [], + "accountTokens": { + "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": { + "rinkeby": [], + "ropsten": [] + } + }, + "assetImages": {}, + "tokens": [], + "suggestedTokens": {}, + "useBlockie": false, + "useNonceField": false, + "usePhishDetect": true, + "featureFlags": { + "showIncomingTransactions": true, + "transactionTime": false + }, + "knownMethodData": {}, + "participateInMetaMetrics": true, + "firstTimeFlowType": "create", + "currentLocale": "en", + "identities": { + "0x5cfe73b6021e818b776b421b1c4db2474086a7e1": { + "address": "0x5cfe73b6021e818b776b421b1c4db2474086a7e1", + "name": "Account 1" + } + }, + "lostIdentities": {}, + "forgottenPassword": false, + "preferences": { + "useNativeCurrencyAsPrimaryCurrency": true + }, + "completedOnboarding": true, + "metaMetricsId": "fake-metrics-id", + "metaMetricsSendCount": 0, + "ipfsGateway": "dweb.link", + "selectedAddress": "0x5cfe73b6021e818b776b421b1c4db2474086a7e1" + }, + "config": {}, + "firstTimeInfo": { + "date": 1575697234195, + "version": "7.7.0" + }, + "PermissionsController": { + "permissionsRequests": [], + "permissionsDescriptions": {}, + "domains": { + "http://127.0.0.1:8080": { + "permissions": [ + { + "@context": ["https://github.com/MetaMask/rpc-cap"], + "parentCapability": "eth_accounts", + "id": "f55a1c15-ea48-4088-968e-63be474d42fa", + "date": 1594348332268, + "invoker": "http://127.0.0.1:8080", + "caveats": [ + { + "type": "limitResponseLength", + "value": 1, + "name": "primaryAccountOnly" + }, + { + "type": "filterResponse", + "value": ["0x5cfe73b6021e818b776b421b1c4db2474086a7e1"], + "name": "exposedAccounts" + } + ] + } + ] + } + } + } + }, + "meta": { + "version": 47 + } +} diff --git a/test/e2e/helpers.js b/test/e2e/helpers.js index 9e6aa2f9fb63..de598654639f 100644 --- a/test/e2e/helpers.js +++ b/test/e2e/helpers.js @@ -1,5 +1,9 @@ const path = require('path') +const sinon = require('sinon') const createStaticServer = require('../../development/create-static-server') +const { + createSegmentServer, +} = require('../../development/lib/create-segment-server') const Ganache = require('./ganache') const FixtureServer = require('./fixture-server') const { buildWebDriver } = require('./webdriver') @@ -11,10 +15,19 @@ const largeDelayMs = regularDelayMs * 2 const dappPort = 8080 async function withFixtures(options, testSuite) { - const { dapp, fixtures, ganacheOptions, driverOptions, title } = options + const { + dapp, + fixtures, + ganacheOptions, + driverOptions, + mockSegment, + title, + } = options const fixtureServer = new FixtureServer() const ganacheServer = new Ganache() let dappServer + let segmentServer + let segmentSpy let webDriver try { @@ -38,11 +51,23 @@ async function withFixtures(options, testSuite) { dappServer.on('error', reject) }) } + if (mockSegment) { + segmentSpy = sinon.spy() + segmentServer = createSegmentServer((_request, response, events) => { + for (const event of events) { + segmentSpy(event) + } + response.statusCode = 200 + response.end() + }) + await segmentServer.start(9090) + } const { driver } = await buildWebDriver(driverOptions) webDriver = driver await testSuite({ driver, + segmentSpy, }) if (process.env.SELENIUM_BROWSER === 'chrome') { @@ -57,7 +82,11 @@ async function withFixtures(options, testSuite) { } } catch (error) { if (webDriver) { - await webDriver.verboseReportOnFailure(title) + try { + await webDriver.verboseReportOnFailure(title) + } catch (verboseReportError) { + console.error(verboseReportError) + } } throw error } finally { @@ -76,6 +105,9 @@ async function withFixtures(options, testSuite) { }) }) } + if (segmentServer) { + await segmentServer.stop() + } } } diff --git a/test/e2e/metrics.spec.js b/test/e2e/metrics.spec.js new file mode 100644 index 000000000000..e8598b792a76 --- /dev/null +++ b/test/e2e/metrics.spec.js @@ -0,0 +1,57 @@ +const { strict: assert } = require('assert') +const { By, Key } = require('selenium-webdriver') +const { withFixtures } = require('./helpers') + +/** + * WARNING: These tests must be run using a build created with `yarn build:test:metrics`, so that it has + * the correct Segment host and write keys set. Otherwise this test will fail. + */ +describe('Segment metrics', function () { + this.timeout(0) + + it('should send first three Page metric events upon fullscreen page load', async function () { + const ganacheOptions = { + accounts: [ + { + secretKey: + '0x7C9529A67102755B7E6102D6D950AC5D5863C98713805CEC576B945B15B71EAC', + balance: 25000000000000000000, + }, + ], + } + await withFixtures( + { + fixtures: 'metrics-enabled', + ganacheOptions, + title: this.test.title, + mockSegment: true, + }, + async ({ driver, segmentSpy }) => { + const passwordField = await driver.findElement(By.css('#password')) + await passwordField.sendKeys('correct horse battery staple') + await passwordField.sendKeys(Key.ENTER) + + // find arbitary element to ensure Home page has loaded + await driver.findElement(By.css('[data-testid="eth-overview-send"]')) + + assert.ok(segmentSpy.called, 'Segment should receive metrics') + assert.ok( + segmentSpy.callCount >= 3, + 'At least 3 segment events should be sent', + ) + + const firstSegmentEvent = segmentSpy.getCall(0).args[0] + assert.equal(firstSegmentEvent.name, 'Home') + assert.equal(firstSegmentEvent.context.page.path, '/') + + const secondSegmentEvent = segmentSpy.getCall(1).args[0] + assert.equal(secondSegmentEvent.name, 'Unlock Page') + assert.equal(secondSegmentEvent.context.page.path, '/unlock') + + const thirdSegmentEvent = segmentSpy.getCall(2).args[0] + assert.equal(thirdSegmentEvent.name, 'Home') + assert.equal(thirdSegmentEvent.context.page.path, '/') + }, + ) + }) +})