[Enterprise Search] Set up basic scaffolding for Cypress tests in Kibana

src/dev/typescript/projects.ts

@@ -40,6 +40,22 @@ export const PROJECTS = [
   createProject('x-pack/plugins/security_solution/cypress/tsconfig.json', {
     name: 'security_solution/cypress',
   }),
+  createProject(
+    'x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json',
+    { name: 'enterprise_search/shared/cypress' }
+  ),
+  createProject(
+    'x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/tsconfig.json',
+    { name: 'enterprise_search/overview/cypress' }
+  ),
+  createProject(
+    'x-pack/plugins/enterprise_search/public/applications/app_search/cypress/tsconfig.json',
+    { name: 'enterprise_search/app_search/cypress' }
+  ),
+  createProject(
+    'x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/tsconfig.json',
+    { name: 'enterprise_search/workplace_search/cypress' }
+  ),
   createProject('x-pack/plugins/osquery/cypress/tsconfig.json', {
     name: 'osquery/cypress',
   }),

x-pack/plugins/enterprise_search/README.md

@@ -66,6 +66,89 @@ sh jest.sh public/applications/shared/flash_messages/flash_messages_logic.test.t
 
 ### E2E tests
 
+We currently have two testing libraries in which we run E2E tests:
+
+- [Cypress](#cypress-tests)
+  - Will contain the majority of our happy path E2E testing
+- [Kibana's Functional Test Runner (FTR)](#kibana-ftr-tests)
+  - Contains basic tests that only run when the Enterprise Search host is not configured
+  - It's likely we will not continue to expand these tests, and might even trim some over time (to be replaced by Cypress)
+
+#### Cypress tests
+
+Documentation: https://docs.cypress.io/
+
+Cypress tests can be run directly from the `x-pack/plugins/enterprise_search` folder. You can use our handy cypress.sh script to run specific product test suites:
+
+```bash
+# Basic syntax
+sh cypress.sh {run|open} {suite}
+
+# Examples
+sh cypress.sh run overview   # run Enterprise Search overview tests
+sh cypress.sh open overview  # open Enterprise Search overview tests
+
+sh cypress.sh run as         # run App Search tests
+sh cypress.sh open as        # open App Search tests
+
+sh cypress.sh run ws         # run Workplace Search tests
+sh cypress.sh open ws        # open Workplace Search tests
+
+# Overriding env variables
+sh cypress.sh open as --env username=enterprise_search password=123
+
+# Overriding config settings, e.g. changing the base URL to a dev path, or enabling video recording
+sh cypress.sh open as --config baseUrl=http://localhost:5601/xyz video=true
+
+# Only run a single specific test file
+sh cypress.sh run ws --spec '**/example.spec.ts'
+
+# Opt to run Chrome headlessly
+sh cypress.sh run ws --headless
+```
+
+There are 3 ways you can spin up the required environments to run our Cypress tests:
+
+1. Running Cypress against local dev environments:
+   - Elasticsearch:
+     - Start a local instance, or use Kibana's `yarn es snapshot` command (with all configurations/versions required to run Enterprise Search locally)
+     - NOTE: We generally recommend a fresh instance (or blowing away your `data/` folder) to reduce false negatives due to custom user data
+   - Kibana:
+     - You **must** have `csp.strict: false` and `csp.warnLegacyBrowsers: false` set in your `kibana.dev.yml`.
+     - You should either start Kibana with `yarn start --no-base-path` or pass `--config baseUrl=http://localhost:5601/xyz` into your Cypress command.
+   - Enterprise Search:
+     - Nothing extra is required to run Cypress tests, only what is already needed to run Kibana/Enterprise Search locally.
+2. Running Cypress against Kibana's functional test server:
+   - :information_source: While we won't use the runner, we can still make use of Kibana's functional test server to help us spin up Elasticsearch and Kibana instances.
+     - NOTE: We recommend stopping any other local dev processes, to reduce issues with memory/performance
+   - From the `x-pack/` project folder, run `node scripts/functional_tests_server --config test/functional_enterprise_search/cypress.config.ts`
+   - Kibana:
+     - You will need to pass `--config baseUrl=http://localhost:5620` into your Cypress command.
+   - Enterprise Search:
+     - :warning: TODO: We _currently_ do not have a way of spinning up Enterprise Search from Kibana's FTR - for now, you can use local Enterprise Search (pointed at the FTR's `http://localhost:9220` Elasticsearch host instance)
+3. Running Cypress against Enterprise Search dockerized stack scripts
+   - :warning: This is for Enterprise Search devs only, as this requires access to our closed source Enterprise Search repo
+   - `stack_scripts/start-with-es-native-auth.sh --with-kibana`
+   - Note that the tradeoff of an easier one-command start experience is you will not be able to run Cypress tests against any local changes.
+
+##### Debugging
+
+Cypress can either run silently in a headless browser in the command line (`run` or `--headless` mode), which is the default mode used by CI, or opened interactively in an included app and the Chrome browser (`open` or `--headed --no-exit` mode).
+
+For debugging failures locally, we generally recommend using open mode, which allows you to run a single specific test suite, and makes browser dev tools available to you so you can pause and inspect DOM as needed.
+
+> :warning: Although this is more extra caution than a hard-and-fast rule, we generally recommend taking a break and not clicking or continuing to use the app while tests are running. This can eliminate or lower the possibility of hard-to-reproduce/intermittently flaky behavior and timeouts due to user interference.
+
+##### Artifacts
+
+All failed tests will output a screenshot to the `x-pack/plugins/enterprise_search/target/cypress/screenshots` folder. We strongly recommend starting there for debugging failed tests to inspect error messages and UI state at point of failure.
+
+To track what Cypress is doing while running tests, you can pass in `--config video=true` which will output screencaptures to a `videos/` folder for all tests (both successful and failing). This can potentially provide more context leading up to the failure point, if a static screenshot isn't providing enough information.
+
+> :information_source: We have videos turned off in our config to reduce test runtime, especially on CI, but suggest re-enabling it for any deep debugging.
+
+#### Kibana FTR tests
+
 See [our functional test runner README](../../test/functional_enterprise_search).
 
 Our automated accessibility tests can be found in [x-pack/test/accessibility/apps](../../test/accessibility/apps/enterprise_search.ts).

x-pack/plugins/enterprise_search/cypress.sh

@@ -0,0 +1,18 @@
+#! /bin/bash
+
+# Use either `cypress run` or `cypress open` - defaults to run
+MODE="${1:-run}"
+
+# Choose which product folder to use, e.g. `yarn cypress open as`
+PRODUCT="${2}"
+# Provide helpful shorthands
+if [ "$PRODUCT" == "as" ]; then PRODUCT='app_search'; fi
+if [ "$PRODUCT" == "ws" ]; then PRODUCT='workplace_search'; fi
+if [ "$PRODUCT" == "overview" ]; then PRODUCT='enterprise_search'; fi
+
+# Pass all remaining arguments (e.g., ...rest) from the 3rd arg onwards
+# as an open-ended string. Appends onto to the end the Cypress command
+# @see https://docs.cypress.io/guides/guides/command-line.html#Options
+ARGS="${*:3}"
+
+../../../node_modules/.bin/cypress "$MODE" --project "public/applications/$PRODUCT" --browser chrome $ARGS

x-pack/plugins/enterprise_search/public/applications/app_search/cypress.json

@@ -0,0 +1,20 @@
+{
+  "supportFile": "./cypress/support/commands.ts",
+  "pluginsFile": false,
+  "retries": {
+    "runMode": 2
+  },
+  "baseUrl": "http://localhost:5601",
+  "env": {
+    "username": "elastic",
+    "password": "changeme"
+  },
+  "screenshotsFolder": "../../../target/cypress/screenshots",
+  "videosFolder": "../../../target/cypress/videos",
+  "defaultCommandTimeout": 120000,
+  "execTimeout": 120000,
+  "pageLoadTimeout": 180000,
+  "viewportWidth": 1600,
+  "viewportHeight": 1200,
+  "video": false
+}

x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { login } from '../support/commands';
+
+context('Engines', () => {
+  beforeEach(() => {
+    login();
+  });
+
+  it('renders', () => {
+    cy.contains('Engines');
+  });
+});

x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { login as baseLogin } from '../../../shared/cypress/commands';
+import { appSearchPath } from '../../../shared/cypress/routes';
+
+interface Login {
+  path?: string;
+  username?: string;
+  password?: string;
+}
+export const login = ({ path = '/', ...args }: Login = {}) => {
+  baseLogin({ ...args });
+  cy.visit(`${appSearchPath}${path}`);
+};

x-pack/plugins/enterprise_search/public/applications/app_search/cypress/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../shared/cypress/tsconfig.json",
+  "references": [{ "path": "../../shared/cypress/tsconfig.json" }],
+  "compilerOptions": { "outDir": "../../../../target/cypress/types/app_search" },
+  "include": ["./**/*"]
+}

x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.json

@@ -0,0 +1,21 @@
+{
+  "supportFile": false,
+  "pluginsFile": false,
+  "retries": {
+    "runMode": 2
+  },
+  "baseUrl": "http://localhost:5601",
+  "env": {
+    "username": "elastic",
+    "password": "changeme"
+  },
+  "fixturesFolder": false,
+  "screenshotsFolder": "../../../target/cypress/screenshots",
+  "videosFolder": "../../../target/cypress/videos",
+  "defaultCommandTimeout": 120000,
+  "execTimeout": 120000,
+  "pageLoadTimeout": 180000,
+  "viewportWidth": 1600,
+  "viewportHeight": 1200,
+  "video": false
+}

x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { login } from '../../../shared/cypress/commands';
+import { overviewPath } from '../../../shared/cypress/routes';
+
+context('Enterprise Search Overview', () => {
+  beforeEach(() => {
+    login();
+  });
+
+  it('should contain product cards', () => {
+    cy.visit(overviewPath);
+    cy.contains('Welcome to Elastic Enterprise Search');
+
+    cy.get('[data-test-subj="appSearchProductCard"]')
+      .contains('Open App Search')
+      .should('have.attr', 'href')
+      .and('match', /app_search/);
+
+    cy.get('[data-test-subj="workplaceSearchProductCard"]')
+      .contains('Open Workplace Search')
+      .should('have.attr', 'href')
+      .and('match', /workplace_search/);
+  });
+
+  it('should have a setup guide', () => {
+    // @see https://github.com/quasarframework/quasar/issues/2233#issuecomment-492975745
+    // This only appears to occur for setup guides - I haven't (yet?) run into it on other pages
+    cy.on('uncaught:exception', (err) => {
+      if (err.message.includes('> ResizeObserver loop limit exceeded')) return false;
+    });
+
+    cy.visit(`${overviewPath}/setup_guide`);
+    cy.contains('Setup Guide');
+    cy.contains('Add your Enterprise Search host URL to your Kibana configuration');
+  });
+});

x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../shared/cypress/tsconfig.json",
+  "references": [{ "path": "../../shared/cypress/tsconfig.json" }],
+  "compilerOptions": { "outDir": "../../../../target/cypress/types/enterprise_search" },
+  "include": ["./**/*"]
+}

x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * Shared non-product-specific commands
+ */
+
+/*
+ * Log in a user via XHR
+ * @see https://docs.cypress.io/guides/getting-started/testing-your-app#Logging-in
+ */
+interface Login {
+  username?: string;
+  password?: string;
+}
+export const login = ({
+  username = Cypress.env('username'),
+  password = Cypress.env('password'),
+}: Login = {}) => {
+  cy.request({
+    method: 'POST',
+    url: '/internal/security/login',
+    headers: { 'kbn-xsrf': 'cypress' },
+    body: {
+      providerType: 'basic',
+      providerName: 'basic',
+      currentURL: '/',
+      params: { username, password },
+    },
+  });
+};

x-pack/plugins/enterprise_search/public/applications/shared/cypress/routes.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const overviewPath = '/app/enterprise_search/overview';
+export const appSearchPath = '/app/enterprise_search/app_search';
+export const workplaceSearchPath = '/app/enterprise_search/workplace_search';

x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../../../../../../tsconfig.base.json",
+  "include": ["./**/*"],
+  "compilerOptions": {
+    "outDir": "../../../../target/cypress/types/shared",
+    "types": ["cypress", "node"]
+  }
+}

x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.json

@@ -0,0 +1,20 @@
+{
+  "supportFile": "./cypress/support/commands.ts",
+  "pluginsFile": false,
+  "retries": {
+    "runMode": 2
+  },
+  "baseUrl": "http://localhost:5601",
+  "env": {
+    "username": "elastic",
+    "password": "changeme"
+  },
+  "screenshotsFolder": "../../../target/cypress/screenshots",
+  "videosFolder": "../../../target/cypress/videos",
+  "defaultCommandTimeout": 120000,
+  "execTimeout": 120000,
+  "pageLoadTimeout": 180000,
+  "viewportWidth": 1600,
+  "viewportHeight": 1200,
+  "video": false
+}

x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { login } from '../support/commands';
+
+context('Overview', () => {
+  beforeEach(() => {
+    login();
+  });
+
+  it('renders', () => {
+    cy.contains('Workplace Search');
+  });
+});

x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { login as baseLogin } from '../../../shared/cypress/commands';
+import { workplaceSearchPath } from '../../../shared/cypress/routes';
+
+interface Login {
+  path?: string;
+  username?: string;
+  password?: string;
+}
+export const login = ({ path = '/', ...args }: Login = {}) => {
+  baseLogin({ ...args });
+  cy.visit(`${workplaceSearchPath}${path}`);
+};

x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../shared/cypress/tsconfig.json",
+  "references": [{ "path": "../../shared/cypress/tsconfig.json" }],
+  "compilerOptions": { "outDir": "../../../../target/cypress/types/workplace_search" },
+  "include": ["./**/*"]
+}