diff --git a/src/dev/typescript/projects.ts b/src/dev/typescript/projects.ts index 419d4f0854ecc..0244cb2cd9115 100644 --- a/src/dev/typescript/projects.ts +++ b/src/dev/typescript/projects.ts @@ -40,6 +40,22 @@ export const PROJECTS = [ createProject('x-pack/plugins/security_solution/cypress/tsconfig.json', { name: 'security_solution/cypress', }), + createProject( + 'x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json', + { name: 'enterprise_search/shared/cypress' } + ), + createProject( + 'x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/tsconfig.json', + { name: 'enterprise_search/overview/cypress' } + ), + createProject( + 'x-pack/plugins/enterprise_search/public/applications/app_search/cypress/tsconfig.json', + { name: 'enterprise_search/app_search/cypress' } + ), + createProject( + 'x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/tsconfig.json', + { name: 'enterprise_search/workplace_search/cypress' } + ), createProject('x-pack/plugins/osquery/cypress/tsconfig.json', { name: 'osquery/cypress', }), diff --git a/x-pack/plugins/enterprise_search/README.md b/x-pack/plugins/enterprise_search/README.md index ca8ee68c42a34..5c8d767de3099 100644 --- a/x-pack/plugins/enterprise_search/README.md +++ b/x-pack/plugins/enterprise_search/README.md @@ -66,6 +66,89 @@ sh jest.sh public/applications/shared/flash_messages/flash_messages_logic.test.t ### E2E tests +We currently have two testing libraries in which we run E2E tests: + +- [Cypress](#cypress-tests) + - Will contain the majority of our happy path E2E testing +- [Kibana's Functional Test Runner (FTR)](#kibana-ftr-tests) + - Contains basic tests that only run when the Enterprise Search host is not configured + - It's likely we will not continue to expand these tests, and might even trim some over time (to be replaced by Cypress) + +#### Cypress tests + +Documentation: https://docs.cypress.io/ + +Cypress tests can be run directly from the `x-pack/plugins/enterprise_search` folder. You can use our handy cypress.sh script to run specific product test suites: + +```bash +# Basic syntax +sh cypress.sh {run|open} {suite} + +# Examples +sh cypress.sh run overview # run Enterprise Search overview tests +sh cypress.sh open overview # open Enterprise Search overview tests + +sh cypress.sh run as # run App Search tests +sh cypress.sh open as # open App Search tests + +sh cypress.sh run ws # run Workplace Search tests +sh cypress.sh open ws # open Workplace Search tests + +# Overriding env variables +sh cypress.sh open as --env username=enterprise_search password=123 + +# Overriding config settings, e.g. changing the base URL to a dev path, or enabling video recording +sh cypress.sh open as --config baseUrl=http://localhost:5601/xyz video=true + +# Only run a single specific test file +sh cypress.sh run ws --spec '**/example.spec.ts' + +# Opt to run Chrome headlessly +sh cypress.sh run ws --headless +``` + +There are 3 ways you can spin up the required environments to run our Cypress tests: + +1. Running Cypress against local dev environments: + - Elasticsearch: + - Start a local instance, or use Kibana's `yarn es snapshot` command (with all configurations/versions required to run Enterprise Search locally) + - NOTE: We generally recommend a fresh instance (or blowing away your `data/` folder) to reduce false negatives due to custom user data + - Kibana: + - You **must** have `csp.strict: false` and `csp.warnLegacyBrowsers: false` set in your `kibana.dev.yml`. + - You should either start Kibana with `yarn start --no-base-path` or pass `--config baseUrl=http://localhost:5601/xyz` into your Cypress command. + - Enterprise Search: + - Nothing extra is required to run Cypress tests, only what is already needed to run Kibana/Enterprise Search locally. +2. Running Cypress against Kibana's functional test server: + - :information_source: While we won't use the runner, we can still make use of Kibana's functional test server to help us spin up Elasticsearch and Kibana instances. + - NOTE: We recommend stopping any other local dev processes, to reduce issues with memory/performance + - From the `x-pack/` project folder, run `node scripts/functional_tests_server --config test/functional_enterprise_search/cypress.config.ts` + - Kibana: + - You will need to pass `--config baseUrl=http://localhost:5620` into your Cypress command. + - Enterprise Search: + - :warning: TODO: We _currently_ do not have a way of spinning up Enterprise Search from Kibana's FTR - for now, you can use local Enterprise Search (pointed at the FTR's `http://localhost:9220` Elasticsearch host instance) +3. Running Cypress against Enterprise Search dockerized stack scripts + - :warning: This is for Enterprise Search devs only, as this requires access to our closed source Enterprise Search repo + - `stack_scripts/start-with-es-native-auth.sh --with-kibana` + - Note that the tradeoff of an easier one-command start experience is you will not be able to run Cypress tests against any local changes. + +##### Debugging + +Cypress can either run silently in a headless browser in the command line (`run` or `--headless` mode), which is the default mode used by CI, or opened interactively in an included app and the Chrome browser (`open` or `--headed --no-exit` mode). + +For debugging failures locally, we generally recommend using open mode, which allows you to run a single specific test suite, and makes browser dev tools available to you so you can pause and inspect DOM as needed. + +> :warning: Although this is more extra caution than a hard-and-fast rule, we generally recommend taking a break and not clicking or continuing to use the app while tests are running. This can eliminate or lower the possibility of hard-to-reproduce/intermittently flaky behavior and timeouts due to user interference. + +##### Artifacts + +All failed tests will output a screenshot to the `x-pack/plugins/enterprise_search/target/cypress/screenshots` folder. We strongly recommend starting there for debugging failed tests to inspect error messages and UI state at point of failure. + +To track what Cypress is doing while running tests, you can pass in `--config video=true` which will output screencaptures to a `videos/` folder for all tests (both successful and failing). This can potentially provide more context leading up to the failure point, if a static screenshot isn't providing enough information. + +> :information_source: We have videos turned off in our config to reduce test runtime, especially on CI, but suggest re-enabling it for any deep debugging. + +#### Kibana FTR tests + See [our functional test runner README](../../test/functional_enterprise_search). Our automated accessibility tests can be found in [x-pack/test/accessibility/apps](../../test/accessibility/apps/enterprise_search.ts). diff --git a/x-pack/plugins/enterprise_search/cypress.sh b/x-pack/plugins/enterprise_search/cypress.sh new file mode 100644 index 0000000000000..9dbdd81ab788f --- /dev/null +++ b/x-pack/plugins/enterprise_search/cypress.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +# Use either `cypress run` or `cypress open` - defaults to run +MODE="${1:-run}" + +# Choose which product folder to use, e.g. `yarn cypress open as` +PRODUCT="${2}" +# Provide helpful shorthands +if [ "$PRODUCT" == "as" ]; then PRODUCT='app_search'; fi +if [ "$PRODUCT" == "ws" ]; then PRODUCT='workplace_search'; fi +if [ "$PRODUCT" == "overview" ]; then PRODUCT='enterprise_search'; fi + +# Pass all remaining arguments (e.g., ...rest) from the 3rd arg onwards +# as an open-ended string. Appends onto to the end the Cypress command +# @see https://docs.cypress.io/guides/guides/command-line.html#Options +ARGS="${*:3}" + +../../../node_modules/.bin/cypress "$MODE" --project "public/applications/$PRODUCT" --browser chrome $ARGS diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.json b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.json new file mode 100644 index 0000000000000..766aaf6df36ad --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.json @@ -0,0 +1,20 @@ +{ + "supportFile": "./cypress/support/commands.ts", + "pluginsFile": false, + "retries": { + "runMode": 2 + }, + "baseUrl": "http://localhost:5601", + "env": { + "username": "elastic", + "password": "changeme" + }, + "screenshotsFolder": "../../../target/cypress/screenshots", + "videosFolder": "../../../target/cypress/videos", + "defaultCommandTimeout": 120000, + "execTimeout": 120000, + "pageLoadTimeout": 180000, + "viewportWidth": 1600, + "viewportHeight": 1200, + "video": false +} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts new file mode 100644 index 0000000000000..5e651aab075c6 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { login } from '../support/commands'; + +context('Engines', () => { + beforeEach(() => { + login(); + }); + + it('renders', () => { + cy.contains('Engines'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts new file mode 100644 index 0000000000000..50b5fcd179297 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { login as baseLogin } from '../../../shared/cypress/commands'; +import { appSearchPath } from '../../../shared/cypress/routes'; + +interface Login { + path?: string; + username?: string; + password?: string; +} +export const login = ({ path = '/', ...args }: Login = {}) => { + baseLogin({ ...args }); + cy.visit(`${appSearchPath}${path}`); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/tsconfig.json new file mode 100644 index 0000000000000..40361607aa3c5 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../shared/cypress/tsconfig.json", + "references": [{ "path": "../../shared/cypress/tsconfig.json" }], + "compilerOptions": { "outDir": "../../../../target/cypress/types/app_search" }, + "include": ["./**/*"] +} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.json b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.json new file mode 100644 index 0000000000000..8ca8bdfd79a49 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.json @@ -0,0 +1,21 @@ +{ + "supportFile": false, + "pluginsFile": false, + "retries": { + "runMode": 2 + }, + "baseUrl": "http://localhost:5601", + "env": { + "username": "elastic", + "password": "changeme" + }, + "fixturesFolder": false, + "screenshotsFolder": "../../../target/cypress/screenshots", + "videosFolder": "../../../target/cypress/videos", + "defaultCommandTimeout": 120000, + "execTimeout": 120000, + "pageLoadTimeout": 180000, + "viewportWidth": 1600, + "viewportHeight": 1200, + "video": false +} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts new file mode 100644 index 0000000000000..4c9a159a6736f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { login } from '../../../shared/cypress/commands'; +import { overviewPath } from '../../../shared/cypress/routes'; + +context('Enterprise Search Overview', () => { + beforeEach(() => { + login(); + }); + + it('should contain product cards', () => { + cy.visit(overviewPath); + cy.contains('Welcome to Elastic Enterprise Search'); + + cy.get('[data-test-subj="appSearchProductCard"]') + .contains('Open App Search') + .should('have.attr', 'href') + .and('match', /app_search/); + + cy.get('[data-test-subj="workplaceSearchProductCard"]') + .contains('Open Workplace Search') + .should('have.attr', 'href') + .and('match', /workplace_search/); + }); + + it('should have a setup guide', () => { + // @see https://github.com/quasarframework/quasar/issues/2233#issuecomment-492975745 + // This only appears to occur for setup guides - I haven't (yet?) run into it on other pages + cy.on('uncaught:exception', (err) => { + if (err.message.includes('> ResizeObserver loop limit exceeded')) return false; + }); + + cy.visit(`${overviewPath}/setup_guide`); + cy.contains('Setup Guide'); + cy.contains('Add your Enterprise Search host URL to your Kibana configuration'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/tsconfig.json new file mode 100644 index 0000000000000..fd75825bd3e26 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../shared/cypress/tsconfig.json", + "references": [{ "path": "../../shared/cypress/tsconfig.json" }], + "compilerOptions": { "outDir": "../../../../target/cypress/types/enterprise_search" }, + "include": ["./**/*"] +} diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts new file mode 100644 index 0000000000000..5f9738fae5064 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Shared non-product-specific commands + */ + +/* + * Log in a user via XHR + * @see https://docs.cypress.io/guides/getting-started/testing-your-app#Logging-in + */ +interface Login { + username?: string; + password?: string; +} +export const login = ({ + username = Cypress.env('username'), + password = Cypress.env('password'), +}: Login = {}) => { + cy.request({ + method: 'POST', + url: '/internal/security/login', + headers: { 'kbn-xsrf': 'cypress' }, + body: { + providerType: 'basic', + providerName: 'basic', + currentURL: '/', + params: { username, password }, + }, + }); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/routes.ts b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/routes.ts new file mode 100644 index 0000000000000..b1a0aaba95661 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/routes.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const overviewPath = '/app/enterprise_search/overview'; +export const appSearchPath = '/app/enterprise_search/app_search'; +export const workplaceSearchPath = '/app/enterprise_search/workplace_search'; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json new file mode 100644 index 0000000000000..be8ccac5f5e72 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../../../../../tsconfig.base.json", + "include": ["./**/*"], + "compilerOptions": { + "outDir": "../../../../target/cypress/types/shared", + "types": ["cypress", "node"] + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.json b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.json new file mode 100644 index 0000000000000..766aaf6df36ad --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.json @@ -0,0 +1,20 @@ +{ + "supportFile": "./cypress/support/commands.ts", + "pluginsFile": false, + "retries": { + "runMode": 2 + }, + "baseUrl": "http://localhost:5601", + "env": { + "username": "elastic", + "password": "changeme" + }, + "screenshotsFolder": "../../../target/cypress/screenshots", + "videosFolder": "../../../target/cypress/videos", + "defaultCommandTimeout": 120000, + "execTimeout": 120000, + "pageLoadTimeout": 180000, + "viewportWidth": 1600, + "viewportHeight": 1200, + "video": false +} diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts new file mode 100644 index 0000000000000..8ce6e4ebcfb05 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { login } from '../support/commands'; + +context('Overview', () => { + beforeEach(() => { + login(); + }); + + it('renders', () => { + cy.contains('Workplace Search'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts new file mode 100644 index 0000000000000..d91b73fd78c05 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { login as baseLogin } from '../../../shared/cypress/commands'; +import { workplaceSearchPath } from '../../../shared/cypress/routes'; + +interface Login { + path?: string; + username?: string; + password?: string; +} +export const login = ({ path = '/', ...args }: Login = {}) => { + baseLogin({ ...args }); + cy.visit(`${workplaceSearchPath}${path}`); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/tsconfig.json new file mode 100644 index 0000000000000..39f8bea5debc6 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../shared/cypress/tsconfig.json", + "references": [{ "path": "../../shared/cypress/tsconfig.json" }], + "compilerOptions": { "outDir": "../../../../target/cypress/types/workplace_search" }, + "include": ["./**/*"] +} diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index 481c4527d5977..ce288f8b4b97d 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -6,6 +6,7 @@ "declaration": true, "declarationMap": true }, + "exclude": ["public/applications/**/cypress/**/*"], "include": [ "common/**/*", "public/**/*", diff --git a/x-pack/test/functional_enterprise_search/cypress.config.ts b/x-pack/test/functional_enterprise_search/cypress.config.ts new file mode 100644 index 0000000000000..9a6918ab0557d --- /dev/null +++ b/x-pack/test/functional_enterprise_search/cypress.config.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +// TODO: If Kibana CI doesn't end up using this (e.g., uses Dockerized containers +// instead of the functional test server), we can opt to delete this file later. + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const baseConfig = await readConfigFile(require.resolve('./base_config')); + + return { + // default to the xpack functional config + ...baseConfig.getAll(), + + esTestCluster: { + ...baseConfig.get('esTestCluster'), + serverArgs: [ + ...baseConfig.get('esTestCluster.serverArgs'), + 'xpack.security.enabled=true', + 'xpack.security.authc.api_key.enabled=true', + ], + }, + + kbnTestServer: { + ...baseConfig.get('kbnTestServer'), + serverArgs: [ + ...baseConfig.get('kbnTestServer.serverArgs'), + '--csp.strict=false', + '--csp.warnLegacyBrowsers=false', + '--enterpriseSearch.host=http://localhost:3002', + ], + }, + }; +}