Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Synthetics/Uptime] Separate e2e tests by app #152197

Merged
merged 10 commits into from
Jun 7, 2023
Merged
3 changes: 2 additions & 1 deletion .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ disabled:

# Elastic Synthetics configs
- x-pack/plugins/synthetics/e2e/config.ts
- x-pack/plugins/synthetics/e2e/synthetics_run.ts
- x-pack/plugins/synthetics/e2e/uptime/synthetics_run.ts
- x-pack/plugins/synthetics/e2e/synthetics/synthetics_run.ts
- x-pack/plugins/ux/e2e/synthetics_run.ts
- x-pack/plugins/exploratory_view/e2e/synthetics_run.ts

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ steps:
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/exploratory_view/e2e/.journeys/**/*'
retry:
Expand Down
4 changes: 2 additions & 2 deletions .buildkite/pipelines/pull_request/synthetics_plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ steps:
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
- 'x-pack/plugins/synthetics/e2e/synthetics/.journeys/**/*'
shahzad31 marked this conversation as resolved.
Show resolved Hide resolved
retry:
automatic:
- exit_status: '-1'
Expand Down
15 changes: 15 additions & 0 deletions .buildkite/pipelines/pull_request/uptime_plugin.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
steps:
- command: .buildkite/scripts/steps/functional/uptime_plugin.sh
label: 'Uptime @elastic/synthetics Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/synthetics/e2e/uptime/.journeys/**/*'
shahzad31 marked this conversation as resolved.
Show resolved Hide resolved
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
2 changes: 1 addition & 1 deletion .buildkite/pipelines/pull_request/ux_plugin_e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ steps:
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/ux/e2e/.journeys/**/*'
retry:
Expand Down
1 change: 1 addition & 0 deletions .buildkite/scripts/pipelines/pull_request/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ const uploadPipeline = (pipelineContent: string | object) => {
])
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/synthetics_plugin.yml'));
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/uptime_plugin.yml'));
}

if (await doAnyChangesMatch([/^x-pack\/plugins\/ux/, /^x-pack\/plugins\/exploratory_view/])) {
Expand Down
16 changes: 16 additions & 0 deletions .buildkite/scripts/steps/functional/uptime_plugin.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/common/util.sh

.buildkite/scripts/bootstrap.sh
.buildkite/scripts/download_build_artifacts.sh

export JOB=kibana-uptime-plugin

echo "--- Uptime plugin @elastic/synthetics Tests"

cd "$XPACK_DIR"

node plugins/synthetics/scripts/uptime_e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the grep portion used for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's primarily used to filter out tests by journey name

3 changes: 0 additions & 3 deletions packages/kbn-ts-projects/ts_projects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@ export const TS_PROJECTS = TsProject.loadAll({
// type check is explicitly disabled in this project for now
'.buildkite/tsconfig.json',

'x-pack/plugins/synthetics/e2e/tsconfig.json',
'x-pack/plugins/ux/e2e/tsconfig.json',
'x-pack/plugins/exploratory_view/e2e/tsconfig.json',
'x-pack/plugins/threat_intelligence/cypress/tsconfig.json',
],
});
5 changes: 5 additions & 0 deletions x-pack/plugins/exploratory_view/e2e/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,9 @@
"outDir": "target/types",
"types": [ "node"],
},
"kbn_references": [
"@kbn/test",
"@kbn/apm-plugin",
"@kbn/es-archiver",
],
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ export const dataTypes: Array<{ id: AppDataType; label: string }> = [
id: DataTypes.SYNTHETICS,
label: DataTypesLabels[DataTypes.SYNTHETICS],
},
{
id: DataTypes.UPTIME,
label: DataTypesLabels[DataTypes.UPTIME],
},
{
id: DataTypes.UX,
label: DataTypesLabels[DataTypes.UX],
Expand Down
19 changes: 18 additions & 1 deletion x-pack/plugins/synthetics/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,23 @@ with an example run command when it finishes.

### Run the tests

From this directory, `~/x-pack/plugins/synthetics/e2e`, you can now run `node ../../../../scripts/functional_test_runner --config synthetics_run.ts`.
From the same directory you can now run `node node e2e.js --runner`.

In addition to the usual flags like `--grep`, you can also specify `--no-headless` in order to view your tests as you debug/develop.


## Uptime App Tests

These tests rely on the Kibana functional test runner. There is a Kibana config in this directory, and a dedicated
script for standing up the test server.

### Start the server

From `~/x-pack/plugins/synthetics/scripts`, run `node uptime_e2e.js --server`. Wait for the server to startup. It will provide you
with an example run command when it finishes.

### Run the tests

From the same directory you can now run `node node uptime_e2e.js --runner`.

In addition to the usual flags like `--grep`, you can also specify `--no-headless` in order to view your tests as you debug/develop.
7 changes: 2 additions & 5 deletions x-pack/plugins/synthetics/e2e/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,11 @@

import { FtrConfigProviderContext } from '@kbn/test';
import { CA_CERT_PATH } from '@kbn/dev-utils';
import { argv } from './helpers/parse_args_params';
import { readKibanaConfig } from './tasks/read_kibana_config';
const MANIFEST_KEY = 'xpack.uptime.service.manifestUrl';
const SERVICE_PASSWORD = 'xpack.uptime.service.password';
const SERVICE_USERNAME = 'xpack.uptime.service.username';

const { watch } = argv;

async function config({ readConfigFile }: FtrConfigProviderContext) {
const kibanaCommonTestsConfig = await readConfigFile(
require.resolve('../../../../test/common/config.js')
Expand Down Expand Up @@ -44,7 +41,7 @@ async function config({ readConfigFile }: FtrConfigProviderContext) {

kbnTestServer: {
...xpackFunctionalTestsConfig.get('kbnTestServer'),
sourceArgs: watch
sourceArgs: process.env.WATCH_ENABLED
? []
: [...xpackFunctionalTestsConfig.get('kbnTestServer.sourceArgs'), '--no-watch'],
serverArgs: [
Expand All @@ -54,7 +51,7 @@ async function config({ readConfigFile }: FtrConfigProviderContext) {
'--csp.warnLegacyBrowsers=false',
// define custom kibana server args here
`--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
`--elasticsearch.ignoreVersionMismatch=${process.env.CI ? 'false' : 'true'}`,
// `--elasticsearch.ignoreVersionMismatch=${process.env.CI ? 'false' : 'true'}`,
`--elasticsearch.username=kibana_system`,
`--elasticsearch.password=changeme`,
'--xpack.reporting.enabled=false',
Expand Down
5 changes: 0 additions & 5 deletions x-pack/plugins/synthetics/e2e/helpers/parse_args_params.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,6 @@ const { argv } = yargs(process.argv.slice(2))
type: 'boolean',
description: 'Pause on error',
})
.option('watch', {
default: false,
type: 'boolean',
description: 'Runs the server in watch mode, restarting on changes',
})
.option('grep', {
default: undefined,
type: 'string',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { v4 as uuidv4 } from 'uuid';
import { journey, step, expect, Page } from '@elastic/synthetics';
import { recordVideo } from '../../helpers/record_video';
import { FormMonitorType } from '../../../common/runtime_types';
import { syntheticsAppPageProvider } from '../../page_objects/synthetics/synthetics_app';
import { syntheticsAppPageProvider } from '../page_objects/synthetics_app';

const customLocation = process.env.SYNTHETICS_TEST_LOCATION;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { RetryService } from '@kbn/ftr-common-functional-services';
import { v4 as uuidv4 } from 'uuid';
import { recordVideo } from '../../../helpers/record_video';
import { getReasonMessage } from '../../../../server/legacy_uptime/lib/alerts/status_check';
import { syntheticsAppPageProvider } from '../../../page_objects/synthetics/synthetics_app';
import { syntheticsAppPageProvider } from '../../page_objects/synthetics_app';
import { SyntheticsServices } from '../services/synthetics_services';

journey(`DefaultStatusAlert`, async ({ page, params }) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,159 +258,3 @@ export const firstDownHit = ({
dataset: 'http',
},
});

const sampleHits = {
took: 1,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 3,
relation: 'eq',
},
max_score: null,
hits: [
firstUpHit,
{
_index: '.ds-synthetics-http-default-2022.12.18-000001',
_id: 'TfCjJIUB1hhCUz-n3bWY',
_score: null,
_ignored: ['http.response.body.content'],
_source: firstDownHit,
sort: [1671356989976],
},
{
_index: '.ds-synthetics-http-default-2022.12.18-000001',
_id: 'fPCiJIUB1hhCUz-nVbMK',
_score: null,
_source: {
summary: {
up: 1,
down: 0,
},
tcp: {
rtt: {
connect: {
us: 26843,
},
},
},
agent: {
name: 'docker-fleet-server',
id: 'dd39a87d-a1e5-45a1-8dd9-e78d6a1391c6',
ephemeral_id: '264bb432-93f6-4aa6-a14d-266c53b9e7c7',
type: 'heartbeat',
version: '8.7.0',
},
resolve: {
rtt: {
us: 18153,
},
ip: '142.250.181.196',
},
elastic_agent: {
id: 'dd39a87d-a1e5-45a1-8dd9-e78d6a1391c6',
version: '8.7.0',
snapshot: true,
},
monitor: {
duration: {
us: 179022,
},
ip: '142.250.181.196',
origin: 'ui',
name: 'Test Monitor',
id: 'b9d9e146-746f-427f-bbf5-6e786b5b4e73',
timespan: {
lt: '2022-12-18T09:51:09.403Z',
gte: '2022-12-18T09:48:09.403Z',
},
check_group: '144186b2-7eb9-11ed-8949-0242ac120006',
fleet_managed: true,
type: 'http',
status: 'up',
},
url: {
scheme: 'https',
port: 443,
domain: 'www.google.com',
full: 'https://www.google.com',
},
observer: {
geo: {
name: 'Test private location',
},
name: 'Test private location',
},
'@timestamp': '2022-12-18T09:48:09.224Z',
ecs: {
version: '8.0.0',
},
config_id: 'b9d9e146-746f-427f-bbf5-6e786b5b4e73',
data_stream: {
namespace: 'default',
type: 'synthetics',
dataset: 'http',
},
tls: {
cipher: 'TLS-AES-128-GCM-SHA256',
certificate_not_valid_before: '2022-11-28T08:19:01.000Z',
established: true,
server: {
x509: {
not_after: '2023-02-20T08:19:00.000Z',
subject: {
distinguished_name: 'CN=www.google.com',
common_name: 'www.google.com',
},
not_before: '2022-11-28T08:19:01.000Z',
public_key_curve: 'P-256',
public_key_algorithm: 'ECDSA',
signature_algorithm: 'SHA256-RSA',
serial_number: '173037077033925240295268439311466214245',
issuer: {
distinguished_name: 'CN=GTS CA 1C3,O=Google Trust Services LLC,C=US',
common_name: 'GTS CA 1C3',
},
},
hash: {
sha1: 'ea1b44061b864526c45619230b3299117d11bf4e',
sha256: 'a5686448de09cc82b9cdad1e96357f919552ab14244da7948dd412ec0fc37d2b',
},
},
rtt: {
handshake: {
us: 34230,
},
},
version: '1.3',
certificate_not_valid_after: '2023-02-20T08:19:00.000Z',
version_protocol: 'tls',
},
state: {
duration_ms: 0,
checks: 1,
ends: null,
started_at: '2022-12-18T09:48:15.335017193Z',
id: 'Test private location-18524a25067-0',
up: 1,
down: 0,
flap_history: [],
status: 'up',
},
event: {
agent_id_status: 'verified',
ingested: '2022-12-18T09:48:16Z',
dataset: 'http',
},
},
sort: [1671356889224],
},
],
},
};
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
*/

import { journey, step, expect, before, after } from '@elastic/synthetics';
import { byTestId } from '../../helpers/utils';
import { recordVideo } from '../../helpers/record_video';
import { syntheticsAppPageProvider } from '../../page_objects/synthetics/synthetics_app';
import { byTestId } from '../../helpers/utils';
import { syntheticsAppPageProvider } from '../page_objects/synthetics_app';
import { cleanSettings } from './services/settings';

journey('AlertingDefaults', async ({ page, params }) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

import { journey, step, expect, Page } from '@elastic/synthetics';
import { RetryService } from '@kbn/ftr-common-functional-services';
import { byTestId, assertText } from '../../helpers/utils';
import { recordVideo } from '../../helpers/record_video';
import { syntheticsAppPageProvider } from '../../page_objects/synthetics/synthetics_app';
import { syntheticsAppPageProvider } from '../page_objects/synthetics_app';
import { byTestId, assertText } from '../../helpers/utils';

let page1: Page;
journey(`DataRetentionPage`, async ({ page, params }) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import { expect, journey, step } from '@elastic/synthetics';
import { recordVideo } from '../../helpers/record_video';
import { syntheticsAppPageProvider } from '../../page_objects/synthetics/synthetics_app';
import { syntheticsAppPageProvider } from '../page_objects/synthetics_app';

journey('TestMonitorDetailFlyout', async ({ page, params }) => {
recordVideo(page);
Expand Down
Loading