[buildkite] Add uptime playwright tests to PR pipeline

.buildkite/pipelines/pull_request/uptime.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/uptime.sh
+    label: 'Uptime @elastic/synthetics Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -73,6 +73,10 @@ const uploadPipeline = (pipelineContent) => {
     //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     // }
 
+    if (await doAnyChangesMatch([/^x-pack\/plugins\/uptime/])) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/uptime.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     uploadPipeline(pipeline.join('\n'));

.buildkite/scripts/steps/functional/common.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+# Note, changes here might also need to be made in other scripts, e.g. uptime.sh
+
 source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh

.buildkite/scripts/steps/functional/uptime.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export JOB=kibana-uptime-playwright
+
+echo "--- Uptime @elastic/synthetics Tests"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Uptime @elastic/synthetics Tests" \
+  node plugins/uptime/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION"

x-pack/plugins/uptime/e2e/config.ts

@@ -39,7 +39,7 @@ async function config({ readConfigFile }: FtrConfigProviderContext) {
         '--csp.warnLegacyBrowsers=false',
         // define custom kibana server args here
         `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
-        `--elasticsearch.ignoreVersionMismatch=true`,
+        `--elasticsearch.ignoreVersionMismatch=${process.env.CI ? 'false' : 'true'}`,
         `--uiSettings.overrides.theme:darkMode=true`,
         `--elasticsearch.username=kibana_system`,
         `--elasticsearch.password=changeme`,

x-pack/plugins/uptime/e2e/playwright_start.ts

@@ -15,15 +15,10 @@ import './journeys';
 
 export function playwrightRunTests() {
   return async ({ getService }: any) => {
-    try {
-      const result = await playwrightStart(getService);
-
-      if (result && result.uptime.status !== 'succeeded') {
-        process.exit(1);
-      }
-    } catch (error) {
-      console.error('errors: ', error);
-      process.exit(1);
+    const result = await playwrightStart(getService);
+
+    if (result && result.uptime.status !== 'succeeded') {
+      throw new Error('Tests failed');
     }
   };
 }
@@ -42,7 +37,7 @@ async function playwrightStart(getService: any) {
 
   const res = await playwrightRun({
     params: { kibanaUrl },
-    playwrightOptions: { chromiumSandbox: false, timeout: 60 * 1000 },
+    playwrightOptions: { headless: true, chromiumSandbox: false, timeout: 60 * 1000 },
   });
 
   console.log('Removing esArchiver...');

x-pack/plugins/uptime/scripts/e2e.js

@@ -28,9 +28,14 @@ const { argv } = yargs(process.argv.slice(2))
     type: 'boolean',
     description: 'Opens the Playwright Test Runner',
   })
+  .option('kibana-install-dir', {
+    default: '',
+    type: 'string',
+    description: 'Path to the Kibana install directory',
+  })
   .help();
 
-const { server, runner, open } = argv;
+const { server, runner, open, kibanaInstallDir } = argv;
 
 const e2eDir = path.join(__dirname, '../e2e');
 
@@ -44,9 +49,12 @@ if (server) {
 const config = './playwright_run.ts';
 
 function executeRunner() {
-  childProcess.execSync(`node ../../../scripts/${ftrScript} --config ${config}`, {
-    cwd: e2eDir,
-    stdio: 'inherit',
-  });
+  childProcess.execSync(
+    `node ../../../scripts/${ftrScript} --config ${config} --kibana-install-dir '${kibanaInstallDir}'`,
+    {
+      cwd: e2eDir,
+      stdio: 'inherit',
+    }
+  );
 }
 executeRunner();