Merge branch 'main' of github.com:elastic/kibana into analyzer-full-s…

…creen-new-eui

.bazelrc.common

@@ -132,6 +132,9 @@ coverage --instrument_test_targets
 # Metadata settings
 build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 
+# Load remote cache settings, if they exist
+try-import %workspace%/.bazelrc.cache
+
 # Load any settings specific to the current user.
 # .bazelrc.user should appear in .gitignore so that settings are not shared with team members
 # This needs to be last statement in this

.buildkite/pipelines/performance/daily.yml

@@ -0,0 +1,33 @@
+steps:
+  - block: ":gear: Performance Tests Configuration"
+    prompt: "Fill out the details for performance test"
+    fields:
+      - text: ":arrows_counterclockwise: Iterations"
+        key: "performance-test-iteration-count"
+        hint: "How many times you want to run tests? "
+        required: true
+    if: build.env('PERF_TEST_COUNT') == null
+
+  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+
+  - wait
+
+  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+
+  - label: ":muscle: Performance Tests with Playwright config"
+    command: .buildkite/scripts/steps/functional/performance_playwright.sh
+    agents:
+      queue: c2-16
+    depends_on: build
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":male_superhero::skin-tone-2: Post-Build"
+    command: .buildkite/scripts/lifecycle/post_build.sh
+

.buildkite/pipelines/pull_request/base.yml

@@ -166,3 +166,10 @@ steps:
       queue: c2-4
     key: storybooks
     timeout_in_minutes: 60
+
+  - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
+    label: 'Build Webpack Bundle Analyzer reports'
+    agents:
+      queue: n2-2
+    key: webpack_bundle_analyzer
+    timeout_in_minutes: 60

.buildkite/pull_requests.json

@@ -16,7 +16,7 @@
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "skip_ci_labels": ["skip-ci", "jenkins-ci"],
-      "skip_target_branches": ["6.8"]
+      "skip_target_branches": ["6.8", "7.11", "7.12"]
     }
   ]
 }

.buildkite/scripts/common/env.sh

@@ -22,6 +22,9 @@ KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_BRANCH
 export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 
+KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
+export KIBANA_PKG_VERSION
+
 export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
 export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
 export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
@@ -95,12 +98,3 @@ fi
 
 export BUILD_TS_REFS_DISABLE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true
-
-export TEST_KIBANA_HOST=localhost
-export TEST_KIBANA_PORT=6101
-export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
-export TEST_ES_URL="http://elastic:changeme@localhost:6102"
-export TEST_ES_TRANSPORT_PORT=6301-6309
-export TEST_CORS_SERVER_PORT=6106
-export ALERTING_PROXY_PORT=6105
-export TEST_PROXY_SERVER_PORT=6107

.buildkite/scripts/lifecycle/post_command.sh

@@ -9,8 +9,9 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_
 if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   echo "--- Upload Artifacts"
   buildkite-agent artifact upload 'target/junit/**/*'
-  buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*'
+  buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
+  buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'

.buildkite/scripts/lifecycle/pre_command.sh

@@ -92,6 +92,24 @@ export KIBANA_DOCKER_USERNAME
 KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
 export KIBANA_DOCKER_PASSWORD
 
+SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_USERNAME
+
+SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_PASSWORD
+
+SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_MANIFEST
+
+SYNTHETICS_REMOTE_KIBANA_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+export SYNTHETICS_REMOTE_KIBANA_USERNAME
+
+SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+export SYNTHETICS_REMOTE_KIBANA_PASSWORD
+
+SYNTHETICS_REMOTE_KIBANA_URL="$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+export SYNTHETICS_REMOTE_KIBANA_URL
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)

.buildkite/scripts/steps/code_coverage/ingest.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export CODE_COVERAGE=1
+echo "--- Reading Kibana stats cluster creds from vault"
+export USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/prod/coverage/elasticsearch)"
+export PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/prod/coverage/elasticsearch)"
+export HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)"
+export TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")
+
+echo "--- Download previous git sha"
+.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh
+previousSha=$(cat downloaded_previous.txt)
+
+echo "--- Upload new git sha"
+.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Download coverage arctifacts"
+buildkite-agent artifact download target/kibana-coverage/jest/* .
+buildkite-agent artifact download target/kibana-coverage/functional/* .
+
+echo "--- process HTML Links"
+.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh
+
+echo "--- collect VCS Info"
+.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh
+
+# replace path in json files and generate final reports
+echo "--- Replace path in json files"
+export COVERAGE_TEMP_DIR=$KIBANA_DIR/target/kibana-coverage
+sed -i "s|/opt/local-ssd/buildkite/builds/kb-[[:alnum:]\-]\{20,27\}/elastic/kibana-code-coverage-main/kibana|${KIBANA_DIR}|g" $COVERAGE_TEMP_DIR/**/*.json
+
+echo "--- Jest: merging coverage files and generating the final combined report"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js
+
+echo "--- Functional: merging json files and generating the final combined report"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js
+
+# archive reports to upload as build artifacts
+echo "--- Archive and upload combined reports"
+tar -czf target/kibana-coverage/jest/kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined
+tar -czf target/kibana-coverage/functional/kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined
+buildkite-agent artifact upload 'target/kibana-coverage/jest/kibana-jest-coverage.tar.gz'
+buildkite-agent artifact upload 'target/kibana-coverage/functional/kibana-functional-coverage.tar.gz'
+
+echo "--- Upload coverage static site"
+.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh
+
+echo "--- Ingest results to Kibana stats cluster"
+.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' ${BUILDKITE_BUILD_ID} ${BUILDKITE_BUILD_URL} ${previousSha} 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'

.buildkite/scripts/steps/code_coverage/jest.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+is_test_execution_step
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Jest code coverage'
+
+.buildkite/scripts/steps/code_coverage/jest_parallel.sh
+
+tar -czf kibana-jest-thread-coverage.tar.gz target/kibana-coverage/jest
+
+echo "--- Merging code coverage for a thread"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js --reporter json
+rm -rf target/kibana-coverage/jest/*
+mv target/kibana-coverage/jest-combined/coverage-final.json "target/kibana-coverage/jest/jest-merged-coverage-$(date +%s%3N).json"

.buildkite/scripts/steps/code_coverage/jest_integration.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+is_test_execution_step
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Jest Integration code coverage'
+node --max-old-space-size=14336 scripts/jest_integration  --ci --coverage --coverageReporters json || true
+mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/jest-integration-coverage.json"

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -uo pipefail
+
+JOB=$BUILDKITE_PARALLEL_JOB
+JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+
+# a jest failure will result in the script returning an exit code of 10
+
+i=0
+exitCode=0
+
+while read -r config; do
+  if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
+    echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"
+    node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest || true
+    echo "Rename coverage-final.json to avoid overwrite"
+    mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
+    lastCode=$?
+
+    if [ $lastCode -ne 0 ]; then
+      exitCode=10
+      echo "Jest exited with code $lastCode"
+      echo "^^^ +++"
+    fi
+  fi
+
+  ((i=i+1))
+# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
+done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)"
+
+exit $exitCode

.buildkite/scripts/steps/code_coverage/oss_cigroup.sh

@@ -0,0 +1,34 @@
+
+
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/build_kibana_plugins.sh
+
+is_test_execution_step
+
+export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
+export JOB=kibana-oss-ciGroup${CI_GROUP}
+
+export NODE_OPTIONS=--max_old_space_size=8192
+export CODE_COVERAGE=1
+
+echo "--- OSS CI Group $CI_GROUP"
+echo " -> Running Functional tests with code coverage"
+
+node scripts/functional_tests \
+  --include-tag "ciGroup$CI_GROUP" \
+  --exclude-tag "skipCoverage" || true
+
+if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then
+  echo "--- Merging code coverage for CI Group $CI_GROUP"
+  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
+  rm -rf target/kibana-coverage/functional/*
+  mv target/kibana-coverage/functional-combined/coverage-final.json "target/kibana-coverage/functional/oss-${CI_GROUP}-coverage.json"
+else
+  echo "--- Code coverage not found"
+fi

.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "### Prok'd Index File: ..."
+cat src/dev/code_coverage/www/index.html
+
+predicate() {
+  x=$1
+  if [ -n "$x" ]; then
+    return
+  else
+    echo "### 1 or more variables that Code Coverage needs, are undefined"
+    exit 1
+  fi
+}
+CMD="git log --pretty=format"
+XS=("${GIT_BRANCH}" \
+  "$(${CMD}":%h" -1)" \
+  "$(${CMD}":%an" -1)" \
+  "$(${CMD}":%s" -1)")
+touch VCS_INFO.txt
+for X in "${!XS[@]}"; do
+  {
+    predicate "${XS[X]}"
+    echo "${XS[X]}" >> VCS_INFO.txt
+  }
+done
+echo "### VCS_INFO:"
+cat VCS_INFO.txt

.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+gsutil -m cp -r gs://elastic-bekitzur-kibana-coverage-live/previous_pointer/previous.txt . || echo "### Previous Pointer NOT FOUND?"
+
+if [ -e ./previous.txt ]; then
+    mv previous.txt downloaded_previous.txt
+    echo "### downloaded_previous.txt"
+    cat downloaded_previous.txt
+fi