Merge branch 'main' into d/2022-05-11-query-state

elastic · May 19, 2022 · cc66e03 · cc66e03
2 parents f56801a + e2064ae
commit cc66e03
Show file tree

Hide file tree

Showing 571 changed files with 15,251 additions and 8,391 deletions.
diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -62,6 +62,16 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
     agents:

diff --git a/.buildkite/pull_requests.json b/.buildkite/pull_requests.json
@@ -16,7 +16,25 @@
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "skip_ci_labels": ["skip-ci", "jenkins-ci"],
-      "skip_target_branches": ["6.8", "7.11", "7.12"]
+      "skip_target_branches": ["6.8", "7.11", "7.12"],
+      "skip_ci_on_only_changed": [
+        "^docs/",
+        "^rfcs/",
+        "^.ci/.+\\.yml$",
+        "^.ci/es-snapshots/",
+        "^.ci/pipeline-library/",
+        "^.ci/Jenkinsfile_[^/]+$",
+        "^\\.github/",
+        "\\.md$",
+        "^\\.backportrc\\.json$",
+        "^nav-kibana-dev\\.docnav\\.json$",
+        "^src/dev/prs/kibana_qa_pr_list\\.json$",
+        "^\\.buildkite/pull_requests\\.json$"
+      ],
+      "always_require_ci_on_changed": [
+        "^docs/developer/plugin-list.asciidoc$",
+        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
+      ]
     }
   ]
 }
diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh
@@ -38,6 +38,7 @@ export TEST_BROWSER_HEADLESS=1
 export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 export ELASTIC_APM_SERVER_URL=https://kibana-ci-apm.apm.us-central1.gcp.cloud.es.io
+# Not really a secret, if APM supported public auth we would use it and APM requires that we use this name
 export ELASTIC_APM_SECRET_TOKEN=7YKhoXsO4MzjhXjx2c
 
 if is_pr; then

diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.js b/.buildkite/scripts/pipelines/pull_request/pipeline.js
@@ -9,14 +9,11 @@
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
-const { SKIPPABLE_PR_MATCHERS } = require('./skippable_pr_matchers');
-
-const REQUIRED_PATHS = [
-  // this file is auto-generated and changes to it need to be validated with CI
-  /^docs\/developer\/plugin-list.asciidoc$/,
-  // don't skip CI on prs with changes to plugin readme files /i is for case-insensitive matching
-  /\/plugins\/[^\/]+\/readme\.(md|asciidoc)$/i,
-];
+const prConfigs = require('../../../pull_requests.json');
+const prConfig = prConfigs.jobs.find((job) => job.pipelineSlug === 'kibana-pull-request');
+
+const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
+const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
 
 const getPipeline = (filename, removeSteps = true) => {
   const str = fs.readFileSync(filename).toString();

diff --git a/.buildkite/scripts/pipelines/pull_request/skippable_pr_matchers.js b/.buildkite/scripts/pipelines/pull_request/skippable_pr_matchers.js
diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh
@@ -19,6 +19,8 @@ if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
   DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
 elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
   DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
 fi
 
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"

diff --git a/.buildkite/scripts/steps/es_snapshots/build.sh b/.buildkite/scripts/steps/es_snapshots/build.sh
@@ -69,7 +69,6 @@ echo "--- Build Elasticsearch"
   :distribution:archives:darwin-aarch64-tar:assemble \
   :distribution:archives:darwin-tar:assemble \
   :distribution:docker:docker-export:assemble \
-  :distribution:docker:cloud-docker-export:assemble \
   :distribution:archives:linux-aarch64-tar:assemble \
   :distribution:archives:linux-tar:assemble \
   :distribution:archives:windows-zip:assemble \
@@ -86,19 +85,26 @@ docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 
 echo "--- Create kibana-ci docker cloud image archives"
-ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
-ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
-KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
-KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
-
-docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
-
-echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-trap 'docker logout docker.elastic.co' EXIT
-docker image push "$KIBANA_ES_CLOUD_IMAGE"
-
-export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
-export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+# Ignore build failures.  This docker image downloads metricbeat and filebeat.
+# When we bump versions, these dependencies may not exist yet, but we don't want to
+# block the rest of the snapshot promotion process
+set +e
+./gradlew :distribution:docker:cloud-docker-export:assemble && {
+  ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
+  ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
+  KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
+  KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
+  echo $ES_CLOUD_ID $ES_CLOUD_VERSION $KIBANA_ES_CLOUD_VERSION $KIBANA_ES_CLOUD_IMAGE
+  docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
+
+  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  trap 'docker logout docker.elastic.co' EXIT
+  docker image push "$KIBANA_ES_CLOUD_IMAGE"
+
+  export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
+  export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+}
+set -e
 
 echo "--- Create checksums for snapshot files"
 cd "$destination"