Skip to content

Commit

Permalink
Merge branch 'main' into fix-adjust-tooltip-on-run-test-for-on-privat…
Browse files Browse the repository at this point in the history
…e-locations
  • Loading branch information
awahab07 authored Sep 15, 2022
2 parents 3d255ca + f014ca4 commit 6387ad0
Show file tree
Hide file tree
Showing 8,774 changed files with 308,415 additions and 139,015 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
12 changes: 9 additions & 3 deletions .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ disabled:
- x-pack/plugins/observability/e2e/synthetics_run.ts

# Configs that exist but weren't running in CI when this file was introduced
- test/visual_regression/config.ts
- x-pack/test/visual_regression/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts
- x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts
Expand All @@ -58,6 +56,9 @@ disabled:
- x-pack/test/screenshot_creation/config.ts
- x-pack/test/fleet_packages/config.ts

# Scalability testing config that we run in its own pipeline
- x-pack/test/performance/scalability/config.ts

defaultQueue: 'n2-4-spot'
enabled:
- test/accessibility/config.ts
Expand Down Expand Up @@ -117,6 +118,7 @@ enabled:
- x-pack/test/alerting_api_integration/basic/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
- x-pack/test/alerting_api_integration/spaces_only/config.ts
- x-pack/test/api_integration_basic/config.ts
- x-pack/test/api_integration/config_security_basic.ts
Expand All @@ -138,6 +140,7 @@ enabled:
- x-pack/test/detection_engine_api_integration/security_and_spaces/group7/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group8/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
- x-pack/test/encrypted_saved_objects_api_integration/config.ts
- x-pack/test/endpoint_api_integration_no_ingest/config.ts
- x-pack/test/examples/config.ts
Expand Down Expand Up @@ -179,7 +182,9 @@ enabled:
- x-pack/test/functional/apps/maps/group2/config.ts
- x-pack/test/functional/apps/maps/group3/config.ts
- x-pack/test/functional/apps/maps/group4/config.ts
- x-pack/test/functional/apps/ml/anomaly_detection/config.ts
- x-pack/test/functional/apps/ml/anomaly_detection_jobs/config.ts
- x-pack/test/functional/apps/ml/anomaly_detection_integrations/config.ts
- x-pack/test/functional/apps/ml/anomaly_detection_result_views/config.ts
- x-pack/test/functional/apps/ml/data_frame_analytics/config.ts
- x-pack/test/functional/apps/ml/data_visualizer/config.ts
- x-pack/test/functional/apps/ml/permissions/config.ts
Expand Down Expand Up @@ -268,3 +273,4 @@ enabled:
- x-pack/test/performance/journeys/many_fields_discover/config.ts
- x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts
- x-pack/test/performance/journeys/web_logs_dashboard/config.ts
- x-pack/test/performance/journeys/data_stress_test_lens/config.ts
12 changes: 11 additions & 1 deletion .buildkite/pipelines/artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,23 @@ steps:

- command: .buildkite/scripts/steps/artifacts/cloud.sh
label: 'Cloud Deployment'
soft_fail: true
soft_fail:
- exit_status: 255
agents:
queue: n2-2
timeout_in_minutes: 30
if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
retry:
automatic:
# Timeout and graceful shutdown | ecctl deployment create falure
- exit_status: 255
limit: 0

# Timeout and forced shutdown
- exit_status: '-1'
limit: 0

# Test failures
- exit_status: '*'
limit: 1

Expand Down
2 changes: 1 addition & 1 deletion .buildkite/pipelines/on_merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ steps:
label: 'Populate local dev bazel cache (Linux)'
agents:
queue: n2-4-spot
timeout_in_minutes: 15
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '-1'
Expand Down
2 changes: 2 additions & 0 deletions .buildkite/pipelines/performance/daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ steps:
agents:
queue: c2-16
key: build
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"

- label: ':muscle: Performance Tests with Playwright config'
command: .buildkite/scripts/steps/functional/performance_playwright.sh
agents:
queue: kb-static-ubuntu
depends_on: build
key: tests
timeout_in_minutes: 60

- label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
Expand Down
23 changes: 23 additions & 0 deletions .buildkite/pipelines/scalability/daily.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
steps:
- label: ':male-mechanic::skin-tone-2: Pre-Build'
command: .buildkite/scripts/lifecycle/pre_build.sh
agents:
queue: kibana-default
timeout_in_minutes: 10

- wait

- label: ':kibana: Scalability Tests'
command: .buildkite/scripts/steps/scalability/benchmarking.sh
agents:
queue: kb-static-scalability
timeout_in_minutes: 90

- wait: ~
continue_on_failure: true

- label: ':male_superhero::skin-tone-2: Post-Build'
command: .buildkite/scripts/lifecycle/post_build.sh
agents:
queue: kibana-default
timeout_in_minutes: 10
5 changes: 4 additions & 1 deletion .buildkite/pull_requests.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@
"^src/dev/prs/kibana_qa_pr_list\\.json$",
"^\\.buildkite/pull_requests\\.json$"
],
"always_require_ci_on_changed": ["^docs/developer/plugin-list.asciidoc$"],
"always_require_ci_on_changed": [
"^docs/developer/plugin-list.asciidoc$",
"/plugins/[^/]+/readme\\.(md|asciidoc)$"
],
"kibana_versions_check": true,
"kibana_build_reuse": true,
"kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge"],
Expand Down
33 changes: 17 additions & 16 deletions .buildkite/scripts/steps/artifacts/cloud.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,28 +12,28 @@ mkdir -p target

download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

node scripts/build \
--skip-initialize \
--skip-generic-folders \
--skip-platform-folders \
--skip-archives \
--docker-images \
--skip-docker-ubi \
--skip-docker-ubuntu \
--skip-docker-contexts

docker load --input target/kibana-cloud-$FULL_VERSION-docker-image.tar.gz

TAG="$FULL_VERSION-$GIT_COMMIT"
KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG"

docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"

echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
trap 'docker logout docker.elastic.co' EXIT

docker push "$KIBANA_TEST_IMAGE"
if docker manifest inspect $KIBANA_TEST_IMAGE &> /dev/null; then
echo "Distribution already exists, skipping build"
else
node scripts/build \
--skip-initialize \
--skip-generic-folders \
--skip-platform-folders \
--skip-archives \
--docker-images \
--docker-tag-qualifier="$GIT_COMMIT" \
--docker-push \
--skip-docker-ubi \
--skip-docker-ubuntu \
--skip-docker-contexts
fi

docker logout docker.elastic.co

echo "--- Create deployment"
Expand Down Expand Up @@ -62,6 +62,7 @@ function shutdown {
trap "shutdown" EXIT

ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" > "$LOGS"

CLOUD_DEPLOYMENT_USERNAME=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")
Expand Down
9 changes: 8 additions & 1 deletion .buildkite/scripts/steps/checks/bazel_packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,12 @@ source .buildkite/scripts/common/util.sh

echo --- Check Bazel Packages Manifest
node scripts/generate packages_build_manifest

check_for_changed_files 'node scripts/generate packages_build_manifest' true

echo --- Check Codeowners Manifest
if [ -f ".github/CODEOWNERS" ]; then
node scripts/generate codeowners
check_for_changed_files 'node scripts/generate codeowners' true
else
echo "skipping, no existing .github/CODEOWNERS file found"
fi
1 change: 1 addition & 0 deletions .buildkite/scripts/steps/cloud/build_and_deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,4 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
EOF

buildkite-agent meta-data set pr_comment:deploy_cloud:head "* [Cloud Deployment](${CLOUD_DEPLOYMENT_KIBANA_URL})"
buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ source .buildkite/scripts/common/util.sh
USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/apm_parser_performance)"
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/apm_parser_performance)"
ES_SERVER_URL="https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243"
BUILD_ID=${BUILDKITE_BUILD_ID}
BUILD_ID="${BUILDKITE_BUILD_ID}"
GCS_BUCKET="gs://kibana-performance/scalability-tests"
OUTPUT_REL="target/scalability_tests/${BUILD_ID}"
OUTPUT_DIR="${KIBANA_DIR}/${OUTPUT_REL}"

.buildkite/scripts/bootstrap.sh

Expand All @@ -20,24 +22,35 @@ for i in "${scalabilityJourneys[@]}"; do
echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces"

node scripts/extract_performance_testing_dataset \
--config "x-pack/test/performance/journeys/${i}/config.ts" \ \
--config "x-pack/test/performance/journeys/${i}/config.ts" \
--buildId "${BUILD_ID}" \
--es-url "${ES_SERVER_URL}" \
--es-username "${USER_FROM_VAULT}" \
--es-password "${PASS_FROM_VAULT}" \
--without-static-resources
done

echo "--- Upload Kibana build, plugins and scalability traces to the public bucket"
mkdir "${BUILD_ID}"
# Archive json files with traces and upload as build artifacts
tar -czf "${BUILD_ID}/scalability_traces.tar.gz" -C target scalability_traces
buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
# Upload Kibana build, plugins, commit sha and traces to the bucket
download_artifact kibana-default.tar.gz ./"${BUILD_ID}"
download_artifact kibana-default-plugins.tar.gz ./"${BUILD_ID}"
echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
echo "--- Creating scalability dataset in ${OUTPUT_REL}"
mkdir -p "${OUTPUT_DIR}"

echo "--- Archiving scalability trace and uploading as build artifact"
tar -czf "${OUTPUT_DIR}/scalability_traces.tar.gz" -C target scalability_traces
buildkite-agent artifact upload "${OUTPUT_DIR}/scalability_traces.tar.gz"

echo "--- Downloading Kibana artifacts used in tests"
download_artifact kibana-default.tar.gz "${OUTPUT_DIR}/" --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact kibana-default-plugins.tar.gz "${OUTPUT_DIR}/" --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

echo "--- Adding commit info"
echo "${BUILDKITE_COMMIT}" > "${OUTPUT_DIR}/KIBANA_COMMIT_HASH"

echo "--- Uploading ${OUTPUT_REL} dir to ${GCS_BUCKET}"
cd "${OUTPUT_DIR}/.."
gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
echo "--- Update reference to the latest CI build"
echo "${BUILD_ID}" > LATEST
gsutil cp LATEST "${GCS_BUCKET}"
cd -

echo "--- Promoting '${BUILD_ID}' dataset to LATEST"
cd "${OUTPUT_DIR}/.."
echo "${BUILD_ID}" > latest
gsutil cp latest "${GCS_BUCKET}"
cd -
122 changes: 122 additions & 0 deletions .buildkite/scripts/steps/scalability/benchmarking.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/common/util.sh

#.buildkite/scripts/bootstrap.sh
echo "--- yarn kbn reset && yarn kbn bootstrap"
yarn kbn reset && yarn kbn bootstrap

GCS_BUCKET="gs://kibana-performance/scalability-tests"
GCS_ARTIFACTS_REL="gcs_artifacts"
GCS_ARTIFACTS_DIR="${WORKSPACE}/${GCS_ARTIFACTS_REL}"
KIBANA_LOAD_TESTING_DIR="${KIBANA_DIR}/kibana-load-testing"

# These tests are running on static workers so we must delete previous build, load runner and scalability artifacts
rm -rf "${KIBANA_BUILD_LOCATION}"
rm -rf "${KIBANA_LOAD_TESTING_DIR}"
rm -rf "${GCS_ARTIFACTS_DIR}"

download_artifacts() {
mkdir -p "${GCS_ARTIFACTS_DIR}"

gsutil cp "$GCS_BUCKET/latest" "${GCS_ARTIFACTS_DIR}/"
HASH=`cat ${GCS_ARTIFACTS_DIR}/latest`
gsutil cp -r "$GCS_BUCKET/$HASH" "${GCS_ARTIFACTS_DIR}/"

export LATEST_RUN_ARTIFACTS_DIR="${GCS_ARTIFACTS_DIR}/${HASH}"

echo "Unzip kibana build, plugins and scalability traces"
cd "$WORKSPACE"
mkdir -p "$KIBANA_BUILD_LOCATION"
tar -xzf "${LATEST_RUN_ARTIFACTS_DIR}/kibana-default.tar.gz" -C "$KIBANA_BUILD_LOCATION" --strip=1

cd "$KIBANA_DIR"
tar -xzf "${LATEST_RUN_ARTIFACTS_DIR}/kibana-default-plugins.tar.gz"
tar -xzf "${LATEST_RUN_ARTIFACTS_DIR}/scalability_traces.tar.gz"
}

checkout_and_compile_load_runner() {
mkdir -p "${KIBANA_LOAD_TESTING_DIR}" && cd "${KIBANA_LOAD_TESTING_DIR}"

if [[ ! -d .git ]]; then
git init
git remote add origin https://github.com/elastic/kibana-load-testing.git
fi
git fetch origin --depth 1 "main"
git reset --hard FETCH_HEAD

KIBANA_LOAD_TESTING_GIT_COMMIT="$(git rev-parse HEAD)"
export KIBANA_LOAD_TESTING_GIT_COMMIT

mvn -q test-compile
echo "Set 'GATLING_PROJECT_PATH' env var for ScalabilityTestRunner"
export GATLING_PROJECT_PATH="$(pwd)"
}

upload_test_results() {
cd "${KIBANA_DIR}"
echo "--- Archive Gatling reports and upload as build artifacts"
tar -czf "scalability_test_report.tar.gz" --exclude=simulation.log -C kibana-load-testing/target gatling
buildkite-agent artifact upload "scalability_test_report.tar.gz"
cd "${LATEST_RUN_ARTIFACTS_DIR}"
echo "Upload scalability traces as build artifacts"
buildkite-agent artifact upload "scalability_traces.tar.gz"
}

echo "--- Download the latest artifacts from single user performance pipeline"
download_artifacts

echo "--- Clone kibana-load-testing repo and compile project"
checkout_and_compile_load_runner

echo "--- Run Scalability Tests with Elasticsearch started only once and Kibana restart before each journey"
cd "$KIBANA_DIR"
node scripts/es snapshot&

esPid=$!
# Set trap on EXIT to stop Elasticsearch process
trap "kill -9 $esPid" EXIT

# unset env vars defined in other parts of CI for automatic APM collection of
# Kibana. We manage APM config in our FTR config and performance service, and
# APM treats config in the ENV with a very high precedence.
unset ELASTIC_APM_ENVIRONMENT
unset ELASTIC_APM_TRANSACTION_SAMPLE_RATE
unset ELASTIC_APM_SERVER_URL
unset ELASTIC_APM_SECRET_TOKEN
unset ELASTIC_APM_ACTIVE
unset ELASTIC_APM_CONTEXT_PROPAGATION_ONLY
unset ELASTIC_APM_GLOBAL_LABELS
unset ELASTIC_APM_MAX_QUEUE_SIZE
unset ELASTIC_APM_METRICS_INTERVAL
unset ELASTIC_APM_CAPTURE_SPAN_STACK_TRACES
unset ELASTIC_APM_BREAKDOWN_METRICS


export TEST_ES_DISABLE_STARTUP=true
ES_HOST="localhost:9200"
export TEST_ES_URL="http://elastic:changeme@${ES_HOST}"
# Overriding Gatling default configuration
export ES_URL="http://${ES_HOST}"

# Pings the ES server every second for 2 mins until its status is green
curl --retry 120 \
--retry-delay 1 \
--retry-connrefused \
-I -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow"

export ELASTIC_APM_ACTIVE=true

for journey in scalability_traces/server/*; do
export SCALABILITY_JOURNEY_PATH="$KIBANA_DIR/$journey"
echo "--- Run scalability file: $SCALABILITY_JOURNEY_PATH"
node scripts/functional_tests \
--config x-pack/test/performance/scalability/config.ts \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--debug
done

echo "--- Upload test results"
upload_test_results
Loading

0 comments on commit 6387ad0

Please sign in to comment.