diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index 1f9a75c72b1cb..a77798345695e 100755 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -9,8 +9,9 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then echo "--- Upload Artifacts" buildkite-agent artifact upload 'target/junit/**/*' - buildkite-agent artifact upload 'target/kibana-*' buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*' + buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*' + buildkite-agent artifact upload 'target/kibana-*' buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png' buildkite-agent artifact upload 'target/test-metrics/*' buildkite-agent artifact upload 'target/test-suites-ci-plan.json' diff --git a/.buildkite/scripts/steps/code_coverage/ingest.sh b/.buildkite/scripts/steps/code_coverage/ingest.sh new file mode 100755 index 0000000000000..a9b95cccb533c --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/ingest.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +export CODE_COVERAGE=1 +echo "--- Reading Kibana stats cluster creds from vault" +export USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/prod/coverage/elasticsearch)" +export PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/prod/coverage/elasticsearch)" +export HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)" +export TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z") + +echo "--- Download previous git sha" +.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh +previousSha=$(cat downloaded_previous.txt) + +echo "--- Upload new git sha" +.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh + +.buildkite/scripts/bootstrap.sh + +echo "--- Download coverage arctifacts" +buildkite-agent artifact download target/kibana-coverage/jest/* . +buildkite-agent artifact download target/kibana-coverage/functional/* . + +echo "--- process HTML Links" +.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh + +echo "--- collect VCS Info" +.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh + +# replace path in json files and generate final reports +echo "--- Replace path in json files" +export COVERAGE_TEMP_DIR=$KIBANA_DIR/target/kibana-coverage +sed -i "s|/opt/local-ssd/buildkite/builds/kb-[[:alnum:]\-]\{20,27\}/elastic/kibana-code-coverage-main/kibana|${KIBANA_DIR}|g" $COVERAGE_TEMP_DIR/**/*.json + +echo "--- Jest: merging coverage files and generating the final combined report" +yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js + +echo "--- Functional: merging json files and generating the final combined report" +yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js + +# archive reports to upload as build artifacts +echo "--- Archive and upload combined reports" +tar -czf target/kibana-coverage/jest/kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined +tar -czf target/kibana-coverage/functional/kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined +buildkite-agent artifact upload 'target/kibana-coverage/jest/kibana-jest-coverage.tar.gz' +buildkite-agent artifact upload 'target/kibana-coverage/functional/kibana-functional-coverage.tar.gz' + +echo "--- Upload coverage static site" +.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh + +echo "--- Ingest results to Kibana stats cluster" +.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' ${BUILDKITE_BUILD_ID} ${BUILDKITE_BUILD_URL} ${previousSha} 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt' diff --git a/.buildkite/scripts/steps/code_coverage/jest.sh b/.buildkite/scripts/steps/code_coverage/jest.sh new file mode 100755 index 0000000000000..0884a27c03aad --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/jest.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +is_test_execution_step + +.buildkite/scripts/bootstrap.sh + +echo '--- Jest code coverage' + +.buildkite/scripts/steps/code_coverage/jest_parallel.sh + +tar -czf kibana-jest-thread-coverage.tar.gz target/kibana-coverage/jest + +echo "--- Merging code coverage for a thread" +yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js --reporter json +rm -rf target/kibana-coverage/jest/* +mv target/kibana-coverage/jest-combined/coverage-final.json "target/kibana-coverage/jest/jest-merged-coverage-$(date +%s%3N).json" \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/jest_integration.sh b/.buildkite/scripts/steps/code_coverage/jest_integration.sh new file mode 100755 index 0000000000000..e35a4aa61cc0c --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/jest_integration.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +is_test_execution_step + +.buildkite/scripts/bootstrap.sh + +echo '--- Jest Integration code coverage' +node --max-old-space-size=14336 scripts/jest_integration --ci --coverage --coverageReporters json || true +mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/jest-integration-coverage.json" \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/jest_parallel.sh b/.buildkite/scripts/steps/code_coverage/jest_parallel.sh new file mode 100755 index 0000000000000..47abafbf0ad06 --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/jest_parallel.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -uo pipefail + +JOB=$BUILDKITE_PARALLEL_JOB +JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT + +# a jest failure will result in the script returning an exit code of 10 + +i=0 +exitCode=0 + +while read -r config; do + if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then + echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest" + node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest || true + echo "Rename coverage-final.json to avoid overwrite" + mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json" + lastCode=$? + + if [ $lastCode -ne 0 ]; then + exitCode=10 + echo "Jest exited with code $lastCode" + echo "^^^ +++" + fi + fi + + ((i=i+1)) +# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode +done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)" + +exit $exitCode \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/oss_cigroup.sh b/.buildkite/scripts/steps/code_coverage/oss_cigroup.sh new file mode 100755 index 0000000000000..44d4bb500e1d2 --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/oss_cigroup.sh @@ -0,0 +1,34 @@ + + +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +.buildkite/scripts/bootstrap.sh +.buildkite/scripts/build_kibana_plugins.sh + +is_test_execution_step + +export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))} +export JOB=kibana-oss-ciGroup${CI_GROUP} + +export NODE_OPTIONS=--max_old_space_size=8192 +export CODE_COVERAGE=1 + +echo "--- OSS CI Group $CI_GROUP" +echo " -> Running Functional tests with code coverage" + +node scripts/functional_tests \ + --include-tag "ciGroup$CI_GROUP" \ + --exclude-tag "skipCoverage" || true + +if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then + echo "--- Merging code coverage for CI Group $CI_GROUP" + yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json + rm -rf target/kibana-coverage/functional/* + mv target/kibana-coverage/functional-combined/coverage-final.json "target/kibana-coverage/functional/oss-${CI_GROUP}-coverage.json" +else + echo "--- Code coverage not found" +fi \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh b/.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh new file mode 100755 index 0000000000000..4e6b3907b6e34 --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "### Prok'd Index File: ..." +cat src/dev/code_coverage/www/index.html + +predicate() { + x=$1 + if [ -n "$x" ]; then + return + else + echo "### 1 or more variables that Code Coverage needs, are undefined" + exit 1 + fi +} +CMD="git log --pretty=format" +XS=("${GIT_BRANCH}" \ + "$(${CMD}":%h" -1)" \ + "$(${CMD}":%an" -1)" \ + "$(${CMD}":%s" -1)") +touch VCS_INFO.txt +for X in "${!XS[@]}"; do + { + predicate "${XS[X]}" + echo "${XS[X]}" >> VCS_INFO.txt + } +done +echo "### VCS_INFO:" +cat VCS_INFO.txt \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh b/.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh new file mode 100755 index 0000000000000..e7158bb07b667 --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail + +gsutil -m cp -r gs://elastic-bekitzur-kibana-coverage-live/previous_pointer/previous.txt . || echo "### Previous Pointer NOT FOUND?" + +if [ -e ./previous.txt ]; then + mv previous.txt downloaded_previous.txt + echo "### downloaded_previous.txt" + cat downloaded_previous.txt +fi \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh b/.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh new file mode 100755 index 0000000000000..9d67cbbfaf5f8 --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "### Ingesting Code Coverage" +echo "" + +COVERAGE_JOB_NAME=$1 +export COVERAGE_JOB_NAME +echo "### debug COVERAGE_JOB_NAME: ${COVERAGE_JOB_NAME}" + +BUILD_ID=$2 +export BUILD_ID + +CI_RUN_URL=$3 +export CI_RUN_URL +echo "### debug CI_RUN_URL: ${CI_RUN_URL}" + +FETCHED_PREVIOUS=$4 +export FETCHED_PREVIOUS +echo "### debug FETCHED_PREVIOUS: ${FETCHED_PREVIOUS}" + +ES_HOST="https://${USER_FROM_VAULT}:${PASS_FROM_VAULT}@${HOST_FROM_VAULT}" +export ES_HOST + +STATIC_SITE_URL_BASE='https://kibana-coverage.elastic.dev' +export STATIC_SITE_URL_BASE + +TEAM_ASSIGN_PATH=$5 +echo "### debug TEAM_ASSIGN_PATH: ${TEAM_ASSIGN_PATH}" + +BUFFER_SIZE=500 +export BUFFER_SIZE +echo "### debug BUFFER_SIZE: ${BUFFER_SIZE}" + +# Build team assignments file +CI_STATS_DISABLED=true node scripts/generate_team_assignments.js --verbose --src '.github/CODEOWNERS' --dest $TEAM_ASSIGN_PATH + +for x in functional jest; do + echo "### Ingesting coverage for ${x}" + COVERAGE_SUMMARY_FILE=target/kibana-coverage/${x}-combined/coverage-summary.json + # running in background to speed up ingestion + CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH & +done +wait + +echo "### Ingesting Code Coverage - Complete" +echo "" \ No newline at end of file diff --git a/.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh b/.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh new file mode 100755 index 0000000000000..f982ee5c581ce --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cat << EOF > src/dev/code_coverage/www/index_partial_2.html + Latest Jest + Latest FTR + + + + + + Use Kibana Stats to mine coverage data + + Dashboard + + + + +
Use Kibana Stats to mine coverage data
+ Dashboard +