From a105f6aaec9460cfa5e601e7dac853871b36e6d4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 1 Jun 2023 16:57:16 +0200 Subject: [PATCH] Update pipeline to test with integrations (#531) --- .buildkite/hooks/post-checkout | 53 +++++ .buildkite/hooks/pre-command | 16 ++ .../pipeline.test-with-integrations-repo.yml | 17 ++ .buildkite/scripts/install_deps.sh | 47 +++++ .buildkite/scripts/test-with-integrations.sh | 188 ++++++++++++++++++ .buildkite/scripts/tooling.sh | 23 +++ 6 files changed, 344 insertions(+) create mode 100644 .buildkite/hooks/post-checkout create mode 100644 .buildkite/hooks/pre-command create mode 100755 .buildkite/scripts/install_deps.sh create mode 100755 .buildkite/scripts/test-with-integrations.sh create mode 100755 .buildkite/scripts/tooling.sh diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100644 index 000000000..67ae21491 --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "Commit information" +git log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 000000000..906b668d7 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/tooling.sh + +GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token + +# Secrets must be redacted +# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "package-spec-test-with-integrations" && "$BUILDKITE_STEP_KEY" == "pr-integrations" ]]; then + export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH}) + export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH}) + export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) +fi diff --git a/.buildkite/pipeline.test-with-integrations-repo.yml b/.buildkite/pipeline.test-with-integrations-repo.yml index 58342b47c..b5b4e7be4 100644 --- a/.buildkite/pipeline.test-with-integrations-repo.yml +++ b/.buildkite/pipeline.test-with-integrations-repo.yml @@ -12,4 +12,21 @@ steps: image: "golang:${GOLANG_VERSION}" cpu: "8" memory: "4G" + - label: ":linux: Tests on Linux" + key: test + command: "make test-ci" + agents: + image: "golang:${GOLANG_VERSION}" + cpu: "8" + memory: "4G" + + - label: ":hammer: Create PR in integrations" + key: pr-integrations + command: ".buildkite/scripts/test-with-integrations.sh" + agents: + provider: "gcp" + depends_on: + - check + - test + diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh new file mode 100755 index 000000000..b241c4e5a --- /dev/null +++ b/.buildkite/scripts/install_deps.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/tooling.sh + +add_bin_path(){ + mkdir -p ${WORKSPACE}/bin + export PATH="${WORKSPACE}/bin:${PATH}" +} + +with_go() { + mkdir -p ${WORKSPACE}/bin + retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64" + chmod +x ${WORKSPACE}/bin/gvm + eval "$(gvm $(cat .go-version))" + go version + which go + export PATH="$(go env GOPATH)/bin:${PATH}" +} + +with_github_cli() { + mkdir -p ${WORKSPACE}/bin + mkdir -p ${WORKSPACE}/tmp + + local gh_filename="gh_${GH_CLI_VERSION}_linux_amd64" + local gh_tar_file="${gh_filename}.tar.gz" + local gh_tar_full_path="${WORKSPACE}/tmp/${gh_tar_file}" + + retry 5 curl -sL -o ${gh_tar_full_path} "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}" + + # just extract the binary file from the tar.gz + tar -C ${WORKSPACE}/bin -xpf ${gh_tar_full_path} ${gh_filename}/bin/gh --strip-components=2 + + chmod +x ${WORKSPACE}/bin/gh + rm -rf ${WORKSPACE}/tmp + + gh version +} + +with_jq() { + mkdir -p ${WORKSPACE}/bin + retry 5 curl -sL -o ${WORKSPACE}/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64" + + chmod +x ${WORKSPACE}/bin/jq + jq --version +} diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh new file mode 100755 index 000000000..06b8163a9 --- /dev/null +++ b/.buildkite/scripts/test-with-integrations.sh @@ -0,0 +1,188 @@ +#!/bin/bash + +set -euo pipefail + +WORKSPACE="$(pwd)" + +TMP_FOLDER_TEMPLATE_BASE="tmp.${GITHUB_PR_BASE_REPO}" +TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" + +cleanup() { + echo "Deleting temporal files..." + cd ${WORKSPACE} + rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*" + echo "Done." +} + +trap cleanup EXIT +source .buildkite/scripts/install_deps.sh + +add_bin_path + +echo "--- install gh cli" +with_github_cli + +echo "--- install jq" +with_jq + + +INTEGRATIONS_SOURCE_BRANCH=main +INTEGRATIONS_GITHUB_OWNER=elastic +INTEGRATIONS_GITHUB_REPO_NAME=integrations +INTEGRATIONS_PR_BRANCH="test-${GITHUB_PR_BASE_REPO}-pr-${BUILDKITE_PULL_REQUEST}" +INTEGRATIONS_PR_TITLE="Test branch ${GITHUB_PR_BASE_REPO}#${BUILDKITE_PULL_REQUEST} - DO NOT MERGE" +VERSION_DEP="/v2" + +get_pr_number() { + # requires GITHUB_TOKEN + local branch="$1" + gh pr list -H "${branch}" --json number | jq -r '.[]|.number' +} + +get_integrations_pr_link() { + local pr_number=$1 + echo "https://github.com/elastic/integrations/pull/${pr_number}" +} + +get_source_pr_link() { + echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/pull/${BUILDKITE_PULL_REQUEST}" +} + +get_source_commit_link() { + echo "https://github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}/commit/${GITHUB_PR_HEAD_SHA}" +} + +set_git_config() { + git config user.name "${GITHUB_USERNAME_SECRET}" + git config user.email "${GITHUB_EMAIL_SECRET}" +} + +git_push_with_auth() { + local owner="$1" + local repository="$2" + local branch="$3" + + retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/${owner}/${repository}.git "${branch}" +} + +clone_repository() { + local target="$1" + retry 5 git clone https://github.com/elastic/integrations ${target} +} + +create_integrations_pull_request() { + # requires GITHUB_TOKEN + local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE}) + echo "Creating Pull Request" + message="Update ${GITHUB_PR_BASE_REPO} reference to $(get_source_commit_link).\nAutomated by [Buildkite build](${BUILDKITE_BUILD_URL})\n\nRelates: $(get_source_pr_link)" + echo -e $message > ${temp_path}/body-pr.txt + retry 3 \ + gh pr create \ + --title "${INTEGRATIONS_PR_TITLE}" \ + --body-file ${temp_path}/body-pr.txt \ + --draft \ + --base ${INTEGRATIONS_SOURCE_BRANCH} \ + --head ${INTEGRATIONS_PR_BRANCH} \ + --assignee ${GITHUB_PR_HEAD_USER} +} + +update_dependency() { + # it needs to set the Golang version from the integrations repository (.go-version file) + echo "--- install go for integrations repository :go:" + with_go + + echo "--- Updating go.mod and go.sum with ${GITHUB_PR_HEAD_SHA} :hammer_and_wrench:" + local source_dep="github.com/${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}${VERSION_DEP}" + local target_dep="github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}${VERSION_DEP}@${GITHUB_PR_HEAD_SHA}" + + go mod edit -replace ${source_dep}=${target_dep} + go mod tidy + + git add go.mod + git add go.sum + + # allow not to commit if there are no changes + # previous execution could fail and just pushed the branch but PR is not created + if ! git diff-index --quiet HEAD ; then + git commit -m "Test elastic-package from PR ${BUILDKITE_PULL_REQUEST} - ${GITHUB_PR_HEAD_SHA}" + fi + + echo "" + git --no-pager show --format=oneline HEAD + echo "" +} + + +exists_branch() { + local owner="$1" + local repository="$2" + local branch="$3" + + git ls-remote --exit-code --heads https://github.com/${owner}/${repository}.git ${branch} +} + +create_or_update_pull_request() { + local temp_path=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE}) + local repo_path="${temp_path}/elastic-integrations" + local checkout_options="" + local integrations_pr_number="" + + echo "Cloning repository" + clone_repository "${repo_path}" + + pushd "${repo_path}" > /dev/null + + set_git_config + + echo "Checking branch ${INTEGRATIONS_PR_BRANCH} in remote ${INTEGRATIONS_GITHUB_OWNER}/${INTEGRATIONS_GITHUB_REPO_NAME}" + if ! exists_branch ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} ; then + checkout_options=" -b " + echo "Creating a new branch..." + else + echo "Already existed" + fi + + integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}") + if [ -z "${integrations_pr_number}" ]; then + echo "Exists PR in integrations repository: ${integrations_pr_number}" + fi + + git checkout ${checkout_options} ${INTEGRATIONS_PR_BRANCH} + + echo "--- Updating dependency :pushpin:" + update_dependency + + echo "--- Pushing branch ${INTEGRATIONS_PR_BRANCH} to integrations repository..." + git_push_with_auth ${INTEGRATIONS_GITHUB_OWNER} ${INTEGRATIONS_GITHUB_REPO_NAME} ${INTEGRATIONS_PR_BRANCH} + + if [ -z "${integrations_pr_number}" ]; then + echo "--- Creating pull request :github:" + create_integrations_pull_request + + sleep 10 + + integrations_pr_number=$(get_pr_number "${INTEGRATIONS_PR_BRANCH}") + fi + + popd > /dev/null + + rm -rf "${temp_path}" + + echo "--- adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:" + add_pr_comment "${BUILDKITE_PULL_REQUEST}" "$(get_integrations_pr_link ${integrations_pr_number})" +} + + +add_pr_comment() { + local source_pr_number="$1" + local integrations_pr_link="$2" + + retry 3 \ + gh pr comment ${source_pr_number} \ + --body "Created or updated PR in integrations repostiory to test this vesrion. Check ${integrations_pr_link}" \ + --repo ${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO} +} + + +echo "--- creating or updating integrations pull request" +create_or_update_pull_request diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh new file mode 100755 index 000000000..305fc932e --- /dev/null +++ b/.buildkite/scripts/tooling.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +retry() { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} +