Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci] Skip project image build if manifest exists on pull requests #187680

Merged
merged 2 commits into from
Jul 8, 2024
Merged

[ci] Skip project image build if manifest exists on pull requests #187680

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7a5e7c
[ci] Skip project image build if manifest exists on pull requests
jbudz Jul 5, 2024
0b0f58c
Merge branch 'main' into ci/skip-project-build-if-exists
jbudz Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 71 additions & 62 deletions .buildkite/scripts/steps/artifacts/docker_image.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,46 +18,82 @@ export KIBANA_IMAGE="$KIBANA_BASE_IMAGE:$KIBANA_IMAGE_TAG"

echo "--- Verify manifest does not already exist"
echo "Checking manifest for $KIBANA_IMAGE"
if docker manifest inspect $KIBANA_IMAGE &> /dev/null; then
echo "Manifest already exists, exiting"
exit 1
SKIP_BUILD=false
if docker manifest inspect "$KIBANA_IMAGE" &> /dev/null; then
# If a staging build manifest already exists, there's a workflow error and the root cause should be investigated.
if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
echo "Manifest already exists, exiting"
exit 1
else
echo "Manifest already exists, skipping build. Look up previous build for artifacts."
SKIP_BUILD=true
fi
fi

echo "--- Build Kibana"
node scripts/build \
--debug \
--release \
--serverless \
--docker-cross-compile \
--docker-namespace="kibana-ci" \
--docker-tag="$KIBANA_IMAGE_TAG"

echo "--- Tag images"
docker rmi "$KIBANA_IMAGE"
docker load < "target/kibana-serverless-$BASE_VERSION-docker-image.tar.gz"
docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-amd64"

docker rmi "$KIBANA_IMAGE"
docker load < "target/kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.gz"
docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"

echo "--- Push images"
docker image push "$KIBANA_IMAGE-arm64"
docker image push "$KIBANA_IMAGE-amd64"

echo "--- Create and push manifests"
docker manifest create \
"$KIBANA_IMAGE" \
--amend "$KIBANA_IMAGE-arm64" \
--amend "$KIBANA_IMAGE-amd64"
docker manifest push "$KIBANA_IMAGE"

if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] && [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
if [[ "$SKIP_BUILD" == "false" ]]; then
echo "--- Build Kibana"
node scripts/build \
--debug \
--release \
--serverless \
--docker-cross-compile \
--docker-namespace="kibana-ci" \
--docker-tag="$KIBANA_IMAGE_TAG"

echo "--- Tag images"
docker rmi "$KIBANA_IMAGE"
docker load < "target/kibana-serverless-$BASE_VERSION-docker-image.tar.gz"
docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-amd64"

docker rmi "$KIBANA_IMAGE"
docker load < "target/kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.gz"
docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"

echo "--- Push images"
docker image push "$KIBANA_IMAGE-arm64"
docker image push "$KIBANA_IMAGE-amd64"

echo "--- Create and push manifests"
docker manifest create \
"$KIBANA_BASE_IMAGE:latest" \
"$KIBANA_IMAGE" \
--amend "$KIBANA_IMAGE-arm64" \
--amend "$KIBANA_IMAGE-amd64"
docker manifest push "$KIBANA_BASE_IMAGE:latest"
docker manifest push "$KIBANA_IMAGE"

if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] && [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
docker manifest create \
"$KIBANA_BASE_IMAGE:latest" \
--amend "$KIBANA_IMAGE-arm64" \
--amend "$KIBANA_IMAGE-amd64"
docker manifest push "$KIBANA_BASE_IMAGE:latest"
fi

echo "--- Build dependencies report"
node scripts/licenses_csv_report "--csv=target/dependencies-$GIT_ABBREV_COMMIT.csv"

echo "--- Upload archives"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-linux-x86_64.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-linux-aarch64.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-image.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-build-context.tar.gz"
buildkite-agent artifact upload "kibana-$BASE_VERSION-cdn-assets.tar.gz"
buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"

echo "--- Upload CDN assets"
cd target
gcloud auth activate-service-account --key-file <(echo "$GCS_SA_CDN_KEY")

CDN_ASSETS_FOLDER=$(mktemp -d)
tar -xf "kibana-$BASE_VERSION-cdn-assets.tar.gz" -C "$CDN_ASSETS_FOLDER" --strip=1

gsutil -m cp -r "$CDN_ASSETS_FOLDER/*" "gs://$GCS_SA_CDN_BUCKET/$GIT_ABBREV_COMMIT"
gcloud auth revoke "$GCS_SA_CDN_EMAIL"

echo "--- Validate CDN assets"
ts-node "$(git rev-parse --show-toplevel)/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts" \
"$GCS_SA_CDN_URL" \
"$CDN_ASSETS_FOLDER"
fi

cat << EOF | buildkite-agent annotate --style "info" --context image
Expand All @@ -75,33 +111,6 @@ if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
fi

echo "--- Build dependencies report"
node scripts/licenses_csv_report "--csv=target/dependencies-$GIT_ABBREV_COMMIT.csv"

echo "--- Upload CDN assets"
cd target
gcloud auth activate-service-account --key-file <(echo "$GCS_SA_CDN_KEY")

CDN_ASSETS_FOLDER=$(mktemp -d)
tar -xf "kibana-$BASE_VERSION-cdn-assets.tar.gz" -C "$CDN_ASSETS_FOLDER" --strip=1

gsutil -m cp -r "$CDN_ASSETS_FOLDER/*" "gs://$GCS_SA_CDN_BUCKET/$GIT_ABBREV_COMMIT"
gcloud auth revoke "$GCS_SA_CDN_EMAIL"

echo "--- Validate CDN assets"
ts-node "$(git rev-parse --show-toplevel)/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts" \
"$GCS_SA_CDN_URL" \
"$CDN_ASSETS_FOLDER"

echo "--- Upload archives"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-linux-x86_64.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-linux-aarch64.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-image.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.gz"
buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-build-context.tar.gz"
buildkite-agent artifact upload "kibana-$BASE_VERSION-cdn-assets.tar.gz"
buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"

# This part is related with updating the configuration of kibana-controller,
# so that new stack instances contain the latest and greatest image of kibana,
# and the respective stack components of course.
Expand Down