From 50a0850d116a2094f469b7bb5b474cde5483bcce Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 12 Dec 2023 19:59:43 +0800 Subject: [PATCH] build(images): append branch name to the tag of each build-env-* image to support building image for each branch (#1763) Since build-env-* images may vary for different releases, they should be built for released and master branches. We need to append branch name to the tag to create image for each branch. --- .github/workflows/build-push-env-docker.yml | 3 +-- .github/workflows/thirdparty-regular-push.yml | 2 ++ docker/README.md | 16 ++++++------ docker/ci-env/Dockerfile | 2 +- docker/pegasus-docker-compose/README.md | 2 +- docker/pegasus-docker-compose/build_docker.sh | 26 ++++++++++--------- .../image_for_prebuilt_bin/Dockerfile | 3 ++- docker/thirdparties-bin/Dockerfile | 4 +-- docker/thirdparties-src/Dockerfile | 5 ++-- 9 files changed, 34 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-push-env-docker.yml b/.github/workflows/build-push-env-docker.yml index 756ebad072..30181a7c98 100644 --- a/.github/workflows/build-push-env-docker.yml +++ b/.github/workflows/build-push-env-docker.yml @@ -27,7 +27,6 @@ on: paths: - 'docker/pegasus-build-env/**' - '.github/workflows/build-push-env-docker.yml' - - 'thirdparty/**' # for manually triggering workflow workflow_dispatch: @@ -63,6 +62,6 @@ jobs: file: ./docker/pegasus-build-env/${{ matrix.dockertag }}/Dockerfile push: true tags: | - apache/pegasus:build-env-${{ matrix.dockertag }} + apache/pegasus:build-env-${{ matrix.dockertag }}-${{ github.ref_name }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 280d5caec9..a2573b2193 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -25,8 +25,10 @@ on: - ci-test # testing branch for github action - '*dev' # developing branch paths: + - 'docker/pegasus-build-env/**' - 'docker/thirdparties-src/**' - 'docker/thirdparties-bin/**' + - '.github/workflows/build-push-env-docker.yml' - '.github/workflows/thirdparty-regular-push.yml' - 'thirdparty/**' diff --git a/docker/README.md b/docker/README.md index 1021504390..af5e325100 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,10 +35,10 @@ Building environment for Pegasus compilation. Github Actions automatically rebuilds and publishes build-env for every commit. -- `apache/pegasus:build-env-centos7` -- `apache/pegasus:build-env-ubuntu1604` -- `apache/pegasus:build-env-ubuntu1804` -- `apache/pegasus:build-env-ubuntu2004` +- `apache/pegasus:build-env-centos7-` +- `apache/pegasus:build-env-ubuntu1604-` +- `apache/pegasus:build-env-ubuntu1804-` +- `apache/pegasus:build-env-ubuntu2004-` DockerHub: https://hub.docker.com/r/apache/pegasus @@ -58,7 +58,7 @@ without downloading from the cloud object storage. This is a Docker image for Pegasus unit-testing. It prebuilts the thirdparty libraries, so jobs based on this image can skip building third-parties. -- `apache/pegasus:thirdparties-bin-centos7-master` -- `apache/pegasus:thirdparties-bin-ubuntu1604-master` -- `apache/pegasus:thirdparties-bin-ubuntu1804-master` -- `apache/pegasus:thirdparties-bin-ubuntu2004-master` +- `apache/pegasus:thirdparties-bin-centos7-` +- `apache/pegasus:thirdparties-bin-ubuntu1604-` +- `apache/pegasus:thirdparties-bin-ubuntu1804-` +- `apache/pegasus:thirdparties-bin-ubuntu2004-` diff --git a/docker/ci-env/Dockerfile b/docker/ci-env/Dockerfile index 295a749ffd..89a54cdc39 100644 --- a/docker/ci-env/Dockerfile +++ b/docker/ci-env/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM apache/pegasus:build-env-centos7 +FROM apache/pegasus:build-env-centos7-master WORKDIR /root diff --git a/docker/pegasus-docker-compose/README.md b/docker/pegasus-docker-compose/README.md index 7d84672dc8..fb0a1404c4 100644 --- a/docker/pegasus-docker-compose/README.md +++ b/docker/pegasus-docker-compose/README.md @@ -39,7 +39,7 @@ Then build the image: ``` cd /your/local/pegasus-docker/pegasus-docker-compose -./build_docker.sh /your/local/apache-pegasus-source +./build_docker.sh /your/local/apache-pegasus-source github-branch-for-build-env-image(default: master) ``` You will have a docker image called "pegasus:latest" right now built on you machine. Check it out: diff --git a/docker/pegasus-docker-compose/build_docker.sh b/docker/pegasus-docker-compose/build_docker.sh index f665187c70..4a4d4cadfd 100755 --- a/docker/pegasus-docker-compose/build_docker.sh +++ b/docker/pegasus-docker-compose/build_docker.sh @@ -18,14 +18,11 @@ #!/bin/bash -# Usage: -# ./build_docker.sh /your/local/apache-pegasus-source - set -e -if [[ $# -ne 1 ]]; then - echo "ERROR: must specify /your/local/apache-pegasus-source" - exit 1 +if [[ $# -lt 1 ]]; then + echo "USAGE: $0 /your/local/apache-pegasus-source [github-branch-for-build-env-image(default: master)]" + exit 1 fi # ROOT is where the script is. @@ -34,15 +31,20 @@ ROOT=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") # PEGASUS_ROOT is the absolute path of apache-pegasus-source PEGASUS_ROOT="$(readlink -f "$1")" if [[ ! -f "${PEGASUS_ROOT}"/PACKAGE ]]; then - echo "ERROR: no such file ${PEGASUS_ROOT}/PACKAGE" - exit 1 + echo "ERROR: no such file ${PEGASUS_ROOT}/PACKAGE" + exit 1 fi SERVER_PKG_NAME=$(cat "${PEGASUS_ROOT}"/PACKAGE) if [[ ! -f "${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" ]]; then - echo "Failed to find package ${SERVER_PKG_NAME}.tar.gz in ${PEGASUS_ROOT}" - exit 1 + echo "Failed to find package ${SERVER_PKG_NAME}.tar.gz in ${PEGASUS_ROOT}" + exit 1 else - echo "Found package ${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" + echo "Found package ${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" +fi + +GITHUB_BRANCH=master +if [[ $# -ge 2 ]]; then + GITHUB_BRANCH=$2 fi ### @@ -54,5 +56,5 @@ echo "Building image ${IMAGE_NAME}" cd "${ROOT}"/image_for_prebuilt_bin || exit 1 cp "${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" . -docker build --build-arg SERVER_PKG_NAME="${SERVER_PKG_NAME}" -t ${IMAGE_NAME} . +docker build --build-arg SERVER_PKG_NAME="${SERVER_PKG_NAME}" --build-arg GITHUB_BRANCH="{GITHUB_BRANCH}" -t ${IMAGE_NAME} . rm "${SERVER_PKG_NAME}.tar.gz" diff --git a/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile b/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile index 1e3369a1ed..a61d4cc400 100644 --- a/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile +++ b/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -FROM apache/pegasus:build-env-ubuntu2004 +ARG GITHUB_BRANCH=master +FROM apache/pegasus:build-env-ubuntu2004-${GITHUB_BRANCH} ARG SERVER_PKG_NAME diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index eda3eddfdb..720e724c81 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -18,13 +18,13 @@ ARG GITHUB_BRANCH=master ARG OS_VERSION=centos7 FROM apache/pegasus:thirdparties-src-${GITHUB_BRANCH} as builder -FROM apache/pegasus:build-env-${OS_VERSION} +FROM apache/pegasus:build-env-${OS_VERSION}-${GITHUB_BRANCH} WORKDIR /root COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip -ARG GITHUB_BRANCH=master +ARG GITHUB_BRANCH ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \ && cd incubator-pegasus/thirdparty \ diff --git a/docker/thirdparties-src/Dockerfile b/docker/thirdparties-src/Dockerfile index 48070659e6..64e64c2940 100644 --- a/docker/thirdparties-src/Dockerfile +++ b/docker/thirdparties-src/Dockerfile @@ -15,11 +15,12 @@ # specific language governing permissions and limitations # under the License. -FROM apache/pegasus:build-env-centos7 as builder +ARG GITHUB_BRANCH=master +FROM apache/pegasus:build-env-centos7-${GITHUB_BRANCH} as builder WORKDIR /root -ARG GITHUB_BRANCH=master +ARG GITHUB_BRANCH ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL}