[Enhancement] improve image build efficiency (#1191)

* improve image build efficiency

Signed-off-by: Congrool <[email protected]>

* use original dockerfiles for image release

Signed-off-by: Congrool <[email protected]>

---------

Signed-off-by: Congrool <[email protected]>

Makefile

@@ -103,26 +103,18 @@ lint: install-golint ## Run go lint against code.
 	$(GOLINT_BIN) run -v
 
 # Build the docker images only one arch(specify arch by TARGET_PLATFORMS env)
+# otherwise the platform of host will be used. 
+# e.g.
 #     - build linux/amd64 docker images:
 #       $# make docker-build TARGET_PLATFORMS=linux/amd64
 #     - build linux/arm64 docker images:
 #       $# make docker-build TARGET_PLATFORMS=linux/arm64
-docker-build: docker-build-yurthub docker-build-yurt-controller-manager docker-build-yurt-tunnel-server docker-build-yurt-tunnel-agent docker-build-node-servant
-
-docker-build-yurthub:
-	docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurthub . -t ${IMAGE_REPO}/yurthub:${GIT_VERSION}
-
-docker-build-yurt-controller-manager:
-	docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-controller-manager . -t ${IMAGE_REPO}/yurt-controller-manager:${GIT_VERSION}
-
-docker-build-yurt-tunnel-server:
-	docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-tunnel-server . -t ${IMAGE_REPO}/yurt-tunnel-server:${GIT_VERSION}
-
-docker-build-yurt-tunnel-agent:
-	docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-tunnel-agent . -t ${IMAGE_REPO}/yurt-tunnel-agent:${GIT_VERSION}
-
-docker-build-node-servant:
-	docker buildx build --no-cache --load ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION}
+#     - build a specific image:
+#       $# make docker-build WHAT=yurthub
+#     - build with proxy, maybe useful for Chinese users
+#       $# REGION=cn make docker-build
+docker-build:
+	TARGET_PLATFORMS=${TARGET_PLATFORMS} hack/make-rules/image_build.sh	$(WHAT)
 
 # Build and Push the docker images with multi-arch
 docker-push: docker-push-yurthub docker-push-yurt-controller-manager docker-push-yurt-tunnel-server docker-push-yurt-tunnel-agent docker-push-node-servant
@@ -138,16 +130,16 @@ docker-buildx-builder:
 	docker run --rm --privileged tonistiigi/binfmt --install all
 
 docker-push-yurthub: docker-buildx-builder
-	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurthub . -t ${IMAGE_REPO}/yurthub:${GIT_VERSION}
+	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurthub . -t ${IMAGE_REPO}/yurthub:${GIT_VERSION}
 
 docker-push-yurt-controller-manager: docker-buildx-builder
-	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-controller-manager . -t ${IMAGE_REPO}/yurt-controller-manager:${GIT_VERSION}
+	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-controller-manager . -t ${IMAGE_REPO}/yurt-controller-manager:${GIT_VERSION}
 
 docker-push-yurt-tunnel-server: docker-buildx-builder
-	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-tunnel-server . -t ${IMAGE_REPO}/yurt-tunnel-server:${GIT_VERSION}
+	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-server . -t ${IMAGE_REPO}/yurt-tunnel-server:${GIT_VERSION}
 
 docker-push-yurt-tunnel-agent: docker-buildx-builder
-	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-tunnel-agent . -t ${IMAGE_REPO}/yurt-tunnel-agent:${GIT_VERSION}
+	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent . -t ${IMAGE_REPO}/yurt-tunnel-agent:${GIT_VERSION}
 
 docker-push-node-servant: docker-buildx-builder
-	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/Dockerfile.yurt-node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION}
+	docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS}  --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.node-servant . -t ${IMAGE_REPO}/node-servant:${GIT_VERSION}

hack/dockerfiles/build/Dockerfile.node-servant

@@ -0,0 +1,9 @@
+# multi-arch image building for yurt-node-servant
+
+FROM --platform=${TARGETPLATFORM} alpine:3.14
+ARG TARGETOS TARGETARCH MIRROR_REPO
+RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
+    apk add ca-certificates bash libc6-compat && update-ca-certificates && rm /var/cache/apk/*
+COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-node-servant /usr/local/bin/node-servant
+COPY hack/lib/node-servant-entry.sh /usr/local/bin/entry.sh
+RUN chmod +x /usr/local/bin/entry.sh

hack/dockerfiles/build/Dockerfile.yurt-controller-manager

@@ -0,0 +1,8 @@
+# multi-arch image building for yurt-controller-manager
+
+FROM --platform=${TARGETPLATFORM} alpine:3.14
+ARG TARGETOS TARGETARCH MIRROR_REPO
+RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
+    apk add ca-certificates bash libc6-compat && update-ca-certificates && rm /var/cache/apk/*
+COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-controller-manager /usr/local/bin/yurt-controller-manager
+ENTRYPOINT ["/usr/local/bin/yurt-controller-manager"]

hack/dockerfiles/build/Dockerfile.yurt-tunnel-agent

@@ -0,0 +1,8 @@
+# multi-arch image building for yurt-tunnel-agent
+
+FROM --platform=${TARGETPLATFORM} alpine:3.14
+ARG TARGETOS TARGETARCH MIRROR_REPO
+RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
+    apk add ca-certificates bash libc6-compat && update-ca-certificates && rm /var/cache/apk/*
+COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-tunnel-agent /usr/local/bin/yurt-tunnel-agent
+ENTRYPOINT ["/usr/local/bin/yurt-tunnel-agent"]

hack/dockerfiles/build/Dockerfile.yurt-tunnel-server

@@ -0,0 +1,8 @@
+# multi-arch image building for yurt-tunnel-server
+
+FROM --platform=${TARGETPLATFORM} alpine:3.14
+ARG TARGETOS TARGETARCH MIRROR_REPO
+RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
+    apk add ca-certificates bash libc6-compat iptables ip6tables conntrack-tools && update-ca-certificates && rm /var/cache/apk/*
+COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-tunnel-server /usr/local/bin/yurt-tunnel-server
+ENTRYPOINT ["/usr/local/bin/yurt-tunnel-server"]

hack/dockerfiles/build/Dockerfile.yurthub

@@ -0,0 +1,8 @@
+# multi-arch image building for yurthub
+
+FROM --platform=${TARGETPLATFORM} alpine:3.14
+ARG TARGETOS TARGETARCH MIRROR_REPO
+RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \
+    apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/*
+COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurthub /usr/local/bin/yurthub
+ENTRYPOINT ["/usr/local/bin/yurthub"]

hack/lib/build.sh

@@ -39,7 +39,6 @@ build_binaries() {
     fi
 
     local target_bin_dir=$(get_binary_dir_with_arch ${YURT_LOCAL_BIN_DIR})
-    rm -rf ${target_bin_dir}
     mkdir -p ${target_bin_dir}
     cd ${target_bin_dir}
     for binary in "${targets[@]}"; do

hack/lib/common.sh

@@ -100,4 +100,15 @@ get_maintained_versions() {
         fi
     done
     echo $versions
+}
+
+get_image_tag() {
+    tag=$(git describe --abbrev=0 --tags)
+    commit=$(git rev-parse HEAD) 
+
+    if $(git tag --points-at ${commit}); then
+        echo ${tag}-$(echo ${commit} | cut -c 1-7)
+    else
+        echo ${tag}
+    fi
 }

hack/make-rules/build.sh

@@ -30,4 +30,8 @@ readonly YURT_ALL_TARGETS=(
     yurt-tunnel-agent
 )
 
+# clean old binaries at GOOS and GOARCH
+# eg. _output/local/bin/linux/amd64
+rm -rf $(get_binary_dir_with_arch ${YURT_LOCAL_BIN_DIR})
+
 build_binaries "$@"

hack/make-rules/image_build.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The OpenYurt Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+YURT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+source "${YURT_ROOT}/hack/lib/init.sh"
+source "${YURT_ROOT}/hack/lib/build.sh"
+
+readonly IMAGE_TARGETS=(
+    yurt-controller-manager
+    yurt-tunnel-agent 
+    yurt-tunnel-server
+    yurt-node-servant
+    yurthub
+)
+
+http_proxy=${http_proxy:-}
+https_proxy=${https_proxy:-}
+targets=${@:-${IMAGE_TARGETS[@]}}
+REGION=${REGION:-}
+IMAGE_REPO=${IMAGE_REPO:-"openyurt"}
+IMAGE_TAG=${IMAGE_TAG:-$(get_image_tag)}
+DOCKER_BUILD_ARGS=""
+BUILD_BASE_IMAGE="golang:1.17.1"
+BUILD_GOPROXY=$(go env GOPROXY)
+GOPROXY_CN="https://goproxy.cn"
+APKREPO_MIRROR_CN="mirrors.aliyun.com"
+
+if [ "${REGION}" == "cn" ]; then
+    BUILD_GOPROXY=${GOPROXY_CN}
+    DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg MIRROR_REPO=${APKREPO_MIRROR_CN}"
+fi
+
+if [[ ! -z ${http_proxy} ]]; then 
+    DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg http_proxy=${http_proxy}"
+fi
+
+if [[ ! -z ${https_proxy} ]]; then
+    DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg https_proxy=${https_proxy}"
+fi
+
+if [[ ! -z ${TARGET_PLATFORMS} ]]; then
+    IFS="/" read -r TARGETOS TARGETARCH <<< "${TARGET_PLATFORMS}"
+else
+    echo "TARGET_PLATFORMS should be specified"
+    exit -1
+fi
+
+# build binaries within docker container
+
+# It uses the following two lines:
+# --env GOCACHE=/tmp/
+# --user $(id -u ${USER}):$(id -g ${USER})
+# to enable the docker container to build binaries with the
+# same user:group as the current user:group of host.
+docker run \
+    --rm --name openyurt-build \
+    --mount type=bind,dst=/build/,src=${YURT_ROOT} \
+    --workdir=/build/ \
+    --env GOPROXY=${BUILD_GOPROXY} \
+    --env GOOS=${TARGETOS} \
+    --env GOARCH=${TARGETARCH} \
+    --env GOCACHE=/tmp/ \
+    --user $(id -u ${USER}):$(id -g ${USER}) \
+    ${BUILD_BASE_IMAGE} \
+    ./hack/make-rules/build.sh ${targets[@]}
+
+# build images
+for image in ${targets[@]}; do
+    # The image name and binary name of node-servant
+    # are not same. So we have to this translation.
+    if [[ ${image} == "yurt-node-servant" ]]; then
+        image="node-servant"
+    fi 
+
+    docker buildx build \
+    --no-cache \
+    --load ${DOCKER_BUILD_ARGS} \
+    --platform ${TARGET_PLATFORMS} \
+    --file ${YURT_ROOT}/hack/dockerfiles/build/Dockerfile.${image} \
+    --tag ${IMAGE_REPO}/${image}:${IMAGE_TAG} \
+    ${YURT_ROOT}
+done

hack/make-rules/local-up-openyurt.sh

@@ -120,7 +120,7 @@ function local_up_openyurt {
 }
 
 function cleanup {
-    rm -rf ${YURT_ROOT}/_output
+    rm -rf "$(get_binary_dir_with_arch ${YURT_LOCAL_BIN_DIR})/yurtctl"
     kind delete clusters ${CLUSTER_NAME}
 }