From ef48953fc30971af904cc49bd05acda3da24a834 Mon Sep 17 00:00:00 2001 From: Nitin Garg <113666283+gargnitingoogle@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:21:28 +0530 Subject: [PATCH] Upgrade to go 1.22.1 (with fix for dockerfiles) (#1764) * Upgrade to golang 1.22 (without dockerfile fix) Reapply "Upgrade to golang 1.22 (#1753)" (#1766) This reverts commit a63a46fdbecbcfa58591e76a9bf49af6239ae7f1 and fixes merge conflicts with golang 1.21.7 upgrade which went in earlier. * Fix containerize_gcsfuse/dockerfile for go 1.22 Does the following 1. Create a go module (go.mod) in a directory package_gcsfuse, to bypass the change in go get behaviour introduced in 1.17 and enforced in go 1.22. 2. Fix some paths and commands etc. accordingly in the dockerfile. * Fix package_gcsfuse/dockerfile for go 1.22 Does the following 1. Create a go module (go.mod) in a directory package_gcsfuse, to bypass the change in go get behaviour introduced in 1.17 and enforced in go 1.22. 2. Fix some paths and commands etc. accordingly in the dockerfile. * Switch containerize*/Dockerfile to use git clone * Switch package*/Dockerfile to use git clone * Replace /go with ${GOPATH} * Minimize run of git clone Fetch only the last commit state in docker files during git clone to reduce its runtime. This reduces the runtime of git clone dramatically. * Upgrade from go 1.22.0 to 1.22.1 --- .github/workflows/ci.yml | 4 ++-- Dockerfile | 2 +- go.mod | 2 +- .../scripts/ml_tests/pytorch/run_model.sh | 2 +- .../resnet/setup_scripts/setup_container.sh | 4 ++-- .../presubmit_test/pr_perf_test/build.sh | 4 ++-- perfmetrics/scripts/read_cache/setup.sh | 2 +- perfmetrics/scripts/run_e2e_tests.sh | 4 ++-- tools/cd_scripts/e2e_test.sh | 2 +- tools/containerize_gcsfuse_docker/Dockerfile | 16 +++++++------- tools/package_gcsfuse_docker/Dockerfile | 21 ++++++++----------- 11 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21c8eb402f..b26582df77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: linux-tests: strategy: matrix: - go: [ 1.21.x ] + go: [ 1.22.x ] runs-on: ubuntu-20.04 timeout-minutes: 15 @@ -40,7 +40,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: "1.21" + go-version: "1.22" - name: checkout code uses: actions/checkout@v3 - name: golangci-lint diff --git a/Dockerfile b/Dockerfile index 9ff01e6cd9..b57c83ca0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # Mount the gcsfuse to /mnt/gcs: # > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse -FROM golang:1.21.7-alpine as builder +FROM golang:1.22.1-alpine as builder RUN apk add git diff --git a/go.mod b/go.mod index 336379318d..063c472bab 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/googlecloudplatform/gcsfuse -go 1.21 +go 1.22.1 require ( cloud.google.com/go/compute/metadata v0.2.3 diff --git a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh index 7efd0cf1f8..0e8c004c84 100755 --- a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh +++ b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh @@ -18,7 +18,7 @@ NUM_EPOCHS=80 TEST_BUCKET="gcsfuse-ml-data" # Install golang -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-amd64.tar.gz -q +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -q rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz export PATH=$PATH:/usr/local/go/bin diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh index df7afd1a7c..d0f70bc2c9 100755 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Installs go1.21 on the container, builds gcsfuse using log_rotation file +# Installs go1.22.1 on the container, builds gcsfuse using log_rotation file # and installs tf-models-official v2.13.2, makes update to include clear_kernel_cache # and epochs functionality, and runs the model # Install go lang -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-amd64.tar.gz -q +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 0a881ca34f..bef1bd7827 100755 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -34,8 +34,8 @@ set -e sudo apt-get update echo Installing git sudo apt-get install git -echo Installing go-lang 1.21.7 -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-amd64.tar.gz -q +echo Installing go-lang 1.22.1 +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin export CGO_ENABLED=0 diff --git a/perfmetrics/scripts/read_cache/setup.sh b/perfmetrics/scripts/read_cache/setup.sh index 7e77b3d985..2e096cb606 100755 --- a/perfmetrics/scripts/read_cache/setup.sh +++ b/perfmetrics/scripts/read_cache/setup.sh @@ -64,7 +64,7 @@ sed -i 's/define \+FIO_IO_U_PLAT_GROUP_NR \+\([0-9]\+\)/define FIO_IO_U_PLAT_GRO cd - # Install and validate go. -version=1.21.7 +version=1.22.1 wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go tar -xzf go_tar.tar.gz && sudo mv go /usr/local diff --git a/perfmetrics/scripts/run_e2e_tests.sh b/perfmetrics/scripts/run_e2e_tests.sh index b9a6fa731f..c0a03ede39 100755 --- a/perfmetrics/scripts/run_e2e_tests.sh +++ b/perfmetrics/scripts/run_e2e_tests.sh @@ -43,8 +43,8 @@ upgrade_gcloud_version # e.g. architecture=arm64 or amd64 architecture=$(dpkg --print-architecture) -echo "Installing go-lang 1.21.7..." -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-${architecture}.tar.gz -q +echo "Installing go-lang 1.22.1..." +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-${architecture}.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin # install python3-setuptools tools. diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index b7f3b7960f..fa1a0ebea2 100755 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -111,7 +111,7 @@ else fi # install go -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-${architecture}.tar.gz +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-${architecture}.tar.gz sudo tar -C /usr/local -xzf go_tar.tar.gz export PATH=${PATH}:/usr/local/go/bin #Write gcsfuse and go version to log file diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index dd9a0b1af5..e683cfa04d 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -34,7 +34,7 @@ ARG OS_VERSION ARG OS_NAME # Image with gcsfuse installed and its package (.deb) -FROM golang:1.21.7 as gcsfuse-package +FROM golang:1.22.1 as gcsfuse-package RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler @@ -43,21 +43,21 @@ ENV GOOS=linux ENV GO111MODULE=auto ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse/" -ENV GCSFUSE_PATH "$GOPATH/src/$GCSFUSE_REPO" -RUN go get -d ${GCSFUSE_REPO} +ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" +ARG BRANCH_NAME="v${GCSFUSE_VERSION}" +RUN git clone --branch ${BRANCH_NAME} --depth 1 ${GCSFUSE_REPO} +ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} -# Branch name for building through a particular branch. -ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git checkout "${BRANCH_NAME}" # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile ARG GCSFUSE_BIN="/gcsfuse" WORKDIR ${GOPATH} -RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse + +WORKDIR ${GCSFUSE_PATH}/tools/build_gcsfuse +RUN go install RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 4332093bb6..e47b9dff38 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -17,7 +17,7 @@ # Copy the gcsfuse packages to the host: # > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output -FROM golang:1.21.7 as builder +FROM golang:1.22.1 as builder RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler @@ -26,19 +26,16 @@ ENV GOOS=linux ENV GO111MODULE=auto ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse/" -ENV GCSFUSE_PATH "$GOPATH/src/$GCSFUSE_REPO" -RUN go get -d ${GCSFUSE_REPO} +ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" +ARG BRANCH_NAME="v${GCSFUSE_VERSION}" +RUN git clone --branch ${BRANCH_NAME} --depth 1 ${GCSFUSE_REPO} +ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} + ARG DEBEMAIL="gcs-fuse-maintainers@google.com" ARG DEBFULLNAME="GCSFuse Team" -# Build Arg for building through a particular branch/commit. By default, it uses -# the tag corresponding to passed GCSFUSE VERSION -ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git checkout "${BRANCH_NAME}" - # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile @@ -49,14 +46,14 @@ RUN if [ "${ARCHITECTURE}" != "amd64" ] && [ "${ARCHITECTURE}" != "arm64" ]; the fi ARG GCSFUSE_BIN="/gcsfuse_${GCSFUSE_VERSION}_${ARCHITECTURE}" ARG GCSFUSE_DOC="${GCSFUSE_BIN}/usr/share/doc/gcsfuse" -WORKDIR ${GOPATH} -RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse +WORKDIR ${GCSFUSE_PATH}/tools/build_gcsfuse +RUN go install RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin # Creating structure for debian package as we are using 'dpkg-deb --build' to create debian package -RUN mkdir -p ${GCSFUSE_BIN}/DEBIAN && cp $GOPATH/src/$GCSFUSE_REPO/DEBIAN/* ${GCSFUSE_BIN}/DEBIAN/ +RUN mkdir -p ${GCSFUSE_BIN}/DEBIAN && cp ${GCSFUSE_PATH}/DEBIAN/* ${GCSFUSE_BIN}/DEBIAN/ RUN mkdir -p ${GCSFUSE_DOC} RUN mv ${GCSFUSE_BIN}/DEBIAN/copyright ${GCSFUSE_DOC} && \ mv ${GCSFUSE_BIN}/DEBIAN/changelog ${GCSFUSE_DOC} && \