From 352689f890b44467a944a5a39bdfaca4e7ae1fc2 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 10:44:42 +0800 Subject: [PATCH 1/8] [CI] Free up disk space for K8s IT --- .github/scripts/free_disk_space.sh | 55 ++++++++++++++++++++++++++++++ .github/workflows/master.yml | 4 +++ 2 files changed, 59 insertions(+) create mode 100755 .github/scripts/free_disk_space.sh diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh new file mode 100755 index 00000000000..1855fdd6315 --- /dev/null +++ b/.github/scripts/free_disk_space.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script is copied from Apache Spark + +echo "==================================" +echo "Free up disk space on CI system" +echo "==================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +sudo rm -rf /usr/share/dotnet/ +sudo rm -rf /usr/share/php/ +sudo rm -rf /usr/local/graalvm/ +sudo rm -rf /usr/local/.ghcup/ +sudo rm -rf /usr/local/share/powershell +sudo rm -rf /usr/local/share/chromium +sudo rm -rf /usr/local/lib/android +sudo rm -rf /usr/local/lib/node_modules + +sudo rm -rf /opt/az +sudo rm -rf /opt/hostedtoolcache/CodeQL +sudo rm -rf /opt/hostedtoolcache/go +sudo rm -rf /opt/hostedtoolcache/node + +sudo apt-get remove --purge -y '^aspnet.*' +sudo apt-get remove --purge -y '^dotnet-.*' +sudo apt-get remove --purge -y '^llvm-.*' +sudo apt-get remove --purge -y 'php.*' +sudo apt-get remove --purge -y '^mongodb-.*' +sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox +sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri +sudo apt-get autoremove --purge -y +sudo apt-get clean + +df -h diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b1083cfa218..6a44ffb9df2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -400,6 +400,8 @@ jobs: name: Kyuubi Server On Kubernetes Integration Test runs-on: ubuntu-22.04 steps: + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Checkout uses: actions/checkout@v4 # https://github.com/docker/build-push-action @@ -472,6 +474,8 @@ jobs: name: Spark Engine On Kubernetes Integration Test runs-on: ubuntu-22.04 steps: + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Checkout uses: actions/checkout@v4 - name: Cache Engine Archives From 30b82cdf032aeea4e1ef610929b359e618833faf Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 10:49:34 +0800 Subject: [PATCH 2/8] fix --- .github/workflows/master.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6a44ffb9df2..c68b0f57f35 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -400,10 +400,9 @@ jobs: name: Kyuubi Server On Kubernetes Integration Test runs-on: ubuntu-22.04 steps: + - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - - name: Checkout - uses: actions/checkout@v4 # https://github.com/docker/build-push-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -474,10 +473,9 @@ jobs: name: Spark Engine On Kubernetes Integration Test runs-on: ubuntu-22.04 steps: + - uses: actions/checkout@v4 - name: Free up disk space run: ./.github/scripts/free_disk_space.sh - - name: Checkout - uses: actions/checkout@v4 - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Setup Minikube From 47f90b95e6f590dcc259e597a258f8df27ba7f26 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 11:00:00 +0800 Subject: [PATCH 3/8] nit --- .github/scripts/free_disk_space.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh index 1855fdd6315..6333cf05a29 100755 --- a/.github/scripts/free_disk_space.sh +++ b/.github/scripts/free_disk_space.sh @@ -46,7 +46,7 @@ sudo apt-get remove --purge -y '^aspnet.*' sudo apt-get remove --purge -y '^dotnet-.*' sudo apt-get remove --purge -y '^llvm-.*' sudo apt-get remove --purge -y 'php.*' -sudo apt-get remove --purge -y '^mongodb-.*' +sudo apt-get remove --purge -y '^temurin-.*' sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri sudo apt-get autoremove --purge -y From 9d1ec4e447feef3fe9408728e5ed864668380155 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 11:04:18 +0800 Subject: [PATCH 4/8] extend --- .github/workflows/gluten.yml | 4 ++++ .github/workflows/master.yml | 14 ++++++++++++++ .github/workflows/publish-snapshot-docker.yml | 5 +++-- .github/workflows/publish-snapshot-nexus.yml | 5 +++-- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gluten.yml b/.github/workflows/gluten.yml index 45f4edf545e..d04c1ff212c 100644 --- a/.github/workflows/gluten.yml +++ b/.github/workflows/gluten.yml @@ -30,6 +30,8 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Update and Upgrade @@ -79,6 +81,8 @@ jobs: spark: [ '3.4', '3.3' ] steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Update and Upgrade diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c68b0f57f35..42ac6e37cfc 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -87,6 +87,8 @@ jobs: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -144,6 +146,8 @@ jobs: - '3.4' steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -200,6 +204,8 @@ jobs: comment: 'verify-on-flink-1.18-binary' steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -256,6 +262,8 @@ jobs: comment: 'verify-on-hive-2.1-cdh6-binary' steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -302,6 +310,8 @@ jobs: comment: [ "normal" ] steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -341,6 +351,8 @@ jobs: comment: [ "normal" ] steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -375,6 +387,8 @@ jobs: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK 8 diff --git a/.github/workflows/publish-snapshot-docker.yml b/.github/workflows/publish-snapshot-docker.yml index 0a73dcc2da5..a7df6135ffc 100644 --- a/.github/workflows/publish-snapshot-docker.yml +++ b/.github/workflows/publish-snapshot-docker.yml @@ -27,8 +27,9 @@ jobs: if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-22.04 steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx diff --git a/.github/workflows/publish-snapshot-nexus.yml b/.github/workflows/publish-snapshot-nexus.yml index 64dd1a690b1..2be08301fea 100644 --- a/.github/workflows/publish-snapshot-nexus.yml +++ b/.github/workflows/publish-snapshot-nexus.yml @@ -42,10 +42,11 @@ jobs: - branch: branch-1.8 profiles: -Pflink-provided,spark-provided,hive-provided,spark-3.4 steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Setup JDK 8 uses: actions/setup-java@v4 with: From ab4c3e461083a3ae22d51b3512a4688ec6f1c287 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 11:11:29 +0800 Subject: [PATCH 5/8] keep jdk8 --- .github/scripts/free_disk_space.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh index 6333cf05a29..204a14ca0f8 100755 --- a/.github/scripts/free_disk_space.sh +++ b/.github/scripts/free_disk_space.sh @@ -46,7 +46,7 @@ sudo apt-get remove --purge -y '^aspnet.*' sudo apt-get remove --purge -y '^dotnet-.*' sudo apt-get remove --purge -y '^llvm-.*' sudo apt-get remove --purge -y 'php.*' -sudo apt-get remove --purge -y '^temurin-.*' +sudo apt-get remove --purge -y '^temurin-\d{n,}.*' sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri sudo apt-get autoremove --purge -y From a9c8b035cf54c3516f198346c0906af3b1e8e9f4 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 11:12:42 +0800 Subject: [PATCH 6/8] nit --- .github/scripts/free_disk_space.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh index 204a14ca0f8..f4094aebad2 100755 --- a/.github/scripts/free_disk_space.sh +++ b/.github/scripts/free_disk_space.sh @@ -45,10 +45,11 @@ sudo rm -rf /opt/hostedtoolcache/node sudo apt-get remove --purge -y '^aspnet.*' sudo apt-get remove --purge -y '^dotnet-.*' sudo apt-get remove --purge -y '^llvm-.*' +sudo apt-get remove --purge -y 'mono.*' sudo apt-get remove --purge -y 'php.*' sudo apt-get remove --purge -y '^temurin-\d{n,}.*' sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox -sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel powershell libgl1-mesa-dri +sudo apt-get remove --purge -y azure-cli google-cloud-sdk msbuild powershell libgl1-mesa-dri sudo apt-get autoremove --purge -y sudo apt-get clean From 8c78115a4ff35b7e7ce434cc124697371f2c4c89 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 11:13:04 +0800 Subject: [PATCH 7/8] nit --- .github/scripts/free_disk_space.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh index f4094aebad2..886c2544929 100755 --- a/.github/scripts/free_disk_space.sh +++ b/.github/scripts/free_disk_space.sh @@ -17,7 +17,7 @@ # limitations under the License. # -# This script is copied from Apache Spark +# This script is inspired by Apache Spark echo "==================================" echo "Free up disk space on CI system" From 272df2790f5e9e9afa811b489c135d0b7c25aefd Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 11:20:30 +0800 Subject: [PATCH 8/8] nit --- .github/scripts/free_disk_space.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh index 886c2544929..b51dba8aa9b 100755 --- a/.github/scripts/free_disk_space.sh +++ b/.github/scripts/free_disk_space.sh @@ -45,11 +45,10 @@ sudo rm -rf /opt/hostedtoolcache/node sudo apt-get remove --purge -y '^aspnet.*' sudo apt-get remove --purge -y '^dotnet-.*' sudo apt-get remove --purge -y '^llvm-.*' -sudo apt-get remove --purge -y 'mono.*' sudo apt-get remove --purge -y 'php.*' sudo apt-get remove --purge -y '^temurin-\d{n,}.*' sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox -sudo apt-get remove --purge -y azure-cli google-cloud-sdk msbuild powershell libgl1-mesa-dri +sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel msbuild powershell libgl1-mesa-dri sudo apt-get autoremove --purge -y sudo apt-get clean