From 9740b75a3facaf7d2c92a067db9f0b858ad5c4a5 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 5 Feb 2024 12:12:15 +0800 Subject: [PATCH] [KYUUBI #6047] [CI] Free up disk space I saw [the K8s IT](https://github.com/apache/kyuubi/actions/runs/7776268590/job/21203214430?pr=6045) failed with ``` Error: No space left on device : '/home/runner/runners/2.312.0/_diag/pages/131e7057-723d-4281-b296-b876237f4318_21774d93-c106-57cd-810b-f0e6bf96d865_1.log' ``` Borrow the solution from Apache Spark, just uninstall the un-used large pre-installed packages. - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) Wait CI report --- - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6047 from pan3793/ci-space. Closes #6047 272df2790 [Cheng Pan] nit 8c78115a4 [Cheng Pan] nit a9c8b035c [Cheng Pan] nit ab4c3e461 [Cheng Pan] keep jdk8 9d1ec4e44 [Cheng Pan] extend 47f90b95e [Cheng Pan] nit 30b82cdf0 [Cheng Pan] fix 352689f89 [Cheng Pan] [CI] Free up disk space for K8s IT Authored-by: Cheng Pan Signed-off-by: Cheng Pan (cherry picked from commit 5f99ec8d6a9336d0bd2b3f9db83d0deaacf71e35) Signed-off-by: Cheng Pan --- .github/scripts/free_disk_space.sh | 55 +++++++++++++++++++ .github/workflows/master.yml | 24 ++++++-- .github/workflows/publish-snapshot-docker.yml | 5 +- .github/workflows/publish-snapshot-nexus.yml | 5 +- 4 files changed, 81 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/free_disk_space.sh diff --git a/.github/scripts/free_disk_space.sh b/.github/scripts/free_disk_space.sh new file mode 100755 index 00000000000..b51dba8aa9b --- /dev/null +++ b/.github/scripts/free_disk_space.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script is inspired by Apache Spark + +echo "==================================" +echo "Free up disk space on CI system" +echo "==================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +sudo rm -rf /usr/share/dotnet/ +sudo rm -rf /usr/share/php/ +sudo rm -rf /usr/local/graalvm/ +sudo rm -rf /usr/local/.ghcup/ +sudo rm -rf /usr/local/share/powershell +sudo rm -rf /usr/local/share/chromium +sudo rm -rf /usr/local/lib/android +sudo rm -rf /usr/local/lib/node_modules + +sudo rm -rf /opt/az +sudo rm -rf /opt/hostedtoolcache/CodeQL +sudo rm -rf /opt/hostedtoolcache/go +sudo rm -rf /opt/hostedtoolcache/node + +sudo apt-get remove --purge -y '^aspnet.*' +sudo apt-get remove --purge -y '^dotnet-.*' +sudo apt-get remove --purge -y '^llvm-.*' +sudo apt-get remove --purge -y 'php.*' +sudo apt-get remove --purge -y '^temurin-\d{n,}.*' +sudo apt-get remove --purge -y snapd google-chrome-stable microsoft-edge-stable firefox +sudo apt-get remove --purge -y azure-cli google-cloud-sdk mono-devel msbuild powershell libgl1-mesa-dri +sudo apt-get autoremove --purge -y +sudo apt-get clean + +df -h diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9cd6b756d9e..0682bd20347 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -87,6 +87,8 @@ jobs: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -187,6 +189,8 @@ jobs: - '3.4' steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -243,6 +247,8 @@ jobs: comment: 'verify-on-flink-1.18-binary' steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -296,6 +302,8 @@ jobs: comment: 'verify-on-hive-2.3-binary' steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -342,6 +350,8 @@ jobs: comment: [ "normal" ] steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -381,6 +391,8 @@ jobs: comment: [ "normal" ] steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -415,6 +427,8 @@ jobs: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK 8 @@ -440,8 +454,9 @@ jobs: name: Kyuubi Server On Kubernetes Integration Test runs-on: ubuntu-22.04 steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh # https://github.com/docker/build-push-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -512,8 +527,9 @@ jobs: name: Spark Engine On Kubernetes Integration Test runs-on: ubuntu-22.04 steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - name: Setup Minikube diff --git a/.github/workflows/publish-snapshot-docker.yml b/.github/workflows/publish-snapshot-docker.yml index 3afccee7aa8..a672288bbfb 100644 --- a/.github/workflows/publish-snapshot-docker.yml +++ b/.github/workflows/publish-snapshot-docker.yml @@ -27,8 +27,9 @@ jobs: if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-22.04 steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx diff --git a/.github/workflows/publish-snapshot-nexus.yml b/.github/workflows/publish-snapshot-nexus.yml index 6ef933b52b0..0c8c6aef768 100644 --- a/.github/workflows/publish-snapshot-nexus.yml +++ b/.github/workflows/publish-snapshot-nexus.yml @@ -39,10 +39,11 @@ jobs: - branch: master profiles: -Pflink-provided,spark-provided,hive-provided,spark-3.4 steps: - - name: Checkout repository - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: ref: ${{ matrix.branch }} + - name: Free up disk space + run: ./.github/scripts/free_disk_space.sh - name: Setup JDK 8 uses: actions/setup-java@v3 with: