[GHA] use pigz with tar (openvinotoolkit#26954)

### Details: - Use parallel packaging and extraction to speed up workflows ### Tickets: - *ticket-id*
luweizhou2016 · Oct 8, 2024 · ed53a64 · ed53a64
1 parent 2efd2b1
commit ed53a64
Show file tree

Hide file tree

Showing 38 changed files with 85 additions and 55 deletions.
diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag
@@ -1 +1 @@
-pr-26586
+pr-26954
diff --git a/.github/dockerfiles/ov_build/debian_10_arm/Dockerfile b/.github/dockerfiles/ov_build/debian_10_arm/Dockerfile
@@ -19,6 +19,8 @@ RUN apt-get update && \
         git \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Pythons \
         python3 \
         python3-pip \

diff --git a/.github/dockerfiles/ov_build/fedora_29/Dockerfile b/.github/dockerfiles/ov_build/fedora_29/Dockerfile
@@ -18,6 +18,8 @@ RUN yum update -y && yum install -y \
     tar \
     gcc \
     gcc-c++ \
+    # parallel gzip
+    pigz \
     xz
 
 # Install build dependencies

diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_arm64/Dockerfile
@@ -23,6 +23,8 @@ RUN apt-get update && \
         ca-certificates \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Pythons
         python3.8-dev \
         python3.8-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile
@@ -22,6 +22,8 @@ RUN apt-get update && \
         git \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Pythons
         python3.9-dev \
         python3.9-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile
@@ -24,6 +24,8 @@ RUN apt-get update && \
         ca-certificates \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Pythons
         python3.8-dev \
         python3.8-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile
@@ -30,6 +30,8 @@ RUN apt-get update && \
         ca-certificates \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python \
         python3-dev \
         python3-pip \

diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile
@@ -23,6 +23,8 @@ RUN apt-get update && \
         gpg-agent \
         tzdata \
         libtbb2 \
+        # parallel gzip
+        pigz \
         # Pythons \
         python3.11-dev \
         python3.11-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_cc/Dockerfile
@@ -24,6 +24,8 @@ RUN apt-get update && \
         gpg-agent \
         tzdata \
         libtbb2 \
+        # parallel gzip
+        pigz \
         # Pythons
         python3.8-dev \
         python3.8-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_dpcpp/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_dpcpp/Dockerfile
@@ -25,6 +25,8 @@ RUN apt-get update && \
         gpg-agent \
         tzdata \
         libtbb2 \
+        # parallel gzip
+        pigz \
         # Pythons \
         python3.11-dev \
         python3.11-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_nvidia/Dockerfile
@@ -23,6 +23,8 @@ RUN apt-get update && \
         git \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python
         python3.11-dev \
         python3.11-venv \

diff --git a/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_24_04_x64/Dockerfile
@@ -22,6 +22,8 @@ RUN apt-get update && \
         git \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python
         python3-dev \
         python3-venv \

diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile
@@ -23,6 +23,8 @@ RUN apt-get update && \
         ca-certificates \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python
         python3.11-dev \
         python3.11-venv \

diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile
@@ -22,6 +22,8 @@ RUN apt-get update && \
         git \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python
         python3.9-dev \
         python3.9-venv \

diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile
@@ -23,6 +23,8 @@ RUN apt-get update && \
         ca-certificates \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python
         python3.11-dev \
         python3.11-venv \

diff --git a/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_24_04_x64/Dockerfile
@@ -20,6 +20,8 @@ RUN apt-get update && \
         git \
         gpg-agent \
         tzdata \
+        # parallel gzip
+        pigz \
         # Python
         python3-full \
         libhdf5-dev \

diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml
@@ -48,7 +48,7 @@ jobs:
         run: |
           # install doxygen
           wget https://www.doxygen.nl/files/doxygen-$DOXY_VER.linux.bin.tar.gz
-          tar -xzf doxygen-$DOXY_VER.linux.bin.tar.gz
+          tar -I pigz -xf doxygen-$DOXY_VER.linux.bin.tar.gz
           echo "$(pwd)/doxygen-$DOXY_VER/bin/" >> $GITHUB_PATH
 
       - name: Validate benchmarks files

diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -103,7 +103,7 @@ jobs:
           rm -rf ${COVERITY_TOOL_DIR} && mkdir -p ${COVERITY_TOOL_DIR}
           pushd ${COVERITY_TOOL_DIR}
             wget https://scan.coverity.com/download/linux64 --progress=bar:force:noscroll --post-data "token=${{ secrets.COVERITY_TOKEN }}&project=openvino" -O coverity_tool.tgz
-            tar xvf coverity_tool.tgz && rm coverity_tool.tgz
+            tar -I pigz -xf coverity_tool.tgz && rm coverity_tool.tgz
           popd
 
       - name: Cmake build - OpenVINO with Coverity
@@ -112,7 +112,7 @@ jobs:
       - name: Pack Artefacts
         run: |
           pushd ${BUILD_DIR}
-            tar -C ${BUILD_DIR} -czvf openvino.tgz cov-int
+            tar -C ${BUILD_DIR} -I pigz -cvf openvino.tgz cov-int
           popd
 
       - name: Submit artefacts

diff --git a/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml b/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml
@@ -158,11 +158,11 @@ jobs:
         run: |
 
           pushd ${INSTALL_DIR}
-            tar -czvf ${BUILD_DIR}/openvino_package.tar.gz *
+            tar -I pigz -cvf ${BUILD_DIR}/openvino_package.tar.gz *
           popd
 
           pushd ${INSTALL_TEST_DIR}
-            tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz *
+            tar -I pigz -cvf ${BUILD_DIR}/openvino_tests.tar.gz *
           popd
 
       #
@@ -230,11 +230,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
 
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Snippets func tests
@@ -287,11 +287,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
 
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Fetch setup_python action

diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml
@@ -189,15 +189,15 @@ jobs:
           cp -R ${ONNX_RUNTIME_UTILS} ${INSTALL_DIR}
 
           pushd ${INSTALL_DIR}
-            tar -czvf ${BUILD_DIR}/openvino_package.tar.gz *
+            tar -I pigz -cvf ${BUILD_DIR}/openvino_package.tar.gz *
           popd
 
           pushd ${DEVELOPER_PACKAGE_DIR}
-            tar -czvf ${BUILD_DIR}/openvino_developer_package.tar.gz *
+            tar -I pigz -cvf ${BUILD_DIR}/openvino_developer_package.tar.gz *
           popd
 
           pushd ${INSTALL_TEST_DIR}
-            tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz *
+            tar -I pigz -cvf ${BUILD_DIR}/openvino_tests.tar.gz *
           popd
 
       - name: Build Debian packages

diff --git a/.github/workflows/job_cpu_functional_tests.yml b/.github/workflows/job_cpu_functional_tests.yml
@@ -55,11 +55,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
 
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Fetch setup_python action

diff --git a/.github/workflows/job_cxx_unit_tests.yml b/.github/workflows/job_cxx_unit_tests.yml
@@ -64,10 +64,10 @@ jobs:
         if: ${{ runner.os != 'Windows' }}
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Extract OpenVINO packages (Windows)

diff --git a/.github/workflows/job_gpu_tests.yml b/.github/workflows/job_gpu_tests.yml
@@ -59,10 +59,10 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Install dependencies (Linux)

diff --git a/.github/workflows/job_jax_models_tests.yml b/.github/workflows/job_jax_models_tests.yml
@@ -64,11 +64,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
           popd
 
           pushd ${INSTALL_TEST_DIR}
-            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_tests.tar.gz -C ${INSTALL_DIR}
           popd
 
       - name: Fetch setup_python action

diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml
@@ -62,11 +62,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
           popd
 
           pushd ${INSTALL_TEST_DIR}
-            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_tests.tar.gz -C ${INSTALL_DIR}
           popd
 
       # Issue 148922

diff --git a/.github/workflows/job_onnx_runtime.yml b/.github/workflows/job_onnx_runtime.yml
@@ -60,7 +60,7 @@ jobs:
       - name: Extract OpenVINO package
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
           popd
 
       - name: Clone ONNX Runtime

diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml
@@ -66,10 +66,10 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Fetch setup_python action

diff --git a/.github/workflows/job_pytorch_layer_tests.yml b/.github/workflows/job_pytorch_layer_tests.yml
@@ -67,10 +67,10 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Extract OpenVINO packages (Windows)

diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml
@@ -77,11 +77,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
           popd
 
           pushd ${INSTALL_TEST_DIR}
-            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_tests.tar.gz -C ${INSTALL_DIR}
           popd
 
       - name: Fetch setup_python action

diff --git a/.github/workflows/job_samples_tests.yml b/.github/workflows/job_samples_tests.yml
@@ -55,10 +55,10 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Install OpenVINO dependencies (mac)

diff --git a/.github/workflows/job_tensorflow_layer_tests.yml b/.github/workflows/job_tensorflow_layer_tests.yml
@@ -73,10 +73,10 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           pushd $INSTALL_DIR
-            tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_package.tar.gz -C $INSTALL_DIR
           popd
           pushd $INSTALL_TEST_DIR
-            tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
+            tar -I pigz -xf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
 
       - name: Extract OpenVINO packages (Windows)

diff --git a/.github/workflows/job_tensorflow_models_tests.yml b/.github/workflows/job_tensorflow_models_tests.yml
@@ -69,11 +69,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
           popd
 
           pushd ${INSTALL_TEST_DIR}
-            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_tests.tar.gz -C ${INSTALL_DIR}
           popd
 
       - name: Fetch setup_python action

diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml
@@ -83,7 +83,7 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
           popd
 
       - name: Extract OpenVINO packages (Windows)
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,6 +18,8 @@ RUN yum update -y && yum install -y \ @@
         tar \
         gcc \
         gcc-c++ \
+        # parallel gzip
+        pigz \
         xz
     # Install build dependencies
@@ Expand Down @@