Merge pull request mlcommons#921 from scap3yvt/920-feature-standardiz…

…e-dependency-installation-for-github-actions Standardized dependency installation for GitHub actions
benmalef · Oct 21, 2024 · 17600a7 · 17600a7
2 parents 301c188 + 718a482
commit 17600a7
Show file tree

Hide file tree

Showing 14 changed files with 143 additions and 196 deletions.
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
@@ -9,7 +9,7 @@ jobs:
       matrix:
         python-version: [3.9]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Extract black version from setup.py
         run: |

diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml
@@ -31,4 +31,4 @@ jobs:
     steps:
       # Checkout the repository to the GitHub Actions runner
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -37,7 +37,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml
@@ -0,0 +1,104 @@
+name: 'Install Dependencies'
+
+description: 'Setup and install dependencies for GaNDLF'
+
+outputs:
+    other_modified_files_count:
+      description: "Which files have changed"
+      value: ${{ steps.changed-files.outputs.other_modified_files_count }}
+# on: 
+#   workflow_call:
+
+# jobs:
+#   install_dependencies:
+#     runs-on: ubuntu-latest
+runs:
+    using: "composite"    
+    steps:
+      - name: Free space
+        shell: bash
+        run: |
+          df -h
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf "$ANDROID_SDK_ROOT"
+          df -h
+
+      # - name: Checkout code
+      #   uses: actions/checkout@v4
+
+      # Use changed-files-specific action to collect file changes.
+      # The following commented condition applied to a step will run that step only if non-docs files have changed.
+      # It should be applied to all functionality-related steps.
+      # if: ${{steps.dependencies.outputs.other_modified_files_count > 0}}
+      - name: Detect and screen file changes
+        # shell: bash
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v44
+        with:
+          files: |
+            .github/*.md
+            .github/ISSUE_TEMPLATE/*.md
+            .github/workflows/devcontainer.yml
+            .github/workflows/docker-image.yml
+            .devcontainer/**
+            docs/**
+            mlcube/**
+            *.md
+            LICENSE
+            Dockerfile-*
+        # run: |
+        #   echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT
+
+      - name: Summarize docs and non-docs modifications
+        id: changed-files
+        shell: bash
+        run: |
+          echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}"
+          echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}"
+          echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}"
+          echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}"
+          echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT
+          echo "$GITHUB_OUTPUT"
+
+
+      ## this did NOT work
+      # - name: Set output
+      #   shell: bash
+      #   run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT
+
+      ## this did NOT work
+      # - name: Check saved output
+      #   shell: bash
+      #   run: echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}"
+
+      # This second step is unnecessary but highly recommended because
+      # It will cache database and saves time re-downloading it if database isn't stale.
+      - name: Cache pip
+        # shell: bash
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Set up Python 3.9
+        # shell: bash
+        if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies and package
+        shell: bash
+        if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change
+        run: |
+          sudo apt-get update
+          sudo apt-get install libvips libvips-tools -y
+          python -m pip install --upgrade pip==24.0
+          python -m pip install wheel
+          python -m pip install openvino-dev==2023.0.1 mlcube_docker
+          pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu
+          pip install -e .
diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml
@@ -19,7 +19,7 @@ jobs:
         df -h
 
     - name: Checkout (GitHub)
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Use changed-files-specific action to collect file changes.
     # The following commented condition applied to a step will run that step only if non-docs files have changed.

diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -35,7 +35,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         lfs: true
         submodules: 'recursive'

diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml
@@ -10,7 +10,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.x

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:

diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml
@@ -12,68 +12,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Free space
-        run: |
-          df -h
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          sudo rm -rf "$ANDROID_SDK_ROOT"
-          df -h
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      # Use changed-files-specific action to collect file changes.
-      # The following commented condition applied to a step will run that step only if non-docs files have changed.
-      # It should be applied to all functionality-related steps.
-      # if: steps.changed-files-specific.outputs.only_modified == 'false'
-      - name: Detect and screen file changes
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v41
-        with:
-          files: |
-            .github/*.md
-            .github/ISSUE_TEMPLATE/*.md
-            .github/workflows/devcontainer.yml
-            .github/workflows/docker-image.yml
-            .devcontainer/**
-            docs/**
-            mlcube/**
-            *.md
-            LICENSE
-            Dockerfile-*
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-      - name: Summarize docs and non-docs modifications
-        run: |
-          echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}"
-          echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}"
+      - name: Call reusable workflow to install dependencies
+        id: dependencies
+        uses: ./.github/workflows/dependencies
 
-      # This second step is unnecessary but highly recommended because
-      # It will cache database and saves time re-downloading it if database isn't stale.
-      - name: Cache pip
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Set up Python 3.9
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-      - name: Install dependencies and package
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
-        run: |
-          sudo apt-get update
-          sudo apt-get install libvips libvips-tools -y
-          python -m pip install --upgrade pip==24.0
-          python -m pip install wheel
-          python -m pip install openvino-dev==2023.0.1 mlcube_docker
-          pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu
-          pip install -e .
       - name: Run mlcube deploy tests
         working-directory: ./testing
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
+        if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change
         run: |
           sh test_deploy.sh
diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml
@@ -17,72 +17,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Free space
-        run: |
-          df -h
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          sudo rm -rf "$ANDROID_SDK_ROOT"
-          df -h
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      # Use changed-files-specific action to collect file changes.
-      # The following commented condition applied to a step will run that step only if non-docs files have changed.
-      # It should be applied to all functionality-related steps.
-      # if: steps.changed-files-specific.outputs.only_modified == 'false'
-      - name: Detect and screen file changes
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v41
-        with:
-          files: |
-            .github/*.md
-            .github/ISSUE_TEMPLATE/*.md
-            .github/workflows/devcontainer.yml
-            .github/workflows/docker-image.yml
-            .devcontainer/**
-            docs/**
-            mlcube/**
-            *.md
-            LICENSE
-            Dockerfile-*
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-      - name: Summarize docs and non-docs modifications
-        run: |
-          echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}"
-          echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}"
+      - name: Call reusable workflow to install dependencies
+        id: dependencies
+        uses: ./.github/workflows/dependencies
 
-      # This second step is unnecessary but highly recommended because
-      # It will cache database and saves time re-downloading it if database isn't stale.
-      - name: Cache pip
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Set up Python 3.9
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-      - name: Install dependencies and package
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
-        run: |
-          sudo apt-get update
-          sudo apt-get install libvips libvips-tools -y
-          python -m pip install --upgrade pip==24.0
-          python -m pip install wheel
-          pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu
-          pip install -e .
       - name: Run generic unit tests to download data and construct CSVs
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
+        if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change
         run: |
           pytest --cov=. --cov-report=xml -k "prepare_data_for_ci"
+
       # openfl tests start here
       - name: Run OpenFL tests
-        if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change
+        if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change
         run: |
           echo "Removing onnx because of protobuf version conflict"
           pip uninstall onnx -y

diff --git a/.github/workflows/ossar-analysis.yml b/.github/workflows/ossar-analysis.yml
@@ -18,7 +18,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Ensure a compatible version of dotnet is installed.
     # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201.

diff --git a/.github/workflows/publish-nightly.yml b/.github/workflows/publish-nightly.yml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
           ref: master
 

diff --git a/.github/workflows/python-install-check.yml b/.github/workflows/python-install-check.yml
@@ -9,7 +9,7 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4