From f8e17fb3f67dfda2f1ac6bb79e6abc77516b67a7 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:07:49 -0400 Subject: [PATCH 01/54] all should be at latest long-term stable tag --- .github/workflows/black.yml | 2 +- .github/workflows/codacy.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/devcontainer.yml | 2 +- .github/workflows/docker-image.yml | 2 +- .github/workflows/docs-ci.yml | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/mlcube-test.yml | 2 +- .github/workflows/openfl-test.yml | 2 +- .github/workflows/ossar-analysis.yml | 2 +- .github/workflows/publish-nightly.yml | 2 +- .github/workflows/python-install-check.yml | 2 +- .github/workflows/python-test.yml | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 3e7fd0864..40790685e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: [3.9] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Extract black version from setup.py run: | diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml index de9c6eba7..7e1aefbbd 100644 --- a/.github/workflows/codacy.yml +++ b/.github/workflows/codacy.yml @@ -31,4 +31,4 @@ jobs: steps: # Checkout the repository to the GitHub Actions runner - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4a8c4338f..ceab64ca5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 730acbff5..4337cd8bd 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -19,7 +19,7 @@ jobs: df -h - name: Checkout (GitHub) - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 502467131..d872c57f8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: lfs: true submodules: 'recursive' diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index b92a1f625..8b471e9e9 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -10,7 +10,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.x diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba5e79b7a..2c286a20b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 23f91b178..e7e4643a0 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -21,7 +21,7 @@ jobs: sudo rm -rf "$ANDROID_SDK_ROOT" df -h - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 30e83d5fb..67c91c18f 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -22,7 +22,7 @@ jobs: sudo rm -rf "$ANDROID_SDK_ROOT" df -h - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. diff --git a/.github/workflows/ossar-analysis.yml b/.github/workflows/ossar-analysis.yml index b0323a7b2..804ed0dea 100644 --- a/.github/workflows/ossar-analysis.yml +++ b/.github/workflows/ossar-analysis.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Ensure a compatible version of dotnet is installed. # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201. diff --git a/.github/workflows/publish-nightly.yml b/.github/workflows/publish-nightly.yml index 4d8ff05a3..fb587e476 100644 --- a/.github/workflows/publish-nightly.yml +++ b/.github/workflows/publish-nightly.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master diff --git a/.github/workflows/python-install-check.yml b/.github/workflows/python-install-check.yml index 7ec37528a..7fdb3283c 100644 --- a/.github/workflows/python-install-check.yml +++ b/.github/workflows/python-install-check.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 78c10901e..c7ecfc2df 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -22,7 +22,7 @@ jobs: sudo rm -rf "$ANDROID_SDK_ROOT" df -h - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. From 94df3a8700e3d17db100011bd64fb1ee36523d7a Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:21:00 -0400 Subject: [PATCH 02/54] added dependency workflow --- .github/workflows/dependencies.yml | 70 ++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/dependencies.yml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 000000000..cb72a5e82 --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,70 @@ +name: Reusable Workflow + +on: [workflow_call] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Free space + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "$ANDROID_SDK_ROOT" + df -h + + - name: Checkout code + uses: actions/checkout@v4 + + # Use changed-files-specific action to collect file changes. + # The following commented condition applied to a step will run that step only if non-docs files have changed. + # It should be applied to all functionality-related steps. + # if: steps.changed-files-specific.outputs.only_modified == 'false' + - name: Detect and screen file changes + id: changed-files-specific + uses: tj-actions/changed-files@v44 + with: + files: | + .github/*.md + .github/ISSUE_TEMPLATE/*.md + .github/workflows/devcontainer.yml + .github/workflows/docker-image.yml + .devcontainer/** + docs/** + mlcube/** + *.md + LICENSE + Dockerfile-* + + - name: Summarize docs and non-docs modifications + run: | + echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" + echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + + # This second step is unnecessary but highly recommended because + # It will cache database and saves time re-downloading it if database isn't stale. + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python 3.9 + if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install dependencies and package + if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + run: | + sudo apt-get update + sudo apt-get install libvips libvips-tools -y + python -m pip install --upgrade pip==24.0 + python -m pip install wheel + pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu + pip install -e . From c6cc9537fc23eace4c7916359911c9b84930824f Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:21:23 -0400 Subject: [PATCH 03/54] using the base actions for the rest --- .github/workflows/mlcube-test.yml | 61 ++----------------------------- .github/workflows/openfl-test.yml | 61 ++----------------------------- .github/workflows/python-test.yml | 61 ++----------------------------- 3 files changed, 10 insertions(+), 173 deletions(-) diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index e7e4643a0..3ece0b7ea 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -12,66 +12,11 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v4 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 + call-reusable-workflow: + uses: ./.github/workflows/dependencies.yml with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + node-version: '14' - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" - - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - python -m pip install openvino-dev==2023.0.1 mlcube_docker - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run mlcube deploy tests working-directory: ./testing if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 67c91c18f..6a0610858 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -13,69 +13,16 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v4 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 + call-reusable-workflow: + uses: ./.github/workflows/dependencies.yml with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* - - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + node-version: '14' - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" + # openfl tests start here - name: Run OpenFL tests if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index c7ecfc2df..9abccbbb0 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,66 +13,11 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v4 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 + call-reusable-workflow: + uses: ./.github/workflows/dependencies.yml with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + node-version: '14' - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" - - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - python -m pip install openvino-dev==2023.0.1 mlcube_docker - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run generic unit tests if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change run: | From 14c057f935022898cc0c33ffcb423523cb551abc Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:55:41 -0400 Subject: [PATCH 04/54] checking if this syntax works --- .github/workflows/mlcube-test.yml | 6 ++---- .github/workflows/openfl-test.yml | 6 ++---- .github/workflows/python-test.yml | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 3ece0b7ea..dd9d6a354 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -12,10 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - call-reusable-workflow: - uses: ./.github/workflows/dependencies.yml - with: - node-version: '14' + - name: Call reusable workflow to install dependencies + uses: ./.github/workflows/install-dependencies.yml - name: Run mlcube deploy tests working-directory: ./testing diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 6a0610858..e96acc083 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -13,10 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - call-reusable-workflow: - uses: ./.github/workflows/dependencies.yml - with: - node-version: '14' + - name: Call reusable workflow to install dependencies + uses: ./.github/workflows/install-dependencies.yml - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 9abccbbb0..60e58f9b0 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,10 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - call-reusable-workflow: - uses: ./.github/workflows/dependencies.yml - with: - node-version: '14' + - name: Call reusable workflow to install dependencies + uses: ./.github/workflows/install-dependencies.yml - name: Run generic unit tests if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change From 5acc9bf1cca6ae0b810ff925c33b9f6f4412593f Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:56:51 -0400 Subject: [PATCH 05/54] fixed file name --- .github/workflows/mlcube-test.yml | 2 +- .github/workflows/openfl-test.yml | 2 +- .github/workflows/python-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index dd9d6a354..89c6b547b 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/install-dependencies.yml + uses: ./.github/workflows/dependencies.yml - name: Run mlcube deploy tests working-directory: ./testing diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index e96acc083..32aef1dea 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/install-dependencies.yml + uses: ./.github/workflows/dependencies.yml - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 60e58f9b0..a936f1ccb 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/install-dependencies.yml + uses: ./.github/workflows/dependencies.yml - name: Run generic unit tests if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change From e46dad80a228a89db9b0c26ce4684160e546fd03 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:58:22 -0400 Subject: [PATCH 06/54] actions/checkout should be in the main workflow --- .github/workflows/dependencies.yml | 3 --- .github/workflows/mlcube-test.yml | 3 +++ .github/workflows/openfl-test.yml | 3 +++ .github/workflows/python-test.yml | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index cb72a5e82..9fcb4796a 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -15,9 +15,6 @@ jobs: sudo rm -rf "$ANDROID_SDK_ROOT" df -h - - name: Checkout code - uses: actions/checkout@v4 - # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. # It should be applied to all functionality-related steps. diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 89c6b547b..9862325db 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -12,6 +12,9 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 32aef1dea..b7083b41e 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -13,6 +13,9 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index a936f1ccb..a707c35ec 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,6 +13,9 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml From 310816aadb56290367dc1deeac424c968e59060f Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:00:41 -0400 Subject: [PATCH 07/54] trying something --- .github/workflows/openfl-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index b7083b41e..c1a5d91ca 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -16,6 +16,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: LS + run: | + ls -lt + - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml From 0e7e76067400ec48f97f6a7c2e55e6fd6947e2bc Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:02:11 -0400 Subject: [PATCH 08/54] check again --- .github/workflows/openfl-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index c1a5d91ca..863f29417 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -18,7 +18,7 @@ jobs: - name: LS run: | - ls -lt + ls -lta - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml From 2e6fb85345bcf3a0063e0a22d23408cee5b29067 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:03:21 -0400 Subject: [PATCH 09/54] checking again --- .github/workflows/openfl-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 863f29417..a1a4fcf48 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -18,7 +18,9 @@ jobs: - name: LS run: | + pwd ls -lta + ls -lt .github - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml From 7709921b74b2add25681f1b364accd7bac462a73 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:06:06 -0400 Subject: [PATCH 10/54] checking again --- .github/workflows/dependencies.yml | 2 +- .github/workflows/openfl-test.yml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 9fcb4796a..2f4a7327e 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,4 +1,4 @@ -name: Reusable Workflow +name: Dependencies on: [workflow_call] diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index a1a4fcf48..845352442 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -13,9 +13,6 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: LS run: | pwd From 473fefcfdaf9db28980a2457b66849be1aba4d11 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:29:20 -0400 Subject: [PATCH 11/54] checking again --- .github/workflows/dependencies.yml | 3 +++ .github/workflows/openfl-test.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 2f4a7327e..9e7dbd9fb 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -15,6 +15,9 @@ jobs: sudo rm -rf "$ANDROID_SDK_ROOT" df -h + - name: Checkout code + uses: actions/checkout@v4 + # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. # It should be applied to all functionality-related steps. diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 845352442..a1a4fcf48 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -13,6 +13,9 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: LS run: | pwd From 7904fa515ea67c9720722f83cf1ee85c27658929 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:31:14 -0400 Subject: [PATCH 12/54] check ls --- .github/workflows/openfl-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index a1a4fcf48..5f28809a9 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -20,7 +20,7 @@ jobs: run: | pwd ls -lta - ls -lt .github + ls -lt .github/workflows - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml From 3d81947883cbabb15a9132c578e9adae7e85de0a Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:32:15 -0400 Subject: [PATCH 13/54] check again --- .github/workflows/openfl-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 5f28809a9..ed934c5b3 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -20,7 +20,7 @@ jobs: run: | pwd ls -lta - ls -lt .github/workflows + file ./.github/workflows/dependencies.yml - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies.yml From cea3c63dd2dfa91f131155188592817f9a494abb Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:35:09 -0400 Subject: [PATCH 14/54] check again --- .github/workflows/dependencies.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 9e7dbd9fb..d7a10202b 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,6 +1,6 @@ name: Dependencies -on: [workflow_call] +on: workflow_call jobs: build: @@ -15,8 +15,8 @@ jobs: sudo rm -rf "$ANDROID_SDK_ROOT" df -h - - name: Checkout code - uses: actions/checkout@v4 + # - name: Checkout code + # uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. From f73bb242121186e81ee9601b8364591a333f08f3 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:37:59 -0400 Subject: [PATCH 15/54] checking with same filename --- .github/workflows/dependencies.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index d7a10202b..8bda2e097 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,6 +1,6 @@ -name: Dependencies +name: dependencies -on: workflow_call +on: [workflow_call] jobs: build: From 31d9a547abc57763c8987caca47f1ee5ca72a0de Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:39:50 -0400 Subject: [PATCH 16/54] checking again --- .github/workflows/openfl-test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index ed934c5b3..c049ceaae 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -9,6 +9,11 @@ on: pull_request: {} jobs: + install-dependencies: + permissions: + contents: none + uses: ./.github/workflows/dependencies.yml + openfl-test: runs-on: ubuntu-latest @@ -22,8 +27,8 @@ jobs: ls -lta file ./.github/workflows/dependencies.yml - - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/dependencies.yml + # - name: Call reusable workflow to install dependencies + # uses: ./.github/workflows/dependencies.yml - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change From dcee7fc52bdfc93c32f95d8a1270e34f69ed1e28 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:45:27 -0400 Subject: [PATCH 17/54] revert --- .github/workflows/openfl-test.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index c049ceaae..ed934c5b3 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -9,11 +9,6 @@ on: pull_request: {} jobs: - install-dependencies: - permissions: - contents: none - uses: ./.github/workflows/dependencies.yml - openfl-test: runs-on: ubuntu-latest @@ -27,8 +22,8 @@ jobs: ls -lta file ./.github/workflows/dependencies.yml - # - name: Call reusable workflow to install dependencies - # uses: ./.github/workflows/dependencies.yml + - name: Call reusable workflow to install dependencies + uses: ./.github/workflows/dependencies.yml - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change From ad9145176210e81d3740f065185e99b46fd1e730 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:56:40 -0400 Subject: [PATCH 18/54] checking if this helps --- .github/workflows/dependencies.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 8bda2e097..6268e1ecf 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,6 +1,7 @@ name: dependencies -on: [workflow_call] +on: + workflow_call: jobs: build: From 51afcf5f1ecb355582da209ff14f577d931ffd25 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 08:42:21 -0400 Subject: [PATCH 19/54] trying another thing --- .github/workflows/dependencies.yml | 3 +-- .github/workflows/mlcube-test.yml | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 6268e1ecf..8bda2e097 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,7 +1,6 @@ name: dependencies -on: - workflow_call: +on: [workflow_call] jobs: build: diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 9862325db..1fc13dea1 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -9,14 +9,16 @@ on: jobs: test-deploy: + uses: ./.github/workflows/dependencies.yml + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/dependencies.yml + # - name: Call reusable workflow to install dependencies + # uses: ./.github/workflows/dependencies.yml - name: Run mlcube deploy tests working-directory: ./testing From c0786298de61b1a40be523c434b7439759e13f29 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 08:53:16 -0400 Subject: [PATCH 20/54] checking if using composite action helps --- .github/workflows/dependencies.yml | 6 +- .github/workflows/dependencies/actions.yml | 79 ++++++++++++++++++++++ .github/workflows/mlcube-test.yml | 6 +- .github/workflows/openfl-test.yml | 2 +- 4 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dependencies/actions.yml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 8bda2e097..394258826 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,10 +1,12 @@ name: dependencies -on: [workflow_call] +on: + workflow_call: jobs: - build: + install_dependencies: runs-on: ubuntu-latest + steps: - name: Free space run: | diff --git a/.github/workflows/dependencies/actions.yml b/.github/workflows/dependencies/actions.yml new file mode 100644 index 000000000..4b2c18fa3 --- /dev/null +++ b/.github/workflows/dependencies/actions.yml @@ -0,0 +1,79 @@ +name: 'Install Dependencies' + +description: 'Setup and install dependencies for GaNDLF' + +outputs: + files-changed: + description: "Which files have changed" + value: ${{ steps.changed-files-specific.outputs.only_modified }} +# on: +# workflow_call: + +# jobs: +# install_dependencies: +# runs-on: ubuntu-latest +runs: + using: "composite" + steps: + - name: Free space + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "$ANDROID_SDK_ROOT" + df -h + + # - name: Checkout code + # uses: actions/checkout@v4 + + # Use changed-files-specific action to collect file changes. + # The following commented condition applied to a step will run that step only if non-docs files have changed. + # It should be applied to all functionality-related steps. + # if: steps.changed-files-specific.outputs.only_modified == 'false' + - name: Detect and screen file changes + id: changed-files-specific + uses: tj-actions/changed-files@v44 + with: + files: | + .github/*.md + .github/ISSUE_TEMPLATE/*.md + .github/workflows/devcontainer.yml + .github/workflows/docker-image.yml + .devcontainer/** + docs/** + mlcube/** + *.md + LICENSE + Dockerfile-* + + - name: Summarize docs and non-docs modifications + run: | + echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" + echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + + # This second step is unnecessary but highly recommended because + # It will cache database and saves time re-downloading it if database isn't stale. + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python 3.9 + if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install dependencies and package + if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + run: | + sudo apt-get update + sudo apt-get install libvips libvips-tools -y + python -m pip install --upgrade pip==24.0 + python -m pip install wheel + pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu + pip install -e . diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 1fc13dea1..9862325db 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -9,16 +9,14 @@ on: jobs: test-deploy: - uses: ./.github/workflows/dependencies.yml - runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - # - name: Call reusable workflow to install dependencies - # uses: ./.github/workflows/dependencies.yml + - name: Call reusable workflow to install dependencies + uses: ./.github/workflows/dependencies.yml - name: Run mlcube deploy tests working-directory: ./testing diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index ed934c5b3..620ec463a 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -23,7 +23,7 @@ jobs: file ./.github/workflows/dependencies.yml - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/dependencies.yml + uses: ./.github/workflows/dependencies - name: Run generic unit tests to download data and construct CSVs if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change From 3a8e32c51aaf4aecb2bbb671cd8b19ef95472059 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 08:58:43 -0400 Subject: [PATCH 21/54] rename --- .github/workflows/dependencies/{actions.yml => action.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/dependencies/{actions.yml => action.yml} (100%) diff --git a/.github/workflows/dependencies/actions.yml b/.github/workflows/dependencies/action.yml similarity index 100% rename from .github/workflows/dependencies/actions.yml rename to .github/workflows/dependencies/action.yml From 4206de4c38a8e938b53bc10a7cb0daf21068571a Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:00:10 -0400 Subject: [PATCH 22/54] added shell --- .github/workflows/dependencies/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 4b2c18fa3..dc372eec7 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -16,6 +16,7 @@ runs: using: "composite" steps: - name: Free space + shell: bash run: | df -h sudo rm -rf /opt/ghc @@ -32,6 +33,7 @@ runs: # It should be applied to all functionality-related steps. # if: steps.changed-files-specific.outputs.only_modified == 'false' - name: Detect and screen file changes + shell: bash id: changed-files-specific uses: tj-actions/changed-files@v44 with: @@ -48,6 +50,7 @@ runs: Dockerfile-* - name: Summarize docs and non-docs modifications + shell: bash run: | echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" @@ -55,6 +58,7 @@ runs: # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. - name: Cache pip + shell: bash uses: actions/cache@v3 with: path: ~/.cache/pip @@ -63,12 +67,14 @@ runs: ${{ runner.os }}-pip- - name: Set up Python 3.9 + shell: bash if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install dependencies and package + shell: bash if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change run: | sudo apt-get update From 212b96c7b54a89b269121a3eeb8d4ba67c24a291 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:24:54 -0400 Subject: [PATCH 23/54] testing another thing --- .github/workflows/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index dc372eec7..eebd51fa0 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -67,9 +67,9 @@ runs: ${{ runner.os }}-pip- - name: Set up Python 3.9 - shell: bash + # shell: bash if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 + uses: actions/setup-python@5 with: python-version: 3.9 From a362bc13ffa000cc73afa94f893cde05d6ca584c Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:28:25 -0400 Subject: [PATCH 24/54] let's see what happens here --- .github/workflows/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index eebd51fa0..dff6630e5 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -33,7 +33,7 @@ runs: # It should be applied to all functionality-related steps. # if: steps.changed-files-specific.outputs.only_modified == 'false' - name: Detect and screen file changes - shell: bash + # shell: bash id: changed-files-specific uses: tj-actions/changed-files@v44 with: @@ -58,7 +58,7 @@ runs: # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. - name: Cache pip - shell: bash + # shell: bash uses: actions/cache@v3 with: path: ~/.cache/pip From 1676656c72f214ccbf32da2d425b347b12a6c75c Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:30:17 -0400 Subject: [PATCH 25/54] updated version to use --- .github/workflows/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index dff6630e5..cf2a93b91 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -69,7 +69,7 @@ runs: - name: Set up Python 3.9 # shell: bash if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@5 + uses: actions/setup-python@v5 with: python-version: 3.9 From dfc600bf442a73fe90a6313f0408636a11cd35e7 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:17:46 -0400 Subject: [PATCH 26/54] updated action to show extra output, updated workflows to use new composite action and removed reusable workflow --- .github/workflows/dependencies.yml | 72 ----------------------- .github/workflows/dependencies/action.yml | 1 + .github/workflows/mlcube-test.yml | 2 +- .github/workflows/openfl-test.yml | 6 -- .github/workflows/python-test.yml | 2 +- 5 files changed, 3 insertions(+), 80 deletions(-) delete mode 100644 .github/workflows/dependencies.yml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml deleted file mode 100644 index 394258826..000000000 --- a/.github/workflows/dependencies.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: dependencies - -on: - workflow_call: - -jobs: - install_dependencies: - runs-on: ubuntu-latest - - steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - # - name: Checkout code - # uses: actions/checkout@v4 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v44 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* - - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" - - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index cf2a93b91..6894d823f 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -54,6 +54,7 @@ runs: run: | echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 9862325db..5d4c04977 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v4 - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/dependencies.yml + uses: ./.github/workflows/dependencies - name: Run mlcube deploy tests working-directory: ./testing diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 620ec463a..766760edd 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -16,12 +16,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: LS - run: | - pwd - ls -lta - file ./.github/workflows/dependencies.yml - - name: Call reusable workflow to install dependencies uses: ./.github/workflows/dependencies diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index a707c35ec..d0ff4628d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Call reusable workflow to install dependencies - uses: ./.github/workflows/dependencies.yml + uses: ./.github/workflows/dependencies - name: Run generic unit tests if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change From 5cb3f001e7dfc0315b83febb4a0e24fab020490c Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:35:29 -0400 Subject: [PATCH 27/54] added more files to check --- .github/workflows/dependencies/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 6894d823f..1d666297e 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -41,7 +41,11 @@ runs: .github/*.md .github/ISSUE_TEMPLATE/*.md .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml + .github/workflows/devcontainer.yml + .github/workflows/main.yml + .github/workflows/mlcube-test.yml + .github/workflows/openfl-test.yml + .github/workflows/python-test.yml .devcontainer/** docs/** mlcube/** From 13869a4db85d96073e2748f5ae4e41d7a5a7bae2 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:11:53 -0400 Subject: [PATCH 28/54] revert --- .github/workflows/dependencies/action.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 1d666297e..6894d823f 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -41,11 +41,7 @@ runs: .github/*.md .github/ISSUE_TEMPLATE/*.md .github/workflows/devcontainer.yml - .github/workflows/devcontainer.yml - .github/workflows/main.yml - .github/workflows/mlcube-test.yml - .github/workflows/openfl-test.yml - .github/workflows/python-test.yml + .github/workflows/docker-image.yml .devcontainer/** docs/** mlcube/** From 6b5739d12f06ccfdafe144b37e7d33c242a56544 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:12:27 -0400 Subject: [PATCH 29/54] checking if full CI gets triggered --- should be reverted --- GANDLF/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GANDLF/version.py b/GANDLF/version.py index 9d601767a..af843f7e7 100644 --- a/GANDLF/version.py +++ b/GANDLF/version.py @@ -2,4 +2,4 @@ # -*- coding: UTF-8 -*- # check GaNDLF wiki for versioning and release guidelines: https://github.com/mlcommons/GaNDLF/wiki -__version__ = "0.1.0-dev" +__version__ = "0.1.0-devTEST" From 1997deec4072452bf966e88853d6fa120e08cfc6 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:15:06 -0400 Subject: [PATCH 30/54] new version to ensure editable version --- GANDLF/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GANDLF/version.py b/GANDLF/version.py index af843f7e7..969da2b13 100644 --- a/GANDLF/version.py +++ b/GANDLF/version.py @@ -2,4 +2,4 @@ # -*- coding: UTF-8 -*- # check GaNDLF wiki for versioning and release guidelines: https://github.com/mlcommons/GaNDLF/wiki -__version__ = "0.1.0-devTEST" +__version__ = "0.1.1-dev" From 007bf0df83998100cbfcaec3af5b04ad20671d6b Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:33:36 -0400 Subject: [PATCH 31/54] updated logic to test for changed non-doc files --- .github/workflows/dependencies/action.yml | 7 ++++--- .github/workflows/mlcube-test.yml | 2 +- .github/workflows/openfl-test.yml | 4 ++-- .github/workflows/python-test.yml | 18 +++++++++--------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 6894d823f..c410eb64f 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -31,7 +31,7 @@ runs: # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' + # if: steps.changed-files-specific.outputs.other_modified_files_count > 0 - name: Detect and screen file changes # shell: bash id: changed-files-specific @@ -54,6 +54,7 @@ runs: run: | echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" # This second step is unnecessary but highly recommended because @@ -69,14 +70,14 @@ runs: - name: Set up Python 3.9 # shell: bash - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies and package shell: bash - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | sudo apt-get update sudo apt-get install libvips libvips-tools -y diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 5d4c04977..1116a162d 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -20,6 +20,6 @@ jobs: - name: Run mlcube deploy tests working-directory: ./testing - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | sh test_deploy.sh diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 766760edd..f37659dea 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -20,13 +20,13 @@ jobs: uses: ./.github/workflows/dependencies - name: Run generic unit tests to download data and construct CSVs - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" # openfl tests start here - name: Run OpenFL tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | echo "Removing onnx because of protobuf version conflict" pip uninstall onnx -y diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index d0ff4628d..e3544ab19 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -20,40 +20,40 @@ jobs: uses: ./.github/workflows/dependencies - name: Run generic unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "generic" - name: Run classification unit tests with histology - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and histology" - name: Run classification unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and not histology" - name: Run segmentation unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "segmentation and not transunet" - name: Run regression unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "regression" - name: Run transunet unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "transunet" - name: Run entrypoints tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "entrypoints" - name: Run test for update_version - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "update_version" - name: Upload coverage - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} From d273d329bf5bd5dfb0640ac927a52c2034778b30 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:40:05 -0400 Subject: [PATCH 32/54] checking logic update --- .github/workflows/dependencies/action.yml | 6 +++--- .github/workflows/mlcube-test.yml | 2 +- .github/workflows/openfl-test.yml | 4 ++-- .github/workflows/python-test.yml | 18 +++++++++--------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index c410eb64f..a376fe7b4 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -31,7 +31,7 @@ runs: # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.other_modified_files_count > 0 + # if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} - name: Detect and screen file changes # shell: bash id: changed-files-specific @@ -70,14 +70,14 @@ runs: - name: Set up Python 3.9 # shell: bash - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies and package shell: bash - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sudo apt-get update sudo apt-get install libvips libvips-tools -y diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 1116a162d..3f12aaf78 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -20,6 +20,6 @@ jobs: - name: Run mlcube deploy tests working-directory: ./testing - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sh test_deploy.sh diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index f37659dea..03f2457c0 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -20,13 +20,13 @@ jobs: uses: ./.github/workflows/dependencies - name: Run generic unit tests to download data and construct CSVs - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" # openfl tests start here - name: Run OpenFL tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | echo "Removing onnx because of protobuf version conflict" pip uninstall onnx -y diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index e3544ab19..581e16adb 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -20,40 +20,40 @@ jobs: uses: ./.github/workflows/dependencies - name: Run generic unit tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "generic" - name: Run classification unit tests with histology - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and histology" - name: Run classification unit tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and not histology" - name: Run segmentation unit tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "segmentation and not transunet" - name: Run regression unit tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "regression" - name: Run transunet unit tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "transunet" - name: Run entrypoints tests - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "entrypoints" - name: Run test for update_version - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "update_version" - name: Upload coverage - if: steps.changed-files-specific.outputs.other_modified_files_count > 0 # Run on any non-docs change + if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} From cee58bf348583e68c25f69de06560f4be6af0686 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:48:22 -0400 Subject: [PATCH 33/54] updated output variable --- .github/workflows/dependencies/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index a376fe7b4..fadfbbd03 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -5,7 +5,7 @@ description: 'Setup and install dependencies for GaNDLF' outputs: files-changed: description: "Which files have changed" - value: ${{ steps.changed-files-specific.outputs.only_modified }} + value: ${{ steps.changed-files-specific.outputs.other_modified_files_count }} # on: # workflow_call: @@ -56,6 +56,7 @@ runs: echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" + # echo "random-number=$(echo $RANDOM)" >> $GITHUB_OUTPUT # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From b13d589e238dc33975f3c886bd97059d59f7cd4f Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:52:20 -0400 Subject: [PATCH 34/54] checking another mechanism --- .github/workflows/dependencies/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index fadfbbd03..76bd1c01d 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -33,7 +33,7 @@ runs: # It should be applied to all functionality-related steps. # if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} - name: Detect and screen file changes - # shell: bash + shell: bash id: changed-files-specific uses: tj-actions/changed-files@v44 with: @@ -48,6 +48,8 @@ runs: *.md LICENSE Dockerfile-* + run: | + echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT - name: Summarize docs and non-docs modifications shell: bash @@ -56,7 +58,6 @@ runs: echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" - # echo "random-number=$(echo $RANDOM)" >> $GITHUB_OUTPUT # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From bc3c3491ea7ddc1aaa7dcfbc4999ac7d0b847825 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:56:23 -0400 Subject: [PATCH 35/54] checking by putting the variable saving in another step --- .github/workflows/dependencies/action.yml | 14 ++++++++------ .github/workflows/mlcube-test.yml | 2 +- .github/workflows/openfl-test.yml | 4 ++-- .github/workflows/python-test.yml | 18 +++++++++--------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 76bd1c01d..ae262a143 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -31,9 +31,9 @@ runs: # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. # It should be applied to all functionality-related steps. - # if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} + # if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} - name: Detect and screen file changes - shell: bash + # shell: bash id: changed-files-specific uses: tj-actions/changed-files@v44 with: @@ -48,16 +48,18 @@ runs: *.md LICENSE Dockerfile-* - run: | - echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT + # run: | + # echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT - name: Summarize docs and non-docs modifications + id: changed-files shell: bash run: | echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" + echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. @@ -72,14 +74,14 @@ runs: - name: Set up Python 3.9 # shell: bash - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies and package shell: bash - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sudo apt-get update sudo apt-get install libvips libvips-tools -y diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 3f12aaf78..91daeeb29 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -20,6 +20,6 @@ jobs: - name: Run mlcube deploy tests working-directory: ./testing - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sh test_deploy.sh diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 03f2457c0..253762596 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -20,13 +20,13 @@ jobs: uses: ./.github/workflows/dependencies - name: Run generic unit tests to download data and construct CSVs - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" # openfl tests start here - name: Run OpenFL tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | echo "Removing onnx because of protobuf version conflict" pip uninstall onnx -y diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 581e16adb..b9be94515 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -20,40 +20,40 @@ jobs: uses: ./.github/workflows/dependencies - name: Run generic unit tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "generic" - name: Run classification unit tests with histology - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and histology" - name: Run classification unit tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and not histology" - name: Run segmentation unit tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "segmentation and not transunet" - name: Run regression unit tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "regression" - name: Run transunet unit tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "transunet" - name: Run entrypoints tests - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "entrypoints" - name: Run test for update_version - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "update_version" - name: Upload coverage - if: ${{steps.changed-files-specific.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} From 1fa6581f874b4d429f4b274af0fbd3e2d95af0f3 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 12:02:53 -0400 Subject: [PATCH 36/54] testing outputs --- .github/workflows/python-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index b9be94515..4004e49e2 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -17,6 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Call reusable workflow to install dependencies + id: changed-files uses: ./.github/workflows/dependencies - name: Run generic unit tests From 625eac22d17bd148d6eef70b1f1235ad67b338a0 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:17:49 -0400 Subject: [PATCH 37/54] trying something else --- .github/workflows/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index ae262a143..08cb4db8d 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -59,7 +59,7 @@ runs: echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" - echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT + echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From 3559fb4bc4b6092edc3c3723e00dd6d773c41345 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:44:29 -0400 Subject: [PATCH 38/54] trying another thing --- .github/workflows/dependencies/action.yml | 6 +++--- .github/workflows/mlcube-test.yml | 3 ++- .github/workflows/openfl-test.yml | 5 +++-- .github/workflows/python-test.yml | 20 ++++++++++---------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 08cb4db8d..1fc99fc95 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -31,7 +31,7 @@ runs: # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. # It should be applied to all functionality-related steps. - # if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} + # if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} - name: Detect and screen file changes # shell: bash id: changed-files-specific @@ -74,14 +74,14 @@ runs: - name: Set up Python 3.9 # shell: bash - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies and package shell: bash - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sudo apt-get update sudo apt-get install libvips libvips-tools -y diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 91daeeb29..fffaa7361 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -16,10 +16,11 @@ jobs: uses: actions/checkout@v4 - name: Call reusable workflow to install dependencies + id: dependencies uses: ./.github/workflows/dependencies - name: Run mlcube deploy tests working-directory: ./testing - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sh test_deploy.sh diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 253762596..101762ab9 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -17,16 +17,17 @@ jobs: uses: actions/checkout@v4 - name: Call reusable workflow to install dependencies + id: dependencies uses: ./.github/workflows/dependencies - name: Run generic unit tests to download data and construct CSVs - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" # openfl tests start here - name: Run OpenFL tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | echo "Removing onnx because of protobuf version conflict" pip uninstall onnx -y diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 4004e49e2..83aa753c9 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -17,44 +17,44 @@ jobs: uses: actions/checkout@v4 - name: Call reusable workflow to install dependencies - id: changed-files + id: dependencies uses: ./.github/workflows/dependencies - name: Run generic unit tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "generic" - name: Run classification unit tests with histology - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and histology" - name: Run classification unit tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and not histology" - name: Run segmentation unit tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "segmentation and not transunet" - name: Run regression unit tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "regression" - name: Run transunet unit tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "transunet" - name: Run entrypoints tests - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "entrypoints" - name: Run test for update_version - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "update_version" - name: Upload coverage - if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} From eb2fb22df3a1de699c191e8a5de63bc4d413866b Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:50:09 -0400 Subject: [PATCH 39/54] changing id --- .github/workflows/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 1fc99fc95..373b67561 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -5,7 +5,7 @@ description: 'Setup and install dependencies for GaNDLF' outputs: files-changed: description: "Which files have changed" - value: ${{ steps.changed-files-specific.outputs.other_modified_files_count }} + value: ${{ steps.changed-files.outputs.other_modified_files_count }} # on: # workflow_call: From 8ddad9a8236151bcdaeea30b0277d4074225f9e1 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:51:08 -0400 Subject: [PATCH 40/54] debugging --- .github/workflows/dependencies/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 373b67561..7b10f8970 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -60,6 +60,7 @@ runs: echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + echo "GITHUB_OUTPUT:$GITHUB_OUTPUT" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. @@ -74,14 +75,14 @@ runs: - name: Set up Python 3.9 # shell: bash - if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies and package shell: bash - if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sudo apt-get update sudo apt-get install libvips libvips-tools -y From 4a71a89bbd5d98b26f41abf3fc2a1f365ae91ee2 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:56:15 -0400 Subject: [PATCH 41/54] checking different syntax --- .github/workflows/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 7b10f8970..4d4bad6ba 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -60,7 +60,7 @@ runs: echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - echo "GITHUB_OUTPUT:$GITHUB_OUTPUT" + echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From 00580782dfe43cd5fb0f1f6a3de8658e19d0b314 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:11:57 -0400 Subject: [PATCH 42/54] trying solution in https://github.com/orgs/community/discussions/11771 --- .github/workflows/python-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 713f64d5a..4ad27019e 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -18,7 +18,7 @@ jobs: - name: Call reusable workflow to install dependencies id: dependencies - uses: ./.github/workflows/dependencies + uses: mlcommons/GaNDLF/.github/workflows/dependencies - name: Run generic unit tests if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change From 6c98878a397db933034c3e1205bc0d889d8a1429 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:43:46 -0400 Subject: [PATCH 43/54] trying suggestion --- .github/workflows/dependencies/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 4d4bad6ba..248bd92bf 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -60,7 +60,8 @@ runs: echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" + # echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" + echo "$GITHUB_OUTPUT" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From 6dcb5358f42cfd3a399ad9740b29392cb459bd16 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:49:04 -0400 Subject: [PATCH 44/54] trying copilot suggestion --- .github/workflows/dependencies/action.yml | 10 +++++++--- .github/workflows/python-test.yml | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 248bd92bf..7a792782e 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -59,9 +59,13 @@ runs: echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" - echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - # echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" - echo "$GITHUB_OUTPUT" + # echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + # # echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" + # echo "$GITHUB_OUTPUT" + + - name: Set output + run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 4ad27019e..713f64d5a 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -18,7 +18,7 @@ jobs: - name: Call reusable workflow to install dependencies id: dependencies - uses: mlcommons/GaNDLF/.github/workflows/dependencies + uses: ./.github/workflows/dependencies - name: Run generic unit tests if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change From 01b644ab8a23be39f1293bde8674d284e7844f10 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:04:35 -0400 Subject: [PATCH 45/54] removed the specific lines altogether to see what happens --- .github/workflows/dependencies/action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 7a792782e..98ce31f72 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -59,9 +59,6 @@ runs: echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" - # echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - # # echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" - # echo "$GITHUB_OUTPUT" - name: Set output run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT From ac26401dd4c9e5943075e381a7d2d22574884dff Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:06:27 -0400 Subject: [PATCH 46/54] let's add the shell --- .github/workflows/dependencies/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 98ce31f72..e88ffb8b2 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -61,6 +61,7 @@ runs: echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" - name: Set output + shell: bash run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT From 1cd823ba2ae8169f8f1f415ab6db8a7c54c262f7 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:07:38 -0400 Subject: [PATCH 47/54] added a debug --- .github/workflows/dependencies/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index e88ffb8b2..dba5c3f94 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -64,6 +64,9 @@ runs: shell: bash run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + - name: Check saved output + shell: bash + run: echo "$GITHUB_OUTPUT" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From c97e734cc36decef0b37c2f07101b885f5d97fac Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:29:05 -0400 Subject: [PATCH 48/54] testing if `other_modified_files_count` gets picked up correctly or not --- GANDLF/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GANDLF/version.py b/GANDLF/version.py index 969da2b13..d960ec2d1 100644 --- a/GANDLF/version.py +++ b/GANDLF/version.py @@ -2,4 +2,4 @@ # -*- coding: UTF-8 -*- # check GaNDLF wiki for versioning and release guidelines: https://github.com/mlcommons/GaNDLF/wiki -__version__ = "0.1.1-dev" +__version__ = "0.1.666-dev" From ad9f93efd0ea52b2b3a93cb11a63ffc060cf3aa4 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:32:07 -0400 Subject: [PATCH 49/54] trying another solution --- .github/workflows/dependencies/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index dba5c3f94..846cf94f3 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -59,14 +59,16 @@ runs: echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" + echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - - name: Set output - shell: bash - run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + ## this did NOT work + # - name: Set output + # shell: bash + # run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - name: Check saved output shell: bash - run: echo "$GITHUB_OUTPUT" + run: echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From b4fd967881b24b7706457f28245a351f95a560a1 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Mon, 30 Sep 2024 19:01:22 -0400 Subject: [PATCH 50/54] copy-paste suggestion --- .github/workflows/dependencies/action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index 846cf94f3..ec1f9ab9e 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -60,15 +60,18 @@ runs: echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + echo "$GITHUB_OUTPUT" + ## this did NOT work # - name: Set output # shell: bash # run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT - - name: Check saved output - shell: bash - run: echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" + ## this did NOT work + # - name: Check saved output + # shell: bash + # run: echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" # This second step is unnecessary but highly recommended because # It will cache database and saves time re-downloading it if database isn't stale. From ec187e27bbae0eec0cc23fbe53ea2960f162e678 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:12:28 -0400 Subject: [PATCH 51/54] trying suggestion --- .github/workflows/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index ec1f9ab9e..b3c2f0387 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -3,7 +3,7 @@ name: 'Install Dependencies' description: 'Setup and install dependencies for GaNDLF' outputs: - files-changed: + other_modified_files_count: description: "Which files have changed" value: ${{ steps.changed-files.outputs.other_modified_files_count }} # on: From 9540f08a976aeecaf59855c8d2f690b48d59f221 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:18:10 -0400 Subject: [PATCH 52/54] revert version --- GANDLF/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GANDLF/version.py b/GANDLF/version.py index d960ec2d1..969da2b13 100644 --- a/GANDLF/version.py +++ b/GANDLF/version.py @@ -2,4 +2,4 @@ # -*- coding: UTF-8 -*- # check GaNDLF wiki for versioning and release guidelines: https://github.com/mlcommons/GaNDLF/wiki -__version__ = "0.1.666-dev" +__version__ = "0.1.1-dev" From 7fb558f5e2dd53faa0a742d1b54f40bd1da78099 Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:36:42 -0400 Subject: [PATCH 53/54] ensure `mlcube_docker` is installed --- .github/workflows/mlcube-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index fffaa7361..7536bb687 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -19,6 +19,11 @@ jobs: id: dependencies uses: ./.github/workflows/dependencies + - name: Install dependencies and package + if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + run: | + python -m pip install mlcube_docker + - name: Run mlcube deploy tests working-directory: ./testing if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change From 718a4823ed3acfa3af1987a667e4443b11b8aa7b Mon Sep 17 00:00:00 2001 From: scap3yvt <149599669+scap3yvt@users.noreply.github.com> Date: Sun, 20 Oct 2024 10:25:23 -0400 Subject: [PATCH 54/54] trying something else --- .github/workflows/dependencies/action.yml | 1 + .github/workflows/mlcube-test.yml | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml index b3c2f0387..32f862b54 100644 --- a/.github/workflows/dependencies/action.yml +++ b/.github/workflows/dependencies/action.yml @@ -99,5 +99,6 @@ runs: sudo apt-get install libvips libvips-tools -y python -m pip install --upgrade pip==24.0 python -m pip install wheel + python -m pip install openvino-dev==2023.0.1 mlcube_docker pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu pip install -e . diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 7536bb687..fffaa7361 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -19,11 +19,6 @@ jobs: id: dependencies uses: ./.github/workflows/dependencies - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - python -m pip install mlcube_docker - - name: Run mlcube deploy tests working-directory: ./testing if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change