Merge branch 'main' into nni

Lightning-AI · Jan 1, 2023 · 5539cd8 · 5539cd8
2 parents b925c86 + 400ad67
commit 5539cd8
Show file tree

Hide file tree

Showing 19 changed files with 134 additions and 167 deletions.
diff --git a/.azure/testing-template.yml b/.azure/testing-template.yml
@@ -29,6 +29,8 @@ jobs:
       # map the output variable from A into this job
       configs: $[ dependencies.check_diff.outputs['files.diff'] ]
       config: "${{ config }}"
+      DEVICES: $( python -c 'name = "$(Agent.Name)" ; gpus = name.split("_")[-1] if "_" in name else "0,1"; print(gpus)' )
+
     condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), contains(variables['configs'], variables['config']))
     # how long to run the job before automatically cancelling
     timeoutInMinutes: 75
@@ -37,26 +39,45 @@ jobs:
     workspace:
       clean: all
 
-    pool: azure-gpus-spot
+    pool: 'lit-rtx-3090'
     # this need to have installed docker in the base image...
     container:
       # base ML image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04
-      image: "pytorchlightning/ecosystem-ci:pt1.8.1"
+      # image: "pytorchlightning/ecosystem-ci:pt1.8.1"
       # image: "nvcr.io/nvidia/pytorch:21.11-py3"
-      # image: "pytorch/pytorch:1.8.1-cuda11.1-cudnn8-devel"
-      options: "-it --rm --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=32g"
+      image: "pytorch/pytorch:1.13.0-cuda11.6-cudnn8-runtime"
+      options: "--gpus=all --shm-size=8g -v /usr/bin/docker:/tmp/docker:ro"
     steps:
 
     - bash: |
+        echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)"
+        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
+      displayName: 'Set environment variables'
+
+    - bash: |
+        whoami && id
         lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
+        echo $CUDA_VISIBLE_DEVICES
+        echo $CONTAINER_ID
         python --version
+        pip --version
+        pip list
+        python -c "import torch ; print(torch.cuda.get_arch_list())"
       displayName: 'Image info & NVIDIA'
 
+    - script: |
+        /tmp/docker exec -t -u 0 $CONTAINER_ID \
+        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+      displayName: 'Install Sudo in container (thanks Microsoft!)'
+
     - bash: |
+        sudo apt-get update -q --fix-missing
+        sudo apt-get install -q -y build-essential gcc g++ cmake git unzip tree --no-install-recommends
+        # Python's dependencies
         pip --version
-        pip install --requirement requirements.txt
+        pip install -r requirements.txt
         pip list
       displayName: 'Install dependencies'
 
@@ -71,8 +92,8 @@ jobs:
 
     - bash: |
         bash prepare_env.sh
+        # pip list
         tree .
-        pip list
       displayName: 'Prepare env.'
 
     - script: |

diff --git a/.github/workflows/ci_test-acts.yml b/.github/workflows/ci_test-acts.yml
@@ -6,6 +6,10 @@ on:  # Trigger the workflow on push or pull request, but only for the main branc
   pull_request:
     branches: [main]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   pytest-internal:
 
@@ -25,15 +29,12 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
 
-    # Note: This uses an internal pip API and may not always work
-    # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
-    - name: Get pip cache
+    - name: Get pip cache dir
       id: pip-cache
-      run: |
-        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
+      run: echo "::set-output name=dir::$(pip cache dir)"
 
     - name: Cache pip
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: ${{ runner.os }}-py${{ matrix.python }}-pip-${{ hashFiles('requirements.txt') }}

diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml
@@ -15,6 +15,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   check-diff:
     runs-on: ubuntu-20.04
@@ -34,19 +38,17 @@ jobs:
         PR_NUMBER: "${{ github.event.pull_request.number }}"
       run: |
         pip install -q -r requirements.txt
-        python actions/assistant.py list_runtimes $PR_NUMBER
-        echo "::set-output name=runtimes::{include: $(python actions/assistant.py list_runtimes $PR_NUMBER 2>&1)}"
+        pr_runtimes=$(python actions/assistant.py list_runtimes $PR_NUMBER 2>&1)
+        echo $pr_runtimes
+        echo "runtimes={include: $pr_runtimes}" >> $GITHUB_OUTPUT
 
   pytest-project:
-
+    needs: check-diff
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.check-diff.outputs.matrix) }}
-    # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 35
-    needs: check-diff
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python }}
@@ -55,18 +57,17 @@ jobs:
         python-version: ${{ matrix.python }}
 
     - name: Cache period
-      run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"
+      run: |
+        offset=$(python -c "import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))")
+        echo "period=$offset" >> $GITHUB_OUTPUT
       id: times
 
-    # Note: This uses an internal pip API and may not always work
-    # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
-    - name: Get pip cache
+    - name: Get pip cache dir
       id: pip-cache
-      run: |
-        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
+      run: echo "::set-output name=dir::$(pip cache dir)"
 
     - name: Cache pip
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: $COMMON-td${{ steps.times.outputs.period }}-pip-$HASH_FILES
@@ -80,26 +81,23 @@ jobs:
         pip install -q -r requirements.txt
         python actions/assistant.py prepare_env --config_file=${{ matrix.config }} > prepare_env.sh
         cat prepare_env.sh
-      shell: bash
 
     - name: Some outputs
       run: |
-        echo "::set-output name=args::$(python actions/assistant.py specify_tests --config_file=${{ matrix.config }} 2>&1)"
-        echo "::set-output name=env::$(python actions/assistant.py dict_env --config_file=${{ matrix.config }} 2>&1)"
-        echo "::set-output name=contacts::$(python actions/assistant.py contacts --config_file=${{ matrix.config }} 2>&1)"
+        echo "args=$(python actions/assistant.py specify_tests --config_file=${{ matrix.config }})" >> $GITHUB_OUTPUT
+        echo "env=$(python actions/assistant.py dict_env --config_file=${{ matrix.config }})" >> $GITHUB_OUTPUT
+        echo "contacts=$(python actions/assistant.py contacts --config_file=${{ matrix.config }})" >> $GITHUB_OUTPUT
       id: extras
 
     - name: Prepare environment
       run: |
         bash prepare_env.sh
-      shell: bash
 
     - name: Show dir tree
       if: runner.os == 'Linux'
       run: |
         sudo apt install -q -y tree
         tree .
-      shell: bash
 
     - name: Testing
       env: ${{ fromJSON(steps.extras.outputs.env) }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,7 +8,7 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
@@ -21,25 +21,25 @@ repos:
       - id: detect-private-key
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.1.0
+    rev: v3.3.1
     hooks:
       - id: pyupgrade
-        args: [--py36-plus]
+        args: [--py38-plus]
         name: Upgrade code
 
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.5.0
+    rev: v1.5.1
     hooks:
       - id: docformatter
         args: [--in-place, --wrap-summaries=120, --wrap-descriptions=120]
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
       - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 22.12.0
     hooks:
       - id: black
         name: Black code
@@ -59,6 +59,6 @@ repos:
       - id: yesqa
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
       - id: flake8
diff --git a/actions/_config.yaml b/actions/_config.yaml
@@ -11,7 +11,9 @@ target_repository:
   # OPTIONAL, define installing package extras
   install_extras: all
   # OPTIONAL, install additional requirements from a file
-  requirements_file: requirements.txt
+  # requirements_file: requirements.txt
+  # passing additional `pip install` flags
+  install_flags: "--use-pep517"
   # copy some  test from the target repository
   copy_tests:
     - integrations
@@ -30,17 +32,16 @@ before_install:
 
 # OPTIONAL, if any installation require some env. variables
 env:
-  HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
-  # HOROVOD_WITHOUT_MXNET: 1
-  # HOROVOD_WITHOUT_TENSORFLOW: 1
+  PACKAGE_NAME: "pytorch"
 
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/1.5.x
+    checkout: release/stable
     # install_extras: all
   - name: Cython
     checkout: 0.29.24
+    install_flags: "--use-pep517"
 
 # OPTIONAL, running before installing your project
 # WARNING: this is custom for given OS, we do not manage any inter OS compatibility

diff --git a/actions/assistant.py b/actions/assistant.py
@@ -2,7 +2,7 @@
 import json
 import os
 import traceback
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Sequence, Union
 
 import fire
 import requests
@@ -26,7 +26,6 @@ def request_url(url: str, auth_token: Optional[str] = None) -> Optional[dict]:
 
 
 class AssistantCLI:
-
     _BASH_SCRIPT = ("set -e",)
     _FIELD_TARGET_REPO = "target_repository"
     _FIELD_REQUIRE = "dependencies"
@@ -127,10 +126,21 @@ def _extras(extras: Union[str, list, tuple] = "") -> str:
         extras = ",".join(extras) if isinstance(extras, (tuple, list, set)) else extras
         return extras
 
+    @staticmethod
+    def _get_flags(repo: dict, defaults: Sequence[str] = ("--quiet",)) -> List[str]:
+        """Extract the install's flags with some defaults."""
+        flags = repo.get("install_flags", [])
+        flags = [flags] if isinstance(flags, str) else flags
+        return list(set(flags + list(defaults)))
+
     @staticmethod
     def _install_pip(repo: Dict[str, str]) -> str:
         """Create command for installing a project from source (if HTTPS is given) or from PyPI (if at least name is
-        given)."""
+        given).
+
+        Args:
+            repo: it is package or repository with additional key fields
+        """
         assert any(k in repo for k in ["HTTPS", "name"]), f"Missing key `HTTPS` or `name` among {repo.keys()}"
         # pip install -q 'https://github.com/...#egg=lightning-flash[tabular]
         name = repo.get("name")
@@ -146,20 +156,22 @@ def _install_pip(repo: Dict[str, str]) -> str:
                 password=repo.get("password"),
             )
 
-            cmd = f"git+{url}"
+            pkg = f"git+{url}"
             if "checkout" in repo:
                 assert isinstance(repo["checkout"], str)
-                cmd += f"@{repo['checkout']}"
+                pkg += f"@{repo['checkout']}"
             if "install_extras" in repo:
-                cmd += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]"
+                pkg += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]"
         else:
             # make installation from pypi package
-            cmd = name
+            pkg = name
             if "install_extras" in repo:
-                cmd += f"[{repo['install_extras']}]"
+                pkg += f"[{repo['install_extras']}]"
             if "checkout" in repo:
-                cmd += f"=={repo['checkout']}"
-        return "pip install --quiet " + cmd
+                pkg += f"=={repo['checkout']}"
+        flags = AssistantCLI._get_flags(repo, defaults=("--quiet", "--upgrade"))
+        cmd = " ".join(["pip install", pkg, " ".join(flags)])
+        return cmd
 
     @staticmethod
     def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
@@ -180,16 +192,21 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
         if "requirements_file" in repo:
             reqs = repo["requirements_file"]
             reqs = [reqs] if isinstance(reqs, str) else reqs
-            cmds.append(f"pip install --quiet --upgrade {' '.join([f'-r {req}' for req in reqs])}")
+            args = [f"-r {req}" for req in reqs] + ["--quiet", "--upgrade"]
+            cmds.append("pip install " + " ".join(args))
 
         if "install_command" in repo:
             cmds.append(repo["install_command"])
         else:
             pip_install = "."
             if "install_extras" in repo:
                 pip_install += f"[{AssistantCLI._extras(repo['install_extras'])}]"
-            cmds.append(f"pip install --quiet {pip_install}")
+
+            flags = AssistantCLI._get_flags(repo)
+            cmds.append("pip install " + " ".join([pip_install] + flags))
+        cmds.append("pip list")
         cmds.append("cd ..")
+
         if remove_dir:
             cmds.append(f"rm -rf {repo_name}")
         return cmds
@@ -271,6 +288,7 @@ def prepare_env(config_file: str = "config.yaml", path_root: str = _PATH_ROOT) -
         reqs = config.get("dependencies", [])
         for req in reqs:
             script.append(AssistantCLI._install_pip(req))
+        script.append("pip list")
 
         script += AssistantCLI.before_commands(config_file, stage="test", as_append=True)
         return os.linesep.join(script)

diff --git a/configs/Lightning-AI/bolts_pl-release.yaml b/configs/Lightning-AI/bolts_pl-release.yaml
@@ -20,7 +20,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/pytorch
+    checkout: release/stable
     # install_extras: all
 
 testing:

diff --git a/configs/Lightning-AI/flash_pl-release.yaml b/configs/Lightning-AI/flash_pl-release.yaml
@@ -25,7 +25,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/pytorch
+    checkout: release/stable
   - name: torchmetrics
     HTTPS: https://github.com/Lightning-AI/metrics.git
     checkout: release/stable