Merge branch 'main' into ci/torch2

.codecov.yml

@@ -20,6 +20,7 @@ coverage:
     # https://codecov.readme.io/v1.0/docs/commit-status
     project:
       default:
+        informational: true
         target: 95% # specify the target coverage for each commit status
         threshold: 30% # allow this little decrease on project
         # https://github.com/codecov/support/wiki/Filtering-Branches
@@ -28,6 +29,7 @@ coverage:
     # https://github.com/codecov/support/wiki/Patch-Status
     patch:
       default:
+        informational: true
         threshold: 50% # allow this much decrease on patch
     changes: false
 

.github/workflows/ci_compatible.yml

@@ -148,7 +148,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
-      uses: slackapi/slack-github-action@v1.23.0
+      uses: slackapi/slack-github-action@v1.24.0
       with:
         # channel-id: C02NJMSLFJR  # alerts-ecosystem-ci
         # For posting a rich message using Block Kit - https://app.slack.com/block-kit-builder

.github/workflows/ci_test-acts.yml

@@ -2,26 +2,24 @@ name: CI internal
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the main branch
-  push: {}
-  pull_request:
+  push:
     branches: [main]
+  pull_request: {}
 
 defaults:
   run:
     shell: bash
 
 jobs:
   pytest-internal:
-
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04 , macOS-10.15, windows-2019]
+        os: [ubuntu-20.04, macOS-11, windows-2022]
         python: ["3.9"]
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 10
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python }}
@@ -41,29 +39,18 @@ jobs:
         restore-keys: ${{ runner.os }}-py${{ matrix.python }}-pip-
 
     - name: Install requirements
-      run: |
-        pip install -q -r requirements.txt
+      run: pip install -q -r requirements.txt
 
     - name: Testing
-      run: |
-        coverage run -m pytest _actions -v --junitxml="junit/test-results-${{ runner.os }}-${{ matrix.python }}.xml"
-
-    - name: Upload pytest test results
-      uses: actions/upload-artifact@v3
-      with:
-        name: pytest-results-${{ runner.os }}-${{ matrix.python }}
-        path: junit/test-results-${{ runner.os }}-${{ matrix.python }}.xml
-      if: failure()
+      run: coverage run -m pytest _actions -v
 
     - name: Statistics
-      if: success()
       run: |
         coverage report
         coverage xml
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
-      if: always()
       # see: https://github.com/actions/toolkit/issues/399
       continue-on-error: true
       with:
@@ -72,3 +59,18 @@ jobs:
         flags: cpu,pytest,${{ runner.os }}
         name: CLI-coverage
         fail_ci_if_error: false
+
+
+  testing-guardian:
+    runs-on: ubuntu-latest
+    needs: pytest-internal
+    if: always()
+    steps:
+    - run: echo "${{ needs.pytester.result }}"
+    - name: failing...
+      if: needs.pytester.result == 'failure'
+      run: exit 1
+    - name: cancelled or skipped...
+      if: contains(fromJSON('["cancelled", "skipped"]'), needs.pytester.result)
+      timeout-minutes: 1
+      run: sleep 90

.gitignore

@@ -117,3 +117,5 @@ lightning_logs
 *.gz
 .DS_Store
 .*_submit.py
+
+.ruff_cache/

.pre-commit-config.yaml

@@ -21,14 +21,14 @@ repos:
       - id: detect-private-key
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.9.0
     hooks:
       - id: pyupgrade
         args: [--py38-plus]
         name: Upgrade code
 
   - repo: https://github.com/omnilib/ufmt
-    rev: v2.0.1
+    rev: v2.1.0
     hooks:
       - id: ufmt
 
@@ -42,14 +42,18 @@ repos:
           - mdformat_frontmatter
 
   - repo: https://github.com/asottile/yesqa
-    rev: v1.4.0
+    rev: v1.5.0
     hooks:
       - id: yesqa
         additional_dependencies:
           - pep8-naming
+          - flake8-pytest-style
+          - flake8-bandit
+          - flake8-builtins
+          - flake8-bugbear
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.254
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.278
     hooks:
       - id: ruff
         args: ["--fix"]

_actions/_config.yaml

@@ -55,6 +55,6 @@ testing:
   pytest_args: --strict
 
 runtimes:
-  - {os: "ubuntu-20.04", python: "3.9"}
-  - {os: "macOS-10.15", python: "3.7"}
+  - {os: "ubuntu-20.04", python: "3.10"}
+  - {os: "macOS-10.15", python: "3.8"}
 #  - {os: "windows-2019", python: "3.8"}  # SKIP, because of the wget

_actions/assistant.py

@@ -98,7 +98,8 @@ def list_runtimes(pr: Optional[int] = None, auth_token: Optional[str] = None) ->
     def _load_config(config_file: str = "config.yaml", strict: bool = True) -> dict:
         if not os.path.isfile(config_file):
             config_file = os.path.join("configs", config_file)
-        assert _file_exits(config_file), f"Missing config file: {config_file}"
+        if not _file_exits(config_file):
+            raise FileNotFoundError(f"Missing config file: {config_file}")
         with open(_file_exits(config_file)) as fp:
             config = yaml.safe_load(fp)
         if strict:
@@ -164,7 +165,8 @@ def _install_pip(repo: Dict[str, str]) -> str:
         Args:
             repo: it is package or repository with additional key fields
         """
-        assert any(k in repo for k in ["HTTPS", "name"]), f"Missing key `HTTPS` or `name` among {repo.keys()}"
+        if not any(k in repo for k in ["HTTPS", "name"]):
+            raise ValueError(f"Missing key `HTTPS` or `name` among {repo.keys()}")
         # pip install -q 'https://github.com/...#egg=lightning-flash[tabular]
         name = repo.get("name")
         if not name:
@@ -181,7 +183,7 @@ def _install_pip(repo: Dict[str, str]) -> str:
 
             pkg = f"git+{url}"
             if "checkout" in repo:
-                assert isinstance(repo["checkout"], str)
+                assert isinstance(repo["checkout"], str)  # noqa: S101
                 pkg += f"@{repo['checkout']}"
             if "install_extras" in repo:
                 pkg += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]"
@@ -198,15 +200,16 @@ def _install_pip(repo: Dict[str, str]) -> str:
     @staticmethod
     def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
         """Create command for installing a project from source assuming it is Git project."""
-        assert "HTTPS" in repo, f"Missing key `HTTPS` among {repo.keys()}"
+        if "HTTPS" not in repo:
+            raise ValueError(f"Missing key `HTTPS` among {repo.keys()}")
         url = AssistantCLI._https(
             repo.get("HTTPS"), token=repo.get("token"), username=repo.get("username"), password=repo.get("password")
         )
         cmd_git = f"git clone {url}"
         repo_name, _ = os.path.splitext(os.path.basename(repo.get("HTTPS")))
         cmds = [cmd_git, f"cd {repo_name}"]
         if "checkout" in repo:
-            assert isinstance(repo["checkout"], str)
+            assert isinstance(repo["checkout"], str)  # noqa: S101
             cmds.append(f"git checkout {repo['checkout']}")
 
         if "requirements_file" in repo:
@@ -220,7 +223,7 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
             pip_install += f"[{AssistantCLI._extras(repo['install_extras'])}]"
 
         flags = AssistantCLI._get_flags(repo)
-        cmds.append("pip install " + " ".join([pip_install] + flags))
+        cmds.append("pip install " + " ".join([pip_install, *flags]))
         cmds.append("pip list")
         cmds.append("cd ..")
 
@@ -302,8 +305,7 @@ def prepare_env(config_file: str = "config.yaml", path_root: str = _PATH_ROOT) -
         script.append(f'rm -rf "{repo_name}"')
 
         reqs = config.get("dependencies", [])
-        for req in reqs:
-            script.append(AssistantCLI._install_pip(req))
+        script += [AssistantCLI._install_pip(req) for req in reqs]
         script.append("pip list")
 
         script += [f"cd {AssistantCLI._FOLDER_TESTS}"]
@@ -335,8 +337,10 @@ def slack_payload(fpath_results: str = "projects.json", dpath_configs: str = "co
 
         Debugging in: https://app.slack.com/block-kit-builder
         """
-        assert os.path.isfile(fpath_results), f"missing results data / JSON: {fpath_results}"
-        assert os.path.isdir(dpath_configs), f"missing config folder: {dpath_configs}"
+        if not os.path.isfile(fpath_results):
+            raise FileNotFoundError(f"missing results data / JSON: {fpath_results}")
+        if not os.path.isdir(dpath_configs):
+            raise NotADirectoryError(f"missing config folder: {dpath_configs}")
 
         with open(fpath_results) as fp:
             data = json.load(fp)

_actions/test_cli.py

@@ -22,5 +22,5 @@
         "specify_tests",
     ],
 )
-def test_assistant_commands(cmd):
+def test_assistant_commands(cmd: str) -> None:
     AssistantCLI().__getattribute__(cmd)(_PATH_CONFIG)

configs/Lightning-AI/bolts_pl-release.yaml

@@ -20,7 +20,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
     # install_extras: all
 
 testing:

configs/Lightning-AI/flash_pl-release.yaml

@@ -25,7 +25,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
   - name: torchmetrics
     HTTPS: https://github.com/Lightning-AI/metrics.git
     checkout: release/stable

configs/Lightning-AI/metrics_pl-release.yaml

@@ -20,7 +20,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
     # install_extras: all
 
 runtimes:

configs/NVIDIA/nemo-release.yaml

@@ -18,7 +18,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
     install_extras: loggers
 
 testing:

configs/SeanNaren/deepspeech-release.yaml

@@ -22,7 +22,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
 
 runtimes:
   - {os: "ubuntu-20.04", python: "3.8"}

configs/manujosephv/pytorch-tabular_lit-release.yaml

@@ -19,7 +19,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
     # install_extras: all
 # TODO: enable after fixing compatibility with task
 #  - name: torchmetrics

configs/microsoft/deepspeed-release.yaml

@@ -16,4 +16,4 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS

configs/neptune-ai/lightning_integration.yaml

@@ -28,7 +28,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
   - name: Faker
     checkout: 15.0.0
 

configs/vturrisi/solo-learn_lit-release.yaml

@@ -19,7 +19,7 @@ env:
 dependencies:
   - name: pytorch-lightning
     HTTPS: https://github.com/Lightning-AI/lightning.git
-    checkout: release/stable
+    checkout: release/LTS
     # install_extras: all
   - name: torchmetrics
     HTTPS: https://github.com/Lightning-AI/metrics.git

pyproject.toml

@@ -44,18 +44,30 @@ line-length = 120
 select = [
     "E", "W",  # see: https://pypi.org/project/pycodestyle
     "F",  # see: https://pypi.org/project/pyflakes
+    "I",  #see: https://pypi.org/project/isort/
     "D",  # see: https://pypi.org/project/pydocstyle
     "N",  # see: https://pypi.org/project/pep8-naming
+    "S",  # see: https://pypi.org/project/flake8-bandit
 ]
 extend-select = [
+    "A",  # see: https://pypi.org/project/flake8-builtins
+    "B",  # see: https://pypi.org/project/flake8-bugbear
     "C4",  # see: https://pypi.org/project/flake8-comprehensions
     "PT",  # see: https://pypi.org/project/flake8-pytest-style
     "RET",  # see: https://pypi.org/project/flake8-return
     "SIM",  # see: https://pypi.org/project/flake8-simplify
+    "YTT",  # see: https://pypi.org/project/flake8-2020
+    "ANN",  # see: https://pypi.org/project/flake8-annotations
+    "TID",  # see: https://pypi.org/project/flake8-tidy-imports/
+    "T10",  # see: https://pypi.org/project/flake8-debugger
+    "Q",  # see: https://pypi.org/project/flake8-quotes
+    "EXE",  # see: https://pypi.org/project/flake8-executable
+    "ISC",  # see: https://pypi.org/project/flake8-implicit-str-concat
+    "PIE",  # see: https://pypi.org/project/flake8-pie
+    "PLE",  # see: https://pypi.org/project/pylint/
+    "PERF",  # see: https://pypi.org/project/perflint/
+    "PYI",  # see: https://pypi.org/project/flake8-pyi/
 ]
-#ignore = [
-#    "E731",  # Do not assign a lambda expression, use a def
-#]
 # Exclude a variety of commonly ignored directories.
 exclude = [
     ".eggs",