From 84fe263e5381e229aaff6cbd9ac0f89196bbca52 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Mon, 5 Dec 2022 10:59:16 +0100
Subject: [PATCH 01/16] add metrics with torch

---
 .../Lightning-AI/metrics_torch-develop.yaml   | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 configs/Lightning-AI/metrics_torch-develop.yaml

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
new file mode 100644
index 00000000..c187d280
--- /dev/null
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -0,0 +1,27 @@
+target_repository:
+  HTTPS: https://github.com/Lightning-AI/metrics.git
+  # checkout a particular branch or a tag
+  checkout: master
+  # define installing package extras
+  install_extras: all
+  # copy some  test from the target repository
+  copy_tests:
+    - tests/unittests
+
+contact:
+  slack:
+    # Nicki Skafte
+    - USZ6EQ8A2
+
+dependencies:
+  - name: torch
+    HTTPS: https://github.com/pytorch/pytorch.git
+    checkout: master
+
+testing:
+  # additional pytest arguments
+  pytest_args: --strict
+
+runtimes:
+  - {os: "ubuntu-22.04", python: "3.10"}
+  - {os: "macOS-12", python: "3.10"}

From 51c7f46b0e6afa486cfb6ab2488b2c1e2b08f3de Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Mon, 5 Dec 2022 13:29:35 +0100
Subject: [PATCH 02/16] 90

---
 .github/workflows/ci_testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml
index 504cac11..b9acc408 100644
--- a/.github/workflows/ci_testing.yml
+++ b/.github/workflows/ci_testing.yml
@@ -44,7 +44,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.check-diff.outputs.matrix) }}
     # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 35
+    timeout-minutes: 90
     needs: check-diff
 
     steps:

From 8c5d414cafe6d329ffd486226e1d316ee9bb06fe Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 6 Dec 2022 09:50:17 +0100
Subject: [PATCH 03/16] nightly

---
 actions/assistant.py                          | 22 ++++++++++++-------
 .../Lightning-AI/metrics_torch-develop.yaml   |  5 +++--
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/actions/assistant.py b/actions/assistant.py
index 3afe14c4..ee9b3f4b 100644
--- a/actions/assistant.py
+++ b/actions/assistant.py
@@ -130,7 +130,11 @@ def _extras(extras: Union[str, list, tuple] = "") -> str:
     @staticmethod
     def _install_pip(repo: Dict[str, str]) -> str:
         """Create command for installing a project from source (if HTTPS is given) or from PyPI (if at least name is
-        given)."""
+        given).
+
+        Args:
+            repo: it is package or repository with additional key fields
+        """
         assert any(k in repo for k in ["HTTPS", "name"]), f"Missing key `HTTPS` or `name` among {repo.keys()}"
         # pip install -q 'https://github.com/...#egg=lightning-flash[tabular]
         name = repo.get("name")
@@ -146,20 +150,22 @@ def _install_pip(repo: Dict[str, str]) -> str:
                 password=repo.get("password"),
             )
 
-            cmd = f"git+{url}"
+            pkg = f"git+{url}"
             if "checkout" in repo:
                 assert isinstance(repo["checkout"], str)
-                cmd += f"@{repo['checkout']}"
+                pkg += f"@{repo['checkout']}"
             if "install_extras" in repo:
-                cmd += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]"
+                pkg += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]"
         else:
             # make installation from pypi package
-            cmd = name
+            pkg = name
             if "install_extras" in repo:
-                cmd += f"[{repo['install_extras']}]"
+                pkg += f"[{repo['install_extras']}]"
             if "checkout" in repo:
-                cmd += f"=={repo['checkout']}"
-        return "pip install --quiet " + cmd
+                pkg += f"=={repo['checkout']}"
+        flags = " ".join(["--quiet"] + repo.get('install_flags', []))
+        cmd = " ".join(["pip install", pkg, flags])
+        return cmd
 
     @staticmethod
     def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index c187d280..57800001 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -15,8 +15,9 @@ contact:
 
 dependencies:
   - name: torch
-    HTTPS: https://github.com/pytorch/pytorch.git
-    checkout: master
+    install_flags:
+      - "-f https://download.pytorch.org/whl/nightly/torch_nightly.html"
+      - "--pre"
 
 testing:
   # additional pytest arguments

From ebfa8719ea15a8a64297cb0e99720430fbd91571 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 6 Dec 2022 11:15:40 +0100
Subject: [PATCH 04/16] list

---
 actions/assistant.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/actions/assistant.py b/actions/assistant.py
index ee9b3f4b..08ad0b6e 100644
--- a/actions/assistant.py
+++ b/actions/assistant.py
@@ -163,8 +163,8 @@ def _install_pip(repo: Dict[str, str]) -> str:
                 pkg += f"[{repo['install_extras']}]"
             if "checkout" in repo:
                 pkg += f"=={repo['checkout']}"
-        flags = " ".join(["--quiet"] + repo.get('install_flags', []))
-        cmd = " ".join(["pip install", pkg, flags])
+        flags = set(["--quiet"] + repo.get("install_flags", []))
+        cmd = " ".join(["pip install", pkg, " ".join(flags)])
         return cmd
 
     @staticmethod
@@ -184,13 +184,18 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
         if "requirements_file" in repo:
             reqs = repo["requirements_file"]
             reqs = [reqs] if isinstance(reqs, str) else reqs
-            cmds.append(f"pip install --quiet --upgrade {' '.join([f'-r {req}' for req in reqs])}")
+            args = [f"-r {req}" for req in reqs] + ["--quiet", "--upgrade"]
+            cmds.append("pip install " + " ".join(args))
 
         pip_install = "."
         if "install_extras" in repo:
             pip_install += f"[{AssistantCLI._extras(repo['install_extras'])}]"
-        cmds.append(f"pip install --quiet {pip_install}")
+
+        flags = ["--quiet"] + repo.get("install_flags", [])
+        cmds.append("pip install " + " ".join([pip_install] + flags))
+        cmds.append("pip list")
         cmds.append("cd ..")
+
         if remove_dir:
             cmds.append(f"rm -rf {repo_name}")
         return cmds

From e7184961da967c7db09a5eb947901bb4e02ed4e8 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 6 Dec 2022 11:41:49 +0100
Subject: [PATCH 05/16] test

---
 configs/Lightning-AI/metrics_torch-develop.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 57800001..11c2caa7 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -3,7 +3,9 @@ target_repository:
   # checkout a particular branch or a tag
   checkout: master
   # define installing package extras
-  install_extras: all
+  install_extras:
+    - all
+    - test
   # copy some  test from the target repository
   copy_tests:
     - tests/unittests

From 1e6d59ebe4b1d26001a127e3fd956cf7f1f8a246 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 6 Dec 2022 11:59:04 +0100
Subject: [PATCH 06/16] list

---
 actions/assistant.py                            | 1 +
 configs/Lightning-AI/metrics_torch-develop.yaml | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/actions/assistant.py b/actions/assistant.py
index 08ad0b6e..3eb3194e 100644
--- a/actions/assistant.py
+++ b/actions/assistant.py
@@ -276,6 +276,7 @@ def prepare_env(config_file: str = "config.yaml", path_root: str = _PATH_ROOT) -
         reqs = config.get("dependencies", [])
         for req in reqs:
             script.append(AssistantCLI._install_pip(req))
+        script.append("pip list")
 
         script += AssistantCLI.before_commands(config_file, stage="test", as_append=True)
         return os.linesep.join(script)
diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 11c2caa7..206cc003 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -26,5 +26,5 @@ testing:
   pytest_args: --strict
 
 runtimes:
-  - {os: "ubuntu-22.04", python: "3.10"}
-  - {os: "macOS-12", python: "3.10"}
+  - {os: "ubuntu-20.04", python: "3.9"}
+  - {os: "macOS-12", python: "3.9"}

From 8b6d3b31077980bcba22450d02dc8999595d0db6 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 13 Dec 2022 17:18:06 +0100
Subject: [PATCH 07/16] audio_test

---
 configs/Lightning-AI/metrics_torch-develop.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 206cc003..7d96ca0d 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -6,6 +6,8 @@ target_repository:
   install_extras:
     - all
     - test
+  # install additional requirements from a file
+  requirements_file: requirements/audio_test.txt
   # copy some  test from the target repository
   copy_tests:
     - tests/unittests

From 2dcfc8e783a02adc9826f49e8caf6128c7ac5875 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 13 Dec 2022 17:37:47 +0100
Subject: [PATCH 08/16] data

---
 configs/Lightning-AI/metrics_torch-develop.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 7d96ca0d..58c6a167 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -23,6 +23,12 @@ dependencies:
       - "-f https://download.pytorch.org/whl/nightly/torch_nightly.html"
       - "--pre"
 
+# running before installing your project
+before_test:
+  - wget https://pl-public-data.s3.amazonaws.com/metrics/data.zip
+  - unzip -o data.zip
+  - ls -l _data/*
+
 testing:
   # additional pytest arguments
   pytest_args: --strict

From c7f7579564ebc3ac6e35177f0a0b642efd56e94a Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 13 Dec 2022 18:10:57 +0100
Subject: [PATCH 09/16] cd

---
 actions/assistant.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/actions/assistant.py b/actions/assistant.py
index e1c1ebf6..9fcc53e1 100644
--- a/actions/assistant.py
+++ b/actions/assistant.py
@@ -285,6 +285,7 @@ def prepare_env(config_file: str = "config.yaml", path_root: str = _PATH_ROOT) -
             script.append(AssistantCLI._install_pip(req))
         script.append("pip list")
 
+        script += [f"cd {AssistantCLI._FOLDER_TESTS}"]
         script += AssistantCLI.before_commands(config_file, stage="test", as_append=True)
         return os.linesep.join(script)
 

From f11018e9e41a64ee360e647b011329f9290fd190 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 13 Dec 2022 18:49:15 +0100
Subject: [PATCH 10/16] mv

---
 configs/Lightning-AI/metrics_torch-develop.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 58c6a167..32f5c3ab 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -27,7 +27,7 @@ dependencies:
 before_test:
   - wget https://pl-public-data.s3.amazonaws.com/metrics/data.zip
   - unzip -o data.zip
-  - ls -l _data/*
+  - mv _data tests/
 
 testing:
   # additional pytest arguments

From 446644eaee436a6ecae46fe6c780ddfc94ac5a7f Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Wed, 14 Dec 2022 02:42:43 +0100
Subject: [PATCH 11/16] to

---
 .github/workflows/ci_testing.yml                | 3 ++-
 actions/assistant.py                            | 2 +-
 configs/Lightning-AI/metrics_torch-develop.yaml | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml
index 398e4103..4a3bc12e 100644
--- a/.github/workflows/ci_testing.yml
+++ b/.github/workflows/ci_testing.yml
@@ -48,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.check-diff.outputs.matrix) }}
-    timeout-minutes: 90
+    timeout-minutes: 150
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python }}
@@ -90,6 +90,7 @@ jobs:
       id: extras
 
     - name: Prepare environment
+      timeout-minutes: 30
       run: |
         bash prepare_env.sh
 
diff --git a/actions/assistant.py b/actions/assistant.py
index 9fcc53e1..5df994c2 100644
--- a/actions/assistant.py
+++ b/actions/assistant.py
@@ -200,7 +200,7 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
 
         flags = AssistantCLI._get_flags(repo)
         cmds.append("pip install " + " ".join([pip_install] + flags))
-        cmds.append("pip list")
+        # cmds.append("pip list")
         cmds.append("cd ..")
 
         if remove_dir:
diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 32f5c3ab..2b74101d 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -31,7 +31,7 @@ before_test:
 
 testing:
   # additional pytest arguments
-  pytest_args: --strict
+  pytest_args: --timeout=120
 
 runtimes:
   - {os: "ubuntu-20.04", python: "3.9"}

From 2559ae902d9523a16e60248370416f9e40d1ed54 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Fri, 30 Dec 2022 12:33:04 +0100
Subject: [PATCH 12/16] --pr

---
 .github/workflows/ci_testing.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml
index 4a3bc12e..2c5aa83b 100644
--- a/.github/workflows/ci_testing.yml
+++ b/.github/workflows/ci_testing.yml
@@ -21,7 +21,7 @@ defaults:
 
 jobs:
   check-diff:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 5
     # Map a step output to a job output
     outputs:
@@ -32,13 +32,15 @@ jobs:
       with:
         python-version: 3.8
 
+    - name: Install requirements
+      run: pip install -q -r requirements.txt
+
     - name: Get PR diff
       id: diff-files
       env:
         PR_NUMBER: "${{ github.event.pull_request.number }}"
       run: |
-        pip install -q -r requirements.txt
-        pr_runtimes=$(python actions/assistant.py list_runtimes $PR_NUMBER 2>&1)
+        pr_runtimes=$(python actions/assistant.py list_runtimes --pr=$PR_NUMBER)
         echo $pr_runtimes
         echo "runtimes={include: $pr_runtimes}" >> $GITHUB_OUTPUT
 

From 78bb03162ac5f59185bed69ceca725a3b90ea873 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Fri, 30 Dec 2022 15:08:48 +0100
Subject: [PATCH 13/16] only ubuntu

---
 configs/Lightning-AI/metrics_torch-develop.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 2b74101d..5437f89e 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -34,5 +34,5 @@ testing:
   pytest_args: --timeout=120
 
 runtimes:
-  - {os: "ubuntu-20.04", python: "3.9"}
-  - {os: "macOS-12", python: "3.9"}
+  - {os: "ubuntu-22.04", python: "3.9"}
+#  - {os: "macOS-12", python: "3.9"}

From 2400efb21de74f9072b08ef90758c1af7e95bf10 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Mon, 16 Jan 2023 09:56:43 +0100
Subject: [PATCH 14/16] append

---
 .github/actions/pytester/action.yml | 1 +
 .github/workflows/ci_compatible.yml | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/actions/pytester/action.yml b/.github/actions/pytester/action.yml
index 588ee144..fb8a269d 100644
--- a/.github/actions/pytester/action.yml
+++ b/.github/actions/pytester/action.yml
@@ -47,6 +47,7 @@ runs:
     shell: bash
 
   - name: Prepare environment
+    timeout-minutes: 30
     run: bash prepare_env.sh
     shell: bash
 
diff --git a/.github/workflows/ci_compatible.yml b/.github/workflows/ci_compatible.yml
index 3dda3097..fd7d12c4 100644
--- a/.github/workflows/ci_compatible.yml
+++ b/.github/workflows/ci_compatible.yml
@@ -32,12 +32,14 @@ jobs:
       with:
         python-version: 3.8
 
+    - name: Install requirements
+      run: pip install -q -r requirements.txt
+
     - name: Get PR diff
       id: diff-files
       env:
         PR_NUMBER: "${{ github.event.pull_request.number }}"
       run: |
-        pip install -q -r requirements.txt
         pr_runtimes=$(python _actions/assistant.py list_runtimes $PR_NUMBER)
         echo $pr_runtimes
         echo "runtimes={include: $pr_runtimes}" >> $GITHUB_OUTPUT
@@ -56,7 +58,7 @@ jobs:
       max-parallel: 25
       fail-fast: false
       matrix: ${{ fromJSON(needs.generator.outputs.matrix) }}
-    timeout-minutes: 35
+    timeout-minutes: 180
     steps:
     - uses: actions/checkout@v3
 

From e2a6deceed9cfd67eae77d24f5c52f5f05324cdb Mon Sep 17 00:00:00 2001
From: Jirka Borovec <6035284+Borda@users.noreply.github.com>
Date: Mon, 16 Jan 2023 16:31:39 +0100
Subject: [PATCH 15/16] Apply suggestions from code review

---
 .github/actions/pytester/action.yml | 1 -
 .github/workflows/ci_compatible.yml | 2 +-
 _actions/assistant.py               | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/actions/pytester/action.yml b/.github/actions/pytester/action.yml
index fb8a269d..588ee144 100644
--- a/.github/actions/pytester/action.yml
+++ b/.github/actions/pytester/action.yml
@@ -47,7 +47,6 @@ runs:
     shell: bash
 
   - name: Prepare environment
-    timeout-minutes: 30
     run: bash prepare_env.sh
     shell: bash
 
diff --git a/.github/workflows/ci_compatible.yml b/.github/workflows/ci_compatible.yml
index fd7d12c4..57b76b1b 100644
--- a/.github/workflows/ci_compatible.yml
+++ b/.github/workflows/ci_compatible.yml
@@ -58,7 +58,7 @@ jobs:
       max-parallel: 25
       fail-fast: false
       matrix: ${{ fromJSON(needs.generator.outputs.matrix) }}
-    timeout-minutes: 180
+    timeout-minutes: 120
     steps:
     - uses: actions/checkout@v3
 
diff --git a/_actions/assistant.py b/_actions/assistant.py
index 63506b87..1dd5ccee 100644
--- a/_actions/assistant.py
+++ b/_actions/assistant.py
@@ -217,7 +217,7 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
 
         flags = AssistantCLI._get_flags(repo)
         cmds.append("pip install " + " ".join([pip_install] + flags))
-        # cmds.append("pip list")
+        cmds.append("pip list")
         cmds.append("cd ..")
 
         if remove_dir:

From 104ea55c63815c882697d7b26a6cf21046251bf8 Mon Sep 17 00:00:00 2001
From: Jirka Borovec <6035284+Borda@users.noreply.github.com>
Date: Mon, 16 Jan 2023 16:33:27 +0100
Subject: [PATCH 16/16] + Justus

---
 configs/Lightning-AI/metrics_torch-develop.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/configs/Lightning-AI/metrics_torch-develop.yaml b/configs/Lightning-AI/metrics_torch-develop.yaml
index 5437f89e..b85f6a19 100644
--- a/configs/Lightning-AI/metrics_torch-develop.yaml
+++ b/configs/Lightning-AI/metrics_torch-develop.yaml
@@ -16,6 +16,8 @@ contact:
   slack:
     # Nicki Skafte
     - USZ6EQ8A2
+    # Justus Schock
+    - U010XFPFETH
 
 dependencies:
   - name: torch