Tests/App: refactor examples - structure (#15770)

* rename _examples dir

* refactor

* clean

* path

* add inits

* skip

* e2e

* azure

* e2e

* rev

* unify single depth for ignore docs req.

* group

.azure/app-cloud-e2e.yml

@@ -27,16 +27,15 @@ pr:
       - "release/*"
   paths:
     include:
+      - ".actions/**"
       - ".azure/app-cloud-e2e.yml"
-      - "requirements/app/**"
       - "src/lightning_app/**"
-      - "tests/tests_app/**"
       - "examples/app_*/**"  # some tests_app tests call examples files
-      - "tests/tests_app_examples/**"
+      - "requirements/app/**"
+      - "tests/tests_examples_app/**"
       - "setup.py"
-      - ".actions/**"
     exclude:
-      - "requirements/app/docs.txt"
+      - "requirements/*/docs.txt"
       - "*.md"
       - "**/*.md"
 
@@ -59,30 +58,42 @@ jobs:
       matrix:
         'App: v0_app':
           name: "v0_app"
+          dir: "public"
         'App: boring_app':
           name: "boring_app"
+          dir: "public"
         'App: template_streamlit_ui':
           name: "template_streamlit_ui"
+          dir: "public"
         'App: template_react_ui':
           name: "template_react_ui"
+          dir: "public"
         # 'App: template_jupyterlab':  # TODO: clarify where these files lives
         #   name: "template_jupyterlab"
-        'App: idle_timeout':
-          name: "idle_timeout"
-        'App: collect_failures':
-          name: "collect_failures"
-        'App: custom_work_dependencies':
-          name: "custom_work_dependencies"
         'App: installation_commands_app':
           name: "installation_commands_app"
+          dir: "public"
         'App: drive':
           name: "drive"
+          dir: "public"
         'App: payload':
           name: "payload"
+          dir: "public"
         'App: commands_and_api':
           name: "commands_and_api"
+          dir: "public"
         'App: quick_start':
           name: "quick_start"
+          dir: "public"
+        'App: idle_timeout':
+          name: "idle_timeout"
+          dir: "local"
+        'App: collect_failures':
+          name: "collect_failures"
+          dir: "local"
+        'App: custom_work_dependencies':
+          name: "custom_work_dependencies"
+          dir: "local"
     timeoutInMinutes: "20"
     cancelTimeoutInMinutes: "2"
     # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
@@ -124,7 +135,7 @@ jobs:
     # - bash: |
     #     rm -rf examples/app_template_jupyterlab || true
     #     git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
-    #     cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
+    #     cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_examples_app/test_template_jupyterlab.py
     #   condition: eq(variables['name'], 'template_jupyterlab')
     #   displayName: 'Clone Template Jupyter Lab Repo'
 
@@ -152,15 +163,17 @@ jobs:
     - bash: |
         mkdir -p ${VIDEO_LOCATION}
         ls -l examples/${TEST_APP_NAME}
-        ls -l tests/tests_app_examples
-        python -m pytest tests/tests_app_examples/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud --timeout=1200 --capture=no -v --color=yes
+        ls -l tests/tests_examples_app/public
+        python -m pytest tests/tests_examples_app/${TEST_APP_FOLDER}/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud \
+          --timeout=1200 --capture=no -v --color=yes
       env:
         HEADLESS: '1'
         PACKAGE_LIGHTNING: '1'
         CLOUD: '1'
         VIDEO_LOCATION: $(video_artifact_dir)
         PR_NUMBER: $(local_id)
         TEST_APP_NAME: $(name)
+        TEST_APP_FOLDER: $(dir)
         HAR_LOCATION: './artifacts/hars'
         SLOW_MO: '50'
         #LAI_USER: $(LAI_USER)  # for STAGING
@@ -191,6 +204,7 @@ jobs:
         LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
         PR_NUMBER: $(local_id)
         TEST_APP_NAME: $(name)
+        TEST_APP_FOLDER: $(dir)
         # GRID_USER_ID: $(LIGHTNING_USER_ID)  # TODO: clarify the meaning
         # GRID_USER_KEY: $(LIGHTNING_API_KEY)  # TODO: clarify the meaning
         # GRID_URL: $(LIGHTNING_CLOUD_URL)

.azure/gpu-tests-lite.yml

@@ -20,6 +20,7 @@ pr:
       - "release/*"
   paths:
     include:
+      - ".actions/**"
       - ".azure/gpu-tests-lite.yml"
       - "examples/lite/**"
       - "examples/run_lite_examples.sh"
@@ -29,7 +30,6 @@ pr:
       - "src/lightning_lite/**"
       - "tests/tests_lite/**"
       - "setup.cfg"  # includes pytest config
-      - ".actions/**"
     exclude:
       - "requirements/lite/docs.txt"
       - "*.md"

.azure/gpu-tests-pytorch.yml

@@ -20,6 +20,7 @@ pr:
       - "release/*"
   paths:
     include:
+      - ".actions/**"
       - ".azure/gpu-tests-pytorch.yml"
       - "examples/run_pl_examples.sh"
       - "examples/pl_basics/backbone_image_classifier.py"
@@ -33,9 +34,8 @@ pr:
       - "setup.cfg"  # includes pytest config
       - "requirements/lite/**"
       - "src/lightning_lite/**"
-      - ".actions/**"
     exclude:
-      - "requirements/**/docs.txt"
+      - "requirements/*/docs.txt"
       - "*.md"
       - "**/*.md"
 

.azure/hpu-tests.yml

@@ -17,6 +17,7 @@ pr:
       - "release/*"
   paths:
     include:
+      - ".actions/**"
       - ".azure/hpu-tests.yml"
       - "examples/pl_hpu/mnist_sample.py"
       - "requirements/lite/**"
@@ -25,9 +26,8 @@ pr:
       - "src/pytorch_lightning/**"
       - "tests/tests_pytorch/**"
       - "setup.cfg"  # includes pytest config
-      - ".actions/**"
     exclude:
-      - "requirements/**/docs.txt"
+      - "requirements/*/docs.txt"
       - "*.md"
       - "**/*.md"
 

.azure/ipu-tests.yml

@@ -15,16 +15,16 @@ pr:
       - "release/*"
   paths:
     include:
+      - ".actions/**"
       - ".azure/ipu-tests.yml"
       - "requirements/lite/**"
       - "src/lightning_lite/**"
       - "requirements/pytorch/**"
       - "src/pytorch_lightning/**"
       - "tests/tests_pytorch/**"
       - "setup.cfg"  # includes pytest config
-      - ".actions/**"
     exclude:
-      - "requirements/**/docs.txt"
+      - "requirements/*/docs.txt"
       - "*.md"
       - "**/*.md"
 

.github/CODEOWNERS

@@ -45,7 +45,7 @@
 # Lightning App
 /src/lightning_app                          @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
 /tests/tests_app                            @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
-/tests/tests_app_examples                   @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
+/tests/tests_examples_app                   @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
 /examples/app_*                             @tchaton @lantiga @awaelchli @hhsecond @ethanwharris
 
 /.github/CODEOWNERS                  @williamfalcon

.github/checkgroup.yml

@@ -245,7 +245,7 @@ subprojects:
     paths:
       - ".github/workflows/ci-app-examples.yml"
       - "src/lightning_app/**"
-      - "tests/tests_app_examples/**"
+      - "tests/tests_examples_app/**"
       - "examples/app_*/**"
       - "requirements/app/**"
       - "setup.py"
@@ -269,9 +269,8 @@ subprojects:
       - ".azure/app-cloud-e2e.yml"
       - "requirements/app/**"
       - "src/lightning_app/**"
-      - "tests/tests_app/**"
       - "examples/app_*/**"  # some tests_app tests call examples files
-      - "tests/tests_app_examples/**"
+      - "tests/tests_examples_app/**"
       - "setup.py"
       - ".actions/**"
       - "!requirements/app/docs.txt"

.github/labeler.yml

@@ -5,7 +5,7 @@ app:
   all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
 - any: ['tests/tests_app/**/*']
   all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
-- any: ['tests/tests_app_examples/**/*']
+- any: ['tests/tests_examples_app/**/*']
   all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
 - any: ['examples/app_*/**/*']
   all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
@@ -16,12 +16,12 @@ app:
 
 pl:
 - any: ['src/pytorch_lightning/**/*']
-  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*']
 - any: ['examples/*pl_*/**/*']
-  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*']
 - any: ['tests/tests_pytorch/**/*']
-  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*']
 - any: ['docs/source-pytorch/**/*']
-  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*']
 - any: ['requirements/pytorch/*']
-  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*']

.github/workflows/ci-app-examples.yml

@@ -10,7 +10,7 @@ on:
     paths:
       - ".github/workflows/ci-app-examples.yml"
       - "src/lightning_app/**"
-      - "tests/tests_app_examples/**"
+      - "tests/tests_examples_app/**"
       - "examples/app_*/**"
       - "requirements/app/**"
       - "setup.py"
@@ -111,7 +111,7 @@ jobs:
           AWS_DEFAULT_REGION: us-east-1
           PYTEST_ARTIFACT: results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
         run: |
-          coverage run --source ${COVERAGE_SCOPE} -m pytest -m "not cloud" tests_app_examples --timeout=300 -vvvv --junitxml=$PYTEST_ARTIFACT --durations=0
+          coverage run --source ${COVERAGE_SCOPE} -m pytest -m "not cloud" tests_examples_app --timeout=300 -vvvv --junitxml=$PYTEST_ARTIFACT --durations=0
 
       - name: Upload pytest test results
         uses: actions/upload-artifact@v3

tests/tests_examples_app/apps/collect_failures/__init__.py

@@ -0,0 +1 @@
+# TODO: check the tests to be able to run without this init file

tests/tests_examples_app/apps/core_features_app/__init__.py

@@ -0,0 +1 @@
+# TODO: check the tests to be able to run without this init file

tests/tests_app_examples/core_features_app/app.py → tests/tests_examples_app/apps/core_features_app/app.py

@@ -8,7 +8,7 @@ def __init__(self):
         super().__init__()
 
     def run(self):
-        # these env vars are set here: tests/tests_app_examples/test_core_features_app.py:15
+        # these env vars are set here: tests/tests_examples_app/test_core_features_app.py:15
         assert os.getenv("FOO", "") == "bar"
         assert os.getenv("BLA", "") == "bloz"
         self._exit()

tests/tests_examples_app/apps/custom_work_dependencies/__init__.py

@@ -0,0 +1 @@
+# TODO: check the tests to be able to run without this init file

tests/tests_examples_app/apps/idle_timeout/__init__.py

@@ -0,0 +1 @@
+# TODO: check the tests to be able to run without this init file

tests/tests_examples_app/local/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+from lightning_app import _PROJECT_ROOT
+
+_PATH_APPS = os.path.join(_PROJECT_ROOT, "tests", "tests_examples_app", "apps")

tests/tests_app_examples/test_collect_failures.py → tests/tests_examples_app/local/test_collect_failures.py

@@ -2,8 +2,8 @@
 from time import sleep
 
 import pytest
+from tests_examples_app.local import _PATH_APPS
 
-from lightning_app import _PROJECT_ROOT
 from lightning_app.testing.testing import run_app_in_cloud
 
 
@@ -22,7 +22,7 @@ def test_collect_failures_example_cloud() -> None:
         "good_value_of_i_4",
         "invalid_value_of_i_5",
     ]
-    with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/collect_failures")) as (
+    with run_app_in_cloud(os.path.join(_PATH_APPS, "collect_failures")) as (
         _,
         _,
         fetch_logs,

tests/tests_app_examples/test_core_features_app.py → tests/tests_examples_app/local/test_core_features_app.py

@@ -1,7 +1,7 @@
 import os
 
 from click.testing import CliRunner
-from tests_app import _PROJECT_ROOT
+from tests_examples_app.local import _PATH_APPS
 
 from lightning_app.cli.lightning_cli import run_app
 
@@ -12,7 +12,7 @@ def test_core_features_app_example():
     result = runner.invoke(
         run_app,
         [
-            os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/core_features_app/app.py"),
+            os.path.join(_PATH_APPS, "core_features_app", "app.py"),
             "--blocking",
             "False",
             "--open-ui",

tests/tests_app_examples/test_custom_work_dependencies.py → tests/tests_examples_app/local/test_custom_work_dependencies.py

@@ -2,7 +2,7 @@
 from time import sleep
 
 import pytest
-from tests_app import _PROJECT_ROOT
+from tests_examples_app.local import _PATH_APPS
 
 from lightning_app.testing.testing import run_app_in_cloud
 
@@ -11,7 +11,7 @@
 def test_custom_work_dependencies_example_cloud() -> None:
     # if requirements not installed, the app will fail
     with run_app_in_cloud(
-        os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/custom_work_dependencies/"),
+        os.path.join(_PATH_APPS, "custom_work_dependencies"),
         app_name="app.py",
     ) as (_, _, fetch_logs, _):
         has_logs = False

tests/tests_app_examples/test_idle_timeout.py → tests/tests_examples_app/local/test_idle_timeout.py

@@ -2,14 +2,14 @@
 from time import sleep
 
 import pytest
-from tests_app import _PROJECT_ROOT
+from tests_examples_app.local import _PATH_APPS
 
 from lightning_app.testing.testing import run_app_in_cloud
 
 
 @pytest.mark.cloud
 def test_idle_timeout_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/idle_timeout")) as (
+    with run_app_in_cloud(os.path.join(_PATH_APPS, "idle_timeout")) as (
         _,
         _,
         fetch_logs,

tests/tests_examples_app/public/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+from lightning_app import _PROJECT_ROOT
+
+_PATH_EXAMPLES = os.path.join(_PROJECT_ROOT, "examples")

tests/tests_app_examples/test_app_dag.py → tests/tests_examples_app/public/test_app_dag.py

@@ -2,14 +2,14 @@
 from time import sleep
 
 import pytest
-from tests_app import _PROJECT_ROOT
+from tests_examples_app.public import _PATH_EXAMPLES
 
 from lightning_app.testing.testing import run_app_in_cloud
 
 
 @pytest.mark.cloud
 def test_app_dag_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_dag")) as (_, _, fetch_logs, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_dag")) as (_, _, fetch_logs, _):
 
         launch_log, finish_log = False, False
         while not (launch_log and finish_log):

tests/tests_app_examples/test_argparse.py → tests/tests_examples_app/public/test_argparse.py

@@ -1,7 +1,8 @@
 import os
 import sys
 
-from lightning_app import _PROJECT_ROOT
+from tests_examples_app.public import _PATH_EXAMPLES
+
 from lightning_app.testing.testing import application_testing
 from lightning_app.utilities.load_app import _patch_sys_argv
 
@@ -10,7 +11,7 @@ def test_app_argparse_example():
     original_argv = sys.argv
 
     command_line = [
-        os.path.join(_PROJECT_ROOT, "examples/app_argparse/app.py"),
+        os.path.join(_PATH_EXAMPLES, "app_argparse", "app.py"),
         "--app_args",
         "--use_gpu",
         "--without-server",