From 59fa32080d5256221aa36da56a735f28ed8c06ef Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Tue, 22 Nov 2022 20:04:29 +0100 Subject: [PATCH] Tests/App: refactor examples - structure (#15770) * rename _examples dir * refactor * clean * path * add inits * skip * e2e * azure * e2e * rev * unify single depth for ignore docs req. * group --- .azure/app-cloud-e2e.yml | 42 ++++++++++++------- .azure/gpu-tests-lite.yml | 2 +- .azure/gpu-tests-pytorch.yml | 4 +- .azure/hpu-tests.yml | 4 +- .azure/ipu-tests.yml | 4 +- .github/CODEOWNERS | 2 +- .github/checkgroup.yml | 5 +-- .github/labeler.yml | 12 +++--- .github/workflows/ci-app-examples.yml | 4 +- .../collect_failures/__init__.py | 0 .../tests_app_examples/components/__init__.py | 0 .../components/python/__init__.py | 0 .../custom_work_dependencies/__init__.py | 0 .../idle_timeout/__init__.py | 0 .../__init__.py | 0 .../apps/collect_failures/__init__.py | 1 + .../apps}/collect_failures/app.py | 0 .../apps}/collect_failures/requirements.txt | 0 .../apps/core_features_app/__init__.py | 1 + .../apps}/core_features_app/app.py | 2 +- .../apps/custom_work_dependencies/__init__.py | 1 + .../apps}/custom_work_dependencies/app.py | 0 .../apps/idle_timeout/__init__.py | 1 + .../apps}/idle_timeout/app.py | 0 .../conftest.py | 0 tests/tests_examples_app/local/__init__.py | 5 +++ .../local}/test_collect_failures.py | 4 +- .../local}/test_core_features_app.py | 4 +- .../local}/test_custom_work_dependencies.py | 4 +- .../local}/test_idle_timeout.py | 4 +- tests/tests_examples_app/public/__init__.py | 5 +++ .../public}/test_app_dag.py | 4 +- .../public}/test_argparse.py | 5 ++- .../public}/test_boring_app.py | 8 +--- .../public}/test_commands_and_api.py | 4 +- .../public}/test_drive.py | 4 +- .../public}/test_installation_commands_app.py | 4 +- .../public/test_layout.py} | 4 +- .../public}/test_multi_node.py | 8 ++-- .../public}/test_payload.py | 4 +- .../public/test_pickle_or_not.py} | 7 +++- .../public}/test_quick_start.py | 8 ++-- .../public}/test_scripts.py | 6 +-- .../public}/test_template_react_ui.py | 4 +- .../public}/test_template_streamlit_ui.py | 4 +- .../public}/test_v0_app.py | 8 ++-- 46 files changed, 110 insertions(+), 83 deletions(-) delete mode 100644 tests/tests_app_examples/collect_failures/__init__.py delete mode 100644 tests/tests_app_examples/components/__init__.py delete mode 100644 tests/tests_app_examples/components/python/__init__.py delete mode 100644 tests/tests_app_examples/custom_work_dependencies/__init__.py delete mode 100644 tests/tests_app_examples/idle_timeout/__init__.py rename tests/{tests_app_examples => tests_examples_app}/__init__.py (100%) create mode 100644 tests/tests_examples_app/apps/collect_failures/__init__.py rename tests/{tests_app_examples => tests_examples_app/apps}/collect_failures/app.py (100%) rename tests/{tests_app_examples => tests_examples_app/apps}/collect_failures/requirements.txt (100%) create mode 100644 tests/tests_examples_app/apps/core_features_app/__init__.py rename tests/{tests_app_examples => tests_examples_app/apps}/core_features_app/app.py (84%) create mode 100644 tests/tests_examples_app/apps/custom_work_dependencies/__init__.py rename tests/{tests_app_examples => tests_examples_app/apps}/custom_work_dependencies/app.py (100%) create mode 100644 tests/tests_examples_app/apps/idle_timeout/__init__.py rename tests/{tests_app_examples => tests_examples_app/apps}/idle_timeout/app.py (100%) rename tests/{tests_app_examples => tests_examples_app}/conftest.py (100%) create mode 100644 tests/tests_examples_app/local/__init__.py rename tests/{tests_app_examples => tests_examples_app/local}/test_collect_failures.py (88%) rename tests/{tests_app_examples => tests_examples_app/local}/test_core_features_app.py (79%) rename tests/{tests_app_examples => tests_examples_app/local}/test_custom_work_dependencies.py (81%) rename tests/{tests_app_examples => tests_examples_app/local}/test_idle_timeout.py (76%) create mode 100644 tests/tests_examples_app/public/__init__.py rename tests/{tests_app_examples => tests_examples_app/public}/test_app_dag.py (78%) rename tests/{tests_app_examples => tests_examples_app/public}/test_argparse.py (93%) rename tests/{tests_app_examples => tests_examples_app/public}/test_boring_app.py (82%) rename tests/{tests_app_examples => tests_examples_app/public}/test_commands_and_api.py (92%) rename tests/{tests_app_examples => tests_examples_app/public}/test_drive.py (77%) rename tests/{tests_app_examples => tests_examples_app/public}/test_installation_commands_app.py (82%) rename tests/{tests_app_examples/layout.py => tests_examples_app/public/test_layout.py} (78%) rename tests/{tests_app_examples => tests_examples_app/public}/test_multi_node.py (87%) rename tests/{tests_app_examples => tests_examples_app/public}/test_payload.py (71%) rename tests/{tests_app_examples/pickle_or_not.py => tests_examples_app/public/test_pickle_or_not.py} (62%) rename tests/{tests_app_examples => tests_examples_app/public}/test_quick_start.py (86%) rename tests/{tests_app_examples/components/python => tests_examples_app/public}/test_scripts.py (79%) rename tests/{tests_app_examples => tests_examples_app/public}/test_template_react_ui.py (86%) rename tests/{tests_app_examples => tests_examples_app/public}/test_template_streamlit_ui.py (86%) rename tests/{tests_app_examples => tests_examples_app/public}/test_v0_app.py (90%) diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index 0b0372e225adb..ec11400089e4b 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -27,16 +27,15 @@ pr: - "release/*" paths: include: + - ".actions/**" - ".azure/app-cloud-e2e.yml" - - "requirements/app/**" - "src/lightning_app/**" - - "tests/tests_app/**" - "examples/app_*/**" # some tests_app tests call examples files - - "tests/tests_app_examples/**" + - "requirements/app/**" + - "tests/tests_examples_app/**" - "setup.py" - - ".actions/**" exclude: - - "requirements/app/docs.txt" + - "requirements/*/docs.txt" - "*.md" - "**/*.md" @@ -59,30 +58,42 @@ jobs: matrix: 'App: v0_app': name: "v0_app" + dir: "public" 'App: boring_app': name: "boring_app" + dir: "public" 'App: template_streamlit_ui': name: "template_streamlit_ui" + dir: "public" 'App: template_react_ui': name: "template_react_ui" + dir: "public" # 'App: template_jupyterlab': # TODO: clarify where these files lives # name: "template_jupyterlab" - 'App: idle_timeout': - name: "idle_timeout" - 'App: collect_failures': - name: "collect_failures" - 'App: custom_work_dependencies': - name: "custom_work_dependencies" 'App: installation_commands_app': name: "installation_commands_app" + dir: "public" 'App: drive': name: "drive" + dir: "public" 'App: payload': name: "payload" + dir: "public" 'App: commands_and_api': name: "commands_and_api" + dir: "public" 'App: quick_start': name: "quick_start" + dir: "public" + 'App: idle_timeout': + name: "idle_timeout" + dir: "local" + 'App: collect_failures': + name: "collect_failures" + dir: "local" + 'App: custom_work_dependencies': + name: "custom_work_dependencies" + dir: "local" timeoutInMinutes: "20" cancelTimeoutInMinutes: "2" # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace @@ -124,7 +135,7 @@ jobs: # - bash: | # rm -rf examples/app_template_jupyterlab || true # git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab - # cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py + # cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_examples_app/test_template_jupyterlab.py # condition: eq(variables['name'], 'template_jupyterlab') # displayName: 'Clone Template Jupyter Lab Repo' @@ -152,8 +163,9 @@ jobs: - bash: | mkdir -p ${VIDEO_LOCATION} ls -l examples/${TEST_APP_NAME} - ls -l tests/tests_app_examples - python -m pytest tests/tests_app_examples/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud --timeout=1200 --capture=no -v --color=yes + ls -l tests/tests_examples_app/public + python -m pytest tests/tests_examples_app/${TEST_APP_FOLDER}/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud \ + --timeout=1200 --capture=no -v --color=yes env: HEADLESS: '1' PACKAGE_LIGHTNING: '1' @@ -161,6 +173,7 @@ jobs: VIDEO_LOCATION: $(video_artifact_dir) PR_NUMBER: $(local_id) TEST_APP_NAME: $(name) + TEST_APP_FOLDER: $(dir) HAR_LOCATION: './artifacts/hars' SLOW_MO: '50' #LAI_USER: $(LAI_USER) # for STAGING @@ -191,6 +204,7 @@ jobs: LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) PR_NUMBER: $(local_id) TEST_APP_NAME: $(name) + TEST_APP_FOLDER: $(dir) # GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning # GRID_USER_KEY: $(LIGHTNING_API_KEY) # TODO: clarify the meaning # GRID_URL: $(LIGHTNING_CLOUD_URL) diff --git a/.azure/gpu-tests-lite.yml b/.azure/gpu-tests-lite.yml index 608390de776cc..e7ea5b6e670f4 100644 --- a/.azure/gpu-tests-lite.yml +++ b/.azure/gpu-tests-lite.yml @@ -20,6 +20,7 @@ pr: - "release/*" paths: include: + - ".actions/**" - ".azure/gpu-tests-lite.yml" - "examples/lite/**" - "examples/run_lite_examples.sh" @@ -29,7 +30,6 @@ pr: - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config - - ".actions/**" exclude: - "requirements/lite/docs.txt" - "*.md" diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 8107d692be23b..be92b6ecd5f26 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -20,6 +20,7 @@ pr: - "release/*" paths: include: + - ".actions/**" - ".azure/gpu-tests-pytorch.yml" - "examples/run_pl_examples.sh" - "examples/pl_basics/backbone_image_classifier.py" @@ -33,9 +34,8 @@ pr: - "setup.cfg" # includes pytest config - "requirements/lite/**" - "src/lightning_lite/**" - - ".actions/**" exclude: - - "requirements/**/docs.txt" + - "requirements/*/docs.txt" - "*.md" - "**/*.md" diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml index 6afedb6df6757..2935f72e2b437 100644 --- a/.azure/hpu-tests.yml +++ b/.azure/hpu-tests.yml @@ -17,6 +17,7 @@ pr: - "release/*" paths: include: + - ".actions/**" - ".azure/hpu-tests.yml" - "examples/pl_hpu/mnist_sample.py" - "requirements/lite/**" @@ -25,9 +26,8 @@ pr: - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - ".actions/**" exclude: - - "requirements/**/docs.txt" + - "requirements/*/docs.txt" - "*.md" - "**/*.md" diff --git a/.azure/ipu-tests.yml b/.azure/ipu-tests.yml index fc292b25cbfd4..97948469e17e5 100644 --- a/.azure/ipu-tests.yml +++ b/.azure/ipu-tests.yml @@ -15,6 +15,7 @@ pr: - "release/*" paths: include: + - ".actions/**" - ".azure/ipu-tests.yml" - "requirements/lite/**" - "src/lightning_lite/**" @@ -22,9 +23,8 @@ pr: - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - ".actions/**" exclude: - - "requirements/**/docs.txt" + - "requirements/*/docs.txt" - "*.md" - "**/*.md" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 980583073c486..dbf5427379494 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -45,7 +45,7 @@ # Lightning App /src/lightning_app @tchaton @lantiga @awaelchli @hhsecond @ethanwharris /tests/tests_app @tchaton @lantiga @awaelchli @hhsecond @ethanwharris -/tests/tests_app_examples @tchaton @lantiga @awaelchli @hhsecond @ethanwharris +/tests/tests_examples_app @tchaton @lantiga @awaelchli @hhsecond @ethanwharris /examples/app_* @tchaton @lantiga @awaelchli @hhsecond @ethanwharris /.github/CODEOWNERS @williamfalcon diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 71c1d875abb60..1d1ef2d6439a1 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -245,7 +245,7 @@ subprojects: paths: - ".github/workflows/ci-app-examples.yml" - "src/lightning_app/**" - - "tests/tests_app_examples/**" + - "tests/tests_examples_app/**" - "examples/app_*/**" - "requirements/app/**" - "setup.py" @@ -269,9 +269,8 @@ subprojects: - ".azure/app-cloud-e2e.yml" - "requirements/app/**" - "src/lightning_app/**" - - "tests/tests_app/**" - "examples/app_*/**" # some tests_app tests call examples files - - "tests/tests_app_examples/**" + - "tests/tests_examples_app/**" - "setup.py" - ".actions/**" - "!requirements/app/docs.txt" diff --git a/.github/labeler.yml b/.github/labeler.yml index 1e0122683f76c..9be04ea82888d 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,7 +5,7 @@ app: all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*'] - any: ['tests/tests_app/**/*'] all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*'] -- any: ['tests/tests_app_examples/**/*'] +- any: ['tests/tests_examples_app/**/*'] all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*'] - any: ['examples/app_*/**/*'] all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*'] @@ -16,12 +16,12 @@ app: pl: - any: ['src/pytorch_lightning/**/*'] - all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*'] + all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*'] - any: ['examples/*pl_*/**/*'] - all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*'] + all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*'] - any: ['tests/tests_pytorch/**/*'] - all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*'] + all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*'] - any: ['docs/source-pytorch/**/*'] - all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*'] + all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*'] - any: ['requirements/pytorch/*'] - all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*'] + all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_examples_app/**/*', '!docs/source-app/**/*', '!requirements/app/*'] diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml index 9646efa27db66..c9d401fa2e9fb 100644 --- a/.github/workflows/ci-app-examples.yml +++ b/.github/workflows/ci-app-examples.yml @@ -10,7 +10,7 @@ on: paths: - ".github/workflows/ci-app-examples.yml" - "src/lightning_app/**" - - "tests/tests_app_examples/**" + - "tests/tests_examples_app/**" - "examples/app_*/**" - "requirements/app/**" - "setup.py" @@ -111,7 +111,7 @@ jobs: AWS_DEFAULT_REGION: us-east-1 PYTEST_ARTIFACT: results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml run: | - coverage run --source ${COVERAGE_SCOPE} -m pytest -m "not cloud" tests_app_examples --timeout=300 -vvvv --junitxml=$PYTEST_ARTIFACT --durations=0 + coverage run --source ${COVERAGE_SCOPE} -m pytest -m "not cloud" tests_examples_app --timeout=300 -vvvv --junitxml=$PYTEST_ARTIFACT --durations=0 - name: Upload pytest test results uses: actions/upload-artifact@v3 diff --git a/tests/tests_app_examples/collect_failures/__init__.py b/tests/tests_app_examples/collect_failures/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/tests_app_examples/components/__init__.py b/tests/tests_app_examples/components/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/tests_app_examples/components/python/__init__.py b/tests/tests_app_examples/components/python/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/tests_app_examples/custom_work_dependencies/__init__.py b/tests/tests_app_examples/custom_work_dependencies/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/tests_app_examples/idle_timeout/__init__.py b/tests/tests_app_examples/idle_timeout/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/tests_app_examples/__init__.py b/tests/tests_examples_app/__init__.py similarity index 100% rename from tests/tests_app_examples/__init__.py rename to tests/tests_examples_app/__init__.py diff --git a/tests/tests_examples_app/apps/collect_failures/__init__.py b/tests/tests_examples_app/apps/collect_failures/__init__.py new file mode 100644 index 0000000000000..b022c62adcdf3 --- /dev/null +++ b/tests/tests_examples_app/apps/collect_failures/__init__.py @@ -0,0 +1 @@ +# TODO: check the tests to be able to run without this init file diff --git a/tests/tests_app_examples/collect_failures/app.py b/tests/tests_examples_app/apps/collect_failures/app.py similarity index 100% rename from tests/tests_app_examples/collect_failures/app.py rename to tests/tests_examples_app/apps/collect_failures/app.py diff --git a/tests/tests_app_examples/collect_failures/requirements.txt b/tests/tests_examples_app/apps/collect_failures/requirements.txt similarity index 100% rename from tests/tests_app_examples/collect_failures/requirements.txt rename to tests/tests_examples_app/apps/collect_failures/requirements.txt diff --git a/tests/tests_examples_app/apps/core_features_app/__init__.py b/tests/tests_examples_app/apps/core_features_app/__init__.py new file mode 100644 index 0000000000000..b022c62adcdf3 --- /dev/null +++ b/tests/tests_examples_app/apps/core_features_app/__init__.py @@ -0,0 +1 @@ +# TODO: check the tests to be able to run without this init file diff --git a/tests/tests_app_examples/core_features_app/app.py b/tests/tests_examples_app/apps/core_features_app/app.py similarity index 84% rename from tests/tests_app_examples/core_features_app/app.py rename to tests/tests_examples_app/apps/core_features_app/app.py index 069384f721290..ae1f3a2104db0 100644 --- a/tests/tests_app_examples/core_features_app/app.py +++ b/tests/tests_examples_app/apps/core_features_app/app.py @@ -8,7 +8,7 @@ def __init__(self): super().__init__() def run(self): - # these env vars are set here: tests/tests_app_examples/test_core_features_app.py:15 + # these env vars are set here: tests/tests_examples_app/test_core_features_app.py:15 assert os.getenv("FOO", "") == "bar" assert os.getenv("BLA", "") == "bloz" self._exit() diff --git a/tests/tests_examples_app/apps/custom_work_dependencies/__init__.py b/tests/tests_examples_app/apps/custom_work_dependencies/__init__.py new file mode 100644 index 0000000000000..b022c62adcdf3 --- /dev/null +++ b/tests/tests_examples_app/apps/custom_work_dependencies/__init__.py @@ -0,0 +1 @@ +# TODO: check the tests to be able to run without this init file diff --git a/tests/tests_app_examples/custom_work_dependencies/app.py b/tests/tests_examples_app/apps/custom_work_dependencies/app.py similarity index 100% rename from tests/tests_app_examples/custom_work_dependencies/app.py rename to tests/tests_examples_app/apps/custom_work_dependencies/app.py diff --git a/tests/tests_examples_app/apps/idle_timeout/__init__.py b/tests/tests_examples_app/apps/idle_timeout/__init__.py new file mode 100644 index 0000000000000..b022c62adcdf3 --- /dev/null +++ b/tests/tests_examples_app/apps/idle_timeout/__init__.py @@ -0,0 +1 @@ +# TODO: check the tests to be able to run without this init file diff --git a/tests/tests_app_examples/idle_timeout/app.py b/tests/tests_examples_app/apps/idle_timeout/app.py similarity index 100% rename from tests/tests_app_examples/idle_timeout/app.py rename to tests/tests_examples_app/apps/idle_timeout/app.py diff --git a/tests/tests_app_examples/conftest.py b/tests/tests_examples_app/conftest.py similarity index 100% rename from tests/tests_app_examples/conftest.py rename to tests/tests_examples_app/conftest.py diff --git a/tests/tests_examples_app/local/__init__.py b/tests/tests_examples_app/local/__init__.py new file mode 100644 index 0000000000000..dbd6181f8e89a --- /dev/null +++ b/tests/tests_examples_app/local/__init__.py @@ -0,0 +1,5 @@ +import os + +from lightning_app import _PROJECT_ROOT + +_PATH_APPS = os.path.join(_PROJECT_ROOT, "tests", "tests_examples_app", "apps") diff --git a/tests/tests_app_examples/test_collect_failures.py b/tests/tests_examples_app/local/test_collect_failures.py similarity index 88% rename from tests/tests_app_examples/test_collect_failures.py rename to tests/tests_examples_app/local/test_collect_failures.py index 6bda6bf4bd253..3e1c003be0cda 100644 --- a/tests/tests_app_examples/test_collect_failures.py +++ b/tests/tests_examples_app/local/test_collect_failures.py @@ -2,8 +2,8 @@ from time import sleep import pytest +from tests_examples_app.local import _PATH_APPS -from lightning_app import _PROJECT_ROOT from lightning_app.testing.testing import run_app_in_cloud @@ -22,7 +22,7 @@ def test_collect_failures_example_cloud() -> None: "good_value_of_i_4", "invalid_value_of_i_5", ] - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/collect_failures")) as ( + with run_app_in_cloud(os.path.join(_PATH_APPS, "collect_failures")) as ( _, _, fetch_logs, diff --git a/tests/tests_app_examples/test_core_features_app.py b/tests/tests_examples_app/local/test_core_features_app.py similarity index 79% rename from tests/tests_app_examples/test_core_features_app.py rename to tests/tests_examples_app/local/test_core_features_app.py index 3fd425350ad0a..d5ca57f3d8be6 100644 --- a/tests/tests_app_examples/test_core_features_app.py +++ b/tests/tests_examples_app/local/test_core_features_app.py @@ -1,7 +1,7 @@ import os from click.testing import CliRunner -from tests_app import _PROJECT_ROOT +from tests_examples_app.local import _PATH_APPS from lightning_app.cli.lightning_cli import run_app @@ -12,7 +12,7 @@ def test_core_features_app_example(): result = runner.invoke( run_app, [ - os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/core_features_app/app.py"), + os.path.join(_PATH_APPS, "core_features_app", "app.py"), "--blocking", "False", "--open-ui", diff --git a/tests/tests_app_examples/test_custom_work_dependencies.py b/tests/tests_examples_app/local/test_custom_work_dependencies.py similarity index 81% rename from tests/tests_app_examples/test_custom_work_dependencies.py rename to tests/tests_examples_app/local/test_custom_work_dependencies.py index b8971e0ef2148..edaaade95119f 100644 --- a/tests/tests_app_examples/test_custom_work_dependencies.py +++ b/tests/tests_examples_app/local/test_custom_work_dependencies.py @@ -2,7 +2,7 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.local import _PATH_APPS from lightning_app.testing.testing import run_app_in_cloud @@ -11,7 +11,7 @@ def test_custom_work_dependencies_example_cloud() -> None: # if requirements not installed, the app will fail with run_app_in_cloud( - os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/custom_work_dependencies/"), + os.path.join(_PATH_APPS, "custom_work_dependencies"), app_name="app.py", ) as (_, _, fetch_logs, _): has_logs = False diff --git a/tests/tests_app_examples/test_idle_timeout.py b/tests/tests_examples_app/local/test_idle_timeout.py similarity index 76% rename from tests/tests_app_examples/test_idle_timeout.py rename to tests/tests_examples_app/local/test_idle_timeout.py index f06181ce86ed3..1de9e03d0bc1b 100644 --- a/tests/tests_app_examples/test_idle_timeout.py +++ b/tests/tests_examples_app/local/test_idle_timeout.py @@ -2,14 +2,14 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.local import _PATH_APPS from lightning_app.testing.testing import run_app_in_cloud @pytest.mark.cloud def test_idle_timeout_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "tests/tests_app_examples/idle_timeout")) as ( + with run_app_in_cloud(os.path.join(_PATH_APPS, "idle_timeout")) as ( _, _, fetch_logs, diff --git a/tests/tests_examples_app/public/__init__.py b/tests/tests_examples_app/public/__init__.py new file mode 100644 index 0000000000000..b70149ce10b11 --- /dev/null +++ b/tests/tests_examples_app/public/__init__.py @@ -0,0 +1,5 @@ +import os + +from lightning_app import _PROJECT_ROOT + +_PATH_EXAMPLES = os.path.join(_PROJECT_ROOT, "examples") diff --git a/tests/tests_app_examples/test_app_dag.py b/tests/tests_examples_app/public/test_app_dag.py similarity index 78% rename from tests/tests_app_examples/test_app_dag.py rename to tests/tests_examples_app/public/test_app_dag.py index 6d9a865a0a000..fb27c54a1a398 100644 --- a/tests/tests_app_examples/test_app_dag.py +++ b/tests/tests_examples_app/public/test_app_dag.py @@ -2,14 +2,14 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud @pytest.mark.cloud def test_app_dag_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_dag")) as (_, _, fetch_logs, _): + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_dag")) as (_, _, fetch_logs, _): launch_log, finish_log = False, False while not (launch_log and finish_log): diff --git a/tests/tests_app_examples/test_argparse.py b/tests/tests_examples_app/public/test_argparse.py similarity index 93% rename from tests/tests_app_examples/test_argparse.py rename to tests/tests_examples_app/public/test_argparse.py index 7d34b7d0c00f3..1e176884934db 100644 --- a/tests/tests_app_examples/test_argparse.py +++ b/tests/tests_examples_app/public/test_argparse.py @@ -1,7 +1,8 @@ import os import sys -from lightning_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES + from lightning_app.testing.testing import application_testing from lightning_app.utilities.load_app import _patch_sys_argv @@ -10,7 +11,7 @@ def test_app_argparse_example(): original_argv = sys.argv command_line = [ - os.path.join(_PROJECT_ROOT, "examples/app_argparse/app.py"), + os.path.join(_PATH_EXAMPLES, "app_argparse", "app.py"), "--app_args", "--use_gpu", "--without-server", diff --git a/tests/tests_app_examples/test_boring_app.py b/tests/tests_examples_app/public/test_boring_app.py similarity index 82% rename from tests/tests_app_examples/test_boring_app.py rename to tests/tests_examples_app/public/test_boring_app.py index 42da5887741f9..a97bccb93229b 100644 --- a/tests/tests_app_examples/test_boring_app.py +++ b/tests/tests_examples_app/public/test_boring_app.py @@ -2,7 +2,7 @@ import pytest from click.testing import CliRunner -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.cli.lightning_cli import show from lightning_app.testing.testing import run_app_in_cloud, wait_for @@ -10,11 +10,7 @@ @pytest.mark.cloud def test_boring_app_example_cloud() -> None: - with run_app_in_cloud( - os.path.join(_PROJECT_ROOT, "examples/app_boring/"), - app_name="app_dynamic.py", - debug=True, - ) as ( + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_boring"), app_name="app_dynamic.py", debug=True,) as ( _, view_page, _, diff --git a/tests/tests_app_examples/test_commands_and_api.py b/tests/tests_examples_app/public/test_commands_and_api.py similarity index 92% rename from tests/tests_app_examples/test_commands_and_api.py rename to tests/tests_examples_app/public/test_commands_and_api.py index a9c5fe0892983..3105624432b78 100644 --- a/tests/tests_app_examples/test_commands_and_api.py +++ b/tests/tests_examples_app/public/test_commands_and_api.py @@ -4,7 +4,7 @@ import pytest import requests -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud @@ -12,7 +12,7 @@ @pytest.mark.timeout(300) @pytest.mark.cloud def test_commands_and_api_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_commands_and_api")) as ( + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_commands_and_api")) as ( admin_page, view_page, fetch_logs, diff --git a/tests/tests_app_examples/test_drive.py b/tests/tests_examples_app/public/test_drive.py similarity index 77% rename from tests/tests_app_examples/test_drive.py rename to tests/tests_examples_app/public/test_drive.py index 630e76b550e9e..cdf699ad9f8d4 100644 --- a/tests/tests_app_examples/test_drive.py +++ b/tests/tests_examples_app/public/test_drive.py @@ -2,14 +2,14 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud @pytest.mark.cloud def test_drive_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_drive")) as ( + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_drive")) as ( _, _, fetch_logs, diff --git a/tests/tests_app_examples/test_installation_commands_app.py b/tests/tests_examples_app/public/test_installation_commands_app.py similarity index 82% rename from tests/tests_app_examples/test_installation_commands_app.py rename to tests/tests_examples_app/public/test_installation_commands_app.py index 41ba18c06bc73..b303a3938c3a6 100644 --- a/tests/tests_app_examples/test_installation_commands_app.py +++ b/tests/tests_examples_app/public/test_installation_commands_app.py @@ -1,7 +1,7 @@ import os import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud @@ -10,7 +10,7 @@ def test_installation_commands_app_example_cloud() -> None: # This is expected to pass, since the "setup" flag is passed with run_app_in_cloud( - os.path.join(_PROJECT_ROOT, "examples/app_installation_commands"), + os.path.join(_PATH_EXAMPLES, "app_installation_commands"), app_name="app.py", extra_args=["--setup"], debug=True, diff --git a/tests/tests_app_examples/layout.py b/tests/tests_examples_app/public/test_layout.py similarity index 78% rename from tests/tests_app_examples/layout.py rename to tests/tests_examples_app/public/test_layout.py index ce65d3d9a825b..7a44a8060bfc0 100644 --- a/tests/tests_app_examples/layout.py +++ b/tests/tests_examples_app/public/test_layout.py @@ -1,7 +1,7 @@ import os from click.testing import CliRunner -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.cli.lightning_cli import run_app @@ -12,7 +12,7 @@ def test_layout_example(): result = runner.invoke( run_app, [ - os.path.join(_PROJECT_ROOT, "examples/app_layout/app.py"), + os.path.join(_PATH_EXAMPLES, "app_layout", "app.py"), "--blocking", "False", "--open-ui", diff --git a/tests/tests_app_examples/test_multi_node.py b/tests/tests_examples_app/public/test_multi_node.py similarity index 87% rename from tests/tests_app_examples/test_multi_node.py rename to tests/tests_examples_app/public/test_multi_node.py index f6fefa5b99fa5..79fd491b98751 100644 --- a/tests/tests_app_examples/test_multi_node.py +++ b/tests/tests_examples_app/public/test_multi_node.py @@ -2,7 +2,7 @@ import sys import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import application_testing, LightningTestApp @@ -16,9 +16,9 @@ def on_before_run_once(self): return res -@pytest.mark.skipif(True, reason="flaky") +@pytest.mark.skip(reason="flaky") def test_multi_node_example(monkeypatch): - monkeypatch.chdir(os.path.join(_PROJECT_ROOT, "examples/app_multi_node")) + monkeypatch.chdir(os.path.join(_PATH_EXAMPLES, "app_multi_node")) command_line = [ "app.py", "--blocking", @@ -51,7 +51,7 @@ def on_before_run_once(self): ) @pytest.mark.skipif(sys.platform == "win32", reason="flaky") def test_multi_node_examples(app_name, monkeypatch): - monkeypatch.chdir(os.path.join(_PROJECT_ROOT, "examples/app_multi_node")) + monkeypatch.chdir(os.path.join(_PATH_EXAMPLES, "app_multi_node")) command_line = [ app_name, "--blocking", diff --git a/tests/tests_app_examples/test_payload.py b/tests/tests_examples_app/public/test_payload.py similarity index 71% rename from tests/tests_app_examples/test_payload.py rename to tests/tests_examples_app/public/test_payload.py index b40b8ca52defd..501c0a298b681 100644 --- a/tests/tests_app_examples/test_payload.py +++ b/tests/tests_examples_app/public/test_payload.py @@ -2,14 +2,14 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud @pytest.mark.cloud def test_payload_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_payload")) as (_, _, fetch_logs, _): + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_payload")) as (_, _, fetch_logs, _): has_logs = False while not has_logs: diff --git a/tests/tests_app_examples/pickle_or_not.py b/tests/tests_examples_app/public/test_pickle_or_not.py similarity index 62% rename from tests/tests_app_examples/pickle_or_not.py rename to tests/tests_examples_app/public/test_pickle_or_not.py index d55f39b2db28c..8e64da650a877 100644 --- a/tests/tests_app_examples/pickle_or_not.py +++ b/tests/tests_examples_app/public/test_pickle_or_not.py @@ -1,18 +1,21 @@ import os +import pytest from click.testing import CliRunner -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.cli.lightning_cli import run_app +# TODO: Investigate why it doesn't work +@pytest.mark.skip(reason="test has been ignored for a while and seems not to be working :(") def test_pickle_or_not_example(): runner = CliRunner() result = runner.invoke( run_app, [ - os.path.join(_PROJECT_ROOT, "examples/app_pickle_or_not/app.py"), + os.path.join(_PATH_EXAMPLES, "app_pickle_or_not", "app.py"), "--blocking", "False", "--open-ui", diff --git a/tests/tests_app_examples/test_quick_start.py b/tests/tests_examples_app/public/test_quick_start.py similarity index 86% rename from tests/tests_app_examples/test_quick_start.py rename to tests/tests_examples_app/public/test_quick_start.py index ebb8583f6bd0f..9e43802d991fe 100644 --- a/tests/tests_app_examples/test_quick_start.py +++ b/tests/tests_examples_app/public/test_quick_start.py @@ -4,7 +4,7 @@ import pytest from click.testing import CliRunner -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app import LightningApp from lightning_app.cli.lightning_cli import run_app @@ -25,7 +25,7 @@ def run_once(self): # TODO: Investigate why it doesn't work -@pytest.mark.skipif(True, reason="test is skipped because CI was blocking all the PRs.") +@pytest.mark.skip(reason="test is skipped because CI was blocking all the PRs.") @_RunIf(pl=True, skip_windows=True, skip_linux=True) def test_quick_start_example(caplog, monkeypatch): """This test ensures the Quick Start example properly train and serve PyTorch Lightning.""" @@ -38,7 +38,7 @@ def test_quick_start_example(caplog, monkeypatch): result = runner.invoke( run_app, [ - os.path.join(_PROJECT_ROOT, "lightning-quick-start/app.py"), + os.path.join(_PATH_EXAMPLES, "lightning-quick-start", "app.py"), "--blocking", "False", "--open-ui", @@ -51,7 +51,7 @@ def test_quick_start_example(caplog, monkeypatch): @pytest.mark.cloud def test_quick_start_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples", "lightning-quick-start/")) as (_, view_page, _, _): + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "lightning-quick-start")) as (_, view_page, _, _): def click_gradio_demo(*_, **__): button = view_page.locator('button:has-text("Interactive demo")') diff --git a/tests/tests_app_examples/components/python/test_scripts.py b/tests/tests_examples_app/public/test_scripts.py similarity index 79% rename from tests/tests_app_examples/components/python/test_scripts.py rename to tests/tests_examples_app/public/test_scripts.py index 53a083cf40ad6..d57731acb9d55 100644 --- a/tests/tests_app_examples/components/python/test_scripts.py +++ b/tests/tests_examples_app/public/test_scripts.py @@ -2,7 +2,7 @@ import pytest from click.testing import CliRunner -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.cli.lightning_cli import run_app from lightning_app.testing.helpers import _run_script, _RunIf @@ -17,7 +17,7 @@ ], ) def test_scripts(file): - _run_script(str(os.path.join(_PROJECT_ROOT, f"examples/app_components/python/{file}"))) + _run_script(str(os.path.join(_PATH_EXAMPLES, f"app_components/python/{file}"))) @pytest.mark.skip(reason="causing some issues with CI, not sure if the test is actually needed") @@ -28,7 +28,7 @@ def test_components_app_example(): result = runner.invoke( run_app, [ - os.path.join(_PROJECT_ROOT, "examples/app_components/python/app.py"), + os.path.join(_PATH_EXAMPLES, "app_components/python/app.py"), "--blocking", "False", "--open-ui", diff --git a/tests/tests_app_examples/test_template_react_ui.py b/tests/tests_examples_app/public/test_template_react_ui.py similarity index 86% rename from tests/tests_app_examples/test_template_react_ui.py rename to tests/tests_examples_app/public/test_template_react_ui.py index 4b4588d2397e5..426e94195f7fd 100644 --- a/tests/tests_app_examples/test_template_react_ui.py +++ b/tests/tests_examples_app/public/test_template_react_ui.py @@ -2,7 +2,7 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud, wait_for @@ -10,7 +10,7 @@ @pytest.mark.cloud def test_template_react_ui_example_cloud() -> None: """This test ensures streamlit works in the cloud by clicking a button and checking the logs.""" - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_template_react_ui")) as ( + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_template_react_ui")) as ( _, view_page, fetch_logs, diff --git a/tests/tests_app_examples/test_template_streamlit_ui.py b/tests/tests_examples_app/public/test_template_streamlit_ui.py similarity index 86% rename from tests/tests_app_examples/test_template_streamlit_ui.py rename to tests/tests_examples_app/public/test_template_streamlit_ui.py index 30d6c38070988..c78022d4c878c 100644 --- a/tests/tests_app_examples/test_template_streamlit_ui.py +++ b/tests/tests_examples_app/public/test_template_streamlit_ui.py @@ -2,7 +2,7 @@ from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import run_app_in_cloud, wait_for @@ -10,7 +10,7 @@ @pytest.mark.cloud def test_template_streamlit_ui_example_cloud() -> None: """This test ensures streamlit works in the cloud by clicking a button and checking the logs.""" - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_template_streamlit_ui")) as ( + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_template_streamlit_ui")) as ( _, view_page, fetch_logs, diff --git a/tests/tests_app_examples/test_v0_app.py b/tests/tests_examples_app/public/test_v0_app.py similarity index 90% rename from tests/tests_app_examples/test_v0_app.py rename to tests/tests_examples_app/public/test_v0_app.py index b0b89eb99a17e..f6e46666c62a3 100644 --- a/tests/tests_app_examples/test_v0_app.py +++ b/tests/tests_examples_app/public/test_v0_app.py @@ -3,7 +3,7 @@ from typing import Tuple import pytest -from tests_app import _PROJECT_ROOT +from tests_examples_app.public import _PATH_EXAMPLES from lightning_app.testing.testing import application_testing, LightningTestApp, run_app_in_cloud, wait_for from lightning_app.utilities.enum import AppStage @@ -20,7 +20,7 @@ def run_once(self) -> Tuple[bool, float]: def test_v0_app_example(): command_line = [ - os.path.join(_PROJECT_ROOT, "examples/app_v0/app.py"), + os.path.join(_PATH_EXAMPLES, "app_v0", "app.py"), "--blocking", "False", "--open-ui", @@ -59,7 +59,7 @@ def check_content(button_name, text_content): ) def test_v0_app_example_byoc_cloud() -> None: with run_app_in_cloud( - os.path.join(_PROJECT_ROOT, "examples/app_v0"), + os.path.join(_PATH_EXAMPLES, "app_v0"), extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")], ) as ( _, @@ -71,7 +71,7 @@ def test_v0_app_example_byoc_cloud() -> None: @pytest.mark.cloud def test_v0_app_example_cloud() -> None: - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_v0")) as ( + with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_v0")) as ( _, view_page, fetch_logs,