Future 4/n: test & legacy in test/ folder

.actions/pull_legacy_checkpoints.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 # Run this script from the project root.
 URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip"
-mkdir -p legacy
+mkdir -p tests/legacy
 # wget is simpler but does not work on Windows
-python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'legacy/checkpoints.zip')"
-ls -l legacy/
-unzip -o legacy/checkpoints.zip -d legacy/
-ls -l legacy/checkpoints/
+python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'tests/legacy/checkpoints.zip')"
+ls -l tests/legacy/
+unzip -o tests/legacy/checkpoints.zip -d tests/legacy/
+ls -l tests/legacy/checkpoints/

.azure-pipelines/gpu-benchmark.yml

@@ -40,8 +40,8 @@ jobs:
         pip list
       displayName: 'Install package'
 
-    - bash: |
-        python -m pytest tests/benchmarks -v --durations=0
-      displayName: 'Testing: benchmarks'
+    - bash: python -m pytest benchmarks -v --durations=0
       env:
         PL_RUNNING_BENCHMARKS: 1
+      workingDirectory: tests/tests_pytorch
+      displayName: 'Testing: PyTorch benchmarks'

.azure-pipelines/gpu-tests.yml

@@ -17,7 +17,7 @@ pr:
   - "release/*"
 
 jobs:
-  - job: pytest
+  - job: testing
     strategy:
       matrix:
         'PyTorch - stable':
@@ -67,24 +67,27 @@ jobs:
     - bash: bash .actions/pull_legacy_checkpoints.sh
       displayName: 'Get legacy checkpoints'
 
-    - bash: python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning
-      workingDirectory: src
-      displayName: 'Testing: doctests'
+    - bash: python -m coverage run --source pytorch_lightning -m pytest
+      workingDirectory: src/pytorch_lightning
+      displayName: 'Testing: PyTorch doctests'
 
-    - bash: python -m coverage run --source pytorch_lightning -m pytest tests --ignore tests/benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50
-      displayName: 'Testing: unittests'
+    - bash: python -m coverage run --source pytorch_lightning -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50
+      displayName: 'Testing: PyTorch standard'
+      workingDirectory: tests/tests_pytorch
 
-    - bash: bash tests/standalone_tests.sh
+    - bash: bash run_standalone_tests.sh
+      workingDirectory: tests/tests_pytorch
       env:
         PL_USE_MOCKED_MNIST: "1"
-      displayName: 'Testing: standalone'
+      displayName: 'Testing: PyTorch standalone tests'
 
     - bash: |
         python -m coverage report
         python -m coverage xml
         python -m coverage html
         python -m codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) --flags=gpu,pytest --name="GPU-coverage" --env=linux,azure
         ls -l
+      workingDirectory: tests/tests_pytorch
       displayName: 'Statistics'
 
     - task: PublishTestResults@2
@@ -94,16 +97,6 @@ jobs:
         testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)'
       condition: succeededOrFailed()
 
-    # todo: re-enable after schema check pass, also atm it seems does not have any effect
-    #- task: PublishCodeCoverageResults@2
-    #  displayName: 'Publish coverage report'
-    #  inputs:
-    #    codeCoverageTool: 'Cobertura'
-    #    summaryFileLocation: 'coverage.xml'
-    #    reportDirectory: '$(Build.SourcesDirectory)/htmlcov'
-    #    testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
-    #  condition: succeededOrFailed()
-
     - script: |
         set -e
         bash run_ddp_examples.sh
@@ -113,8 +106,8 @@ jobs:
       workingDirectory: examples
       env:
         PL_USE_MOCKED_MNIST: "1"
-      displayName: 'Testing: examples'
+      displayName: 'Testing: PyTorch examples'
 
-    - bash: |
-        python -m pytest tests/benchmarks -v --maxfail=2 --durations=0
-      displayName: 'Testing: benchmarks'
+    - bash: python -m pytest benchmarks -v --maxfail=2 --durations=0
+      workingDirectory: tests/tests_pytorch
+      displayName: 'Testing: PyTorch benchmarks'

.azure-pipelines/hpu-tests.yml

@@ -14,8 +14,7 @@ pr:
   - "release/*"
 
 jobs:
-  - job: tests
-
+  - job: testing
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "10"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -36,15 +35,21 @@ jobs:
       displayName: 'Install dependencies'
 
     - bash: |
-        python -m pytest -sv tests/accelerators/test_hpu.py --forked --junitxml=hpu1_test-results.xml
+        python -m pytest -sv accelerators/test_hpu.py --forked --junitxml=hpu1_test-results.xml
+      workingDirectory: tests/tests_pytorch
       displayName: 'Single card HPU test'
 
     - bash: |
-        python -m pytest -sv tests/accelerators/test_hpu.py --forked --hpus 8 --junitxml=hpu8_test-results.xml
+        python -m pytest -sv accelerators/test_hpu.py --forked --hpus 8 --junitxml=hpu8_test-results.xml
+      workingDirectory: tests/tests_pytorch
       displayName: 'Multi card(8) HPU test'
 
     - bash: |
-        python -m pytest -sv tests/plugins/precision/hpu/test_hpu.py --hmp-bf16 'tests/plugins/precision/hpu/ops_bf16.txt' --hmp-fp32 'tests/plugins/precision/hpu/ops_fp32.txt' --forked --junitxml=hpu1_precision_test-results.xml
+        python -m pytest -sv plugins/precision/hpu/test_hpu.py --hmp-bf16 \
+          'plugins/precision/hpu/ops_bf16.txt' --hmp-fp32 \
+          'plugins/precision/hpu/ops_fp32.txt' --forked \
+          --junitxml=hpu1_precision_test-results.xml
+      workingDirectory: tests/tests_pytorch
       displayName: 'HPU precision test'
 
     - bash: |

.azure-pipelines/ipu-tests.yml

@@ -16,8 +16,7 @@ variables:
   value: "poplar_sdk-ubuntu_20_04-2.3.1+793-89796d462d"
 
 jobs:
-  - job: tests
-
+  - job: testing
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "15"
     pool: graphcore-ipus
@@ -73,16 +72,17 @@ jobs:
     - bash: |
         source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
         source ${{ variables.poplar_sdk }}/popart-ubuntu*/enable.sh
-        cd src
-        python -m pytest pytorch_lightning
-      displayName: 'DocTests'
+        python -m coverage run --source pytorch_lightning -m pytest
+      workingDirectory: src/pytorch_lightning
+      displayName: 'Testing: PyTorch doctests'
 
     - bash: |
         source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
         source ${{ variables.poplar_sdk }}/popart-ubuntu*/enable.sh
-        python -m coverage run --source pytorch_lightning -m pytest tests -vv --durations=50
+        python -m coverage run --source pytorch_lightning -m pytest -vv --durations=50
       env:
         MKL_THREADING_LAYER: "GNU"
         POPTORCH_WAIT_FOR_IPU: 1
         PL_RUN_IPU_TESTS: 1
-      displayName: 'UnitTests'
+      displayName: 'Testing: PyTorch standard'
+      workingDirectory: tests/tests_pytorch

.github/CONTRIBUTING.md

@@ -240,11 +240,11 @@ Here is the process to create a new test
 - 3. Use **BoringModel and derivates to test out your code**.
 
 ```python
-# TEST SHOULD BE IN YOUR FILE: tests/..../...py
+# TEST SHOULD BE IN YOUR FILE: tests/.../test_file.py
 # TEST CODE TEMPLATE
 
 # [OPTIONAL] pytest decorator
-# @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
+# @RunIf(min_cuda_gpus=1)
 def test_explain_what_is_being_tested(tmpdir):
     """
     Test description about text reason to be
@@ -269,7 +269,7 @@ def test_explain_what_is_being_tested(tmpdir):
 run our/your test with
 
 ```bash
-python -m pytest tests/..../...py::test_explain_what_is_being_tested -v --capture=no
+python -m pytest tests/.../test_file.py::test_explain_what_is_being_tested -v --capture=no
 ```
 
 #### How to fix PR with mixed base and target branches?

.github/workflows/ci_test-conda.yml

@@ -31,7 +31,7 @@ jobs:
     timeout-minutes: 30
     steps:
     - name: Workaround for https://github.com/actions/checkout/issues/760
-      run: git config --global --add safe.directory /__w/pytorch-lightning/pytorch-lightning
+      run: git config --global --add safe.directory /__w/lightning/lightning
 
     - uses: actions/checkout@v2
 
@@ -66,20 +66,20 @@ jobs:
     - name: Pull legacy checkpoints
       run: bash .actions/pull_legacy_checkpoints.sh
 
-    - name: Testing
-      run: |
-        coverage run --source pytorch_lightning -m pytest --timeout 150 tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
+    - name: Testing PyTorch
+      working-directory: tests/tests_pytorch
+      run: coverage run --source pytorch_lightning -m pytest -v --timeout 150 --durations=50 --junitxml=results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
 
     - name: Upload pytest results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}
-        path: junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
-        if-no-files-found: error
+        name: unittest-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}
+        path: tests/tests_pytorch/results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
       if: failure()
 
     - name: Statistics
       if: success()
+      working-directory: tests/tests_pytorch
       run: |
         coverage report
         coverage xml
@@ -91,7 +91,7 @@ jobs:
       continue-on-error: true
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml
+        file: tests/tests_pytorch/coverage.xml
         flags: cpu,pytest,torch${{ matrix.pytorch-version }}
         name: CPU-coverage
         fail_ci_if_error: false

.github/workflows/ci_test-full.yml

@@ -122,17 +122,16 @@ jobs:
     - name: Sanity check
       run: python requirements/check-avail-extras.py
 
-    - name: Testing
-      run: |
-        # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        coverage run --source pytorch_lightning -m pytest tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+    - name: Testing PyTorch
+      working-directory: tests/tests_pytorch
+      # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
+      run: coverage run --source pytorch_lightning -m pytest -v --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
 
     - name: Upload pytest results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
-        path: junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        if-no-files-found: error
+        name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
+        path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
       if: failure()
 
     - name: Prepare Examples
@@ -147,6 +146,7 @@ jobs:
 
     - name: Statistics
       if: success()
+      working-directory: tests/tests_pytorch
       run: |
         coverage report
         coverage xml
@@ -158,7 +158,7 @@ jobs:
       continue-on-error: true
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml
+        file: tests/tests_pytorch/coverage.xml
         flags: cpu,pytest,python${{ matrix.python-version }}
         name: CPU-coverage
         fail_ci_if_error: false

.github/workflows/ci_test-slow.yml

@@ -53,22 +53,22 @@ jobs:
         pip list
       shell: bash
 
-    - name: Tests
-      run: |
-        coverage run --source pytorch_lightning -m pytest tests -v --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}.xml
+    - name: Testing PyTorch
+      working-directory: tests/tests_pytorch
+      run: coverage run --source pytorch_lightning -m pytest -v --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}.xml
       env:
         PL_RUN_SLOW_TESTS: 1
 
     - name: Upload pytest test results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest-results-${{ runner.os }}-py${{ matrix.python-version }}
-        path: junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}.xml
-        if-no-files-found: error
+        name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}
+        path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}.xml
       if: failure()
 
     - name: Statistics
       if: success()
+      working-directory: tests/tests_pytorch
       run: |
         coverage report
         coverage xml
@@ -80,7 +80,7 @@ jobs:
       continue-on-error: true
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml
+        file: tests/tests_pytorch/coverage.xml
         flags: cpu,pytest,torch${{ matrix.pytorch-version }}
         name: CPU-coverage
         fail_ci_if_error: false

.github/workflows/code-checks.yml

@@ -36,6 +36,6 @@ jobs:
         pip install '.[dev]'
         pip list
 
-    - name: Type check
-      working-directory: ./src
-      run: mypy -p pytorch_lightning --install-types --non-interactive --config-file ../pyproject.toml
+    - name: Type check PyTorch
+      working-directory: src/pytorch_lightning
+      run: mypy . --install-types --non-interactive --config-file ../../pyproject.toml

.github/workflows/legacy-checkpoints.yml

@@ -29,12 +29,12 @@ jobs:
     - name: Generate checkpoint
       run: |
         while IFS= read -r line; do
-            bash legacy/generate_checkpoints.sh $line
-        done <<< $(cat legacy/back-compatible-versions.txt)
+            bash tests/legacy/generate_checkpoints.sh $line
+        done <<< $(cat tests/legacy/back-compatible-versions.txt)
 
     - name: Push files to S3
-      working-directory: ./legacy
+      working-directory: ./tests/legacy
       run: |
-        aws s3 sync legacy/checkpoints/ s3://pl-public-data/legacy/checkpoints/
+        aws s3 sync checkpoints/ s3://pl-public-data/legacy/checkpoints/
         zip -r checkpoints.zip checkpoints
         aws s3 cp checkpoints.zip s3://pl-public-data/legacy/ --acl public-read

.github/workflows/release-pypi.yml

@@ -123,8 +123,9 @@ jobs:
         path: dist
 
     - name: Pull files from S3
+      working-directory: ./tests/legacy
       run: |
-        aws s3 cp --recursive s3://pl-public-data/legacy/checkpoints/ legacy/checkpoints/ #  --acl public-read
+        aws s3 cp --recursive s3://pl-public-data/legacy/checkpoints/ checkpoints/ #  --acl public-read
         ls -l legacy/checkpoints/
 
     - name: Generate checkpoint
@@ -137,7 +138,7 @@ jobs:
         bash legacy/generate_checkpoints.sh $pl_ver
 
     - name: Push files to S3
-      working-directory: ./legacy
+      working-directory: ./tests/legacy
       run: |
         aws s3 sync legacy/checkpoints/ s3://pl-public-data/legacy/checkpoints/
         zip -r checkpoints.zip checkpoints

dockers/tpu-tests/tpu_test_cases.jsonnet

@@ -32,13 +32,13 @@ local tputests = base.BaseTest {
       pip install -e .
       echo $KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS
       export XRT_TPU_CONFIG="tpu_worker;0;${KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS:7}"
+      cd tests/tests_pytorch
       # TODO (@kaushikb11): Add device stats tests here
       coverage run --source=pytorch_lightning -m pytest -v --capture=no \
-          tests/strategies/test_tpu_spawn.py \
-          tests/profiler/test_xla_profiler.py \
-          src/pytorch_lightning/utilities/xla_device.py \
-          tests/accelerators/test_tpu.py \
-          tests/models/test_tpu.py
+          strategies/test_tpu_spawn.py \
+          profiler/test_xla_profiler.py \
+          accelerators/test_tpu.py \
+          models/test_tpu.py
       test_exit_code=$?
       echo "\n||| END PYTEST LOGS |||\n"
       coverage xml

pyproject.toml

@@ -7,10 +7,9 @@ requires = [
 
 [tool.isort]
 known_first_party = [
-    "docs",
     "pl_examples",
     "pytorch_lightning",
-    "tests",
+    "tests_pytorch",
 ]
 profile = "black"
 line_length = 120