Merge pull request #7135 from drew2a/fix/7134

Add profiling for `pytest`

.github/actions/profile/action.yml

@@ -0,0 +1,58 @@
+name: profile
+description: Run pytest with profiling
+inputs:
+  arguments:
+    default: './src'
+    description: 'Arguments for pytest'
+    required: true
+
+  artifact_name:
+    default: 'prof.svg'
+    description: 'A name for the artifact'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install profiling dependencies (linux)
+      if: ${{runner.os == 'Linux'}}
+      shell: bash
+      run: |
+        sudo apt-get -y install graphviz
+
+    - name: Install profiling dependencies (macOS)
+      if: ${{runner.os == 'macOS'}}
+      shell: bash
+      run: |
+        brew install graphviz
+
+    - name: Install profiling dependencies (Windows)
+      if: ${{runner.os == 'Windows'}}
+      shell: bash
+      run: |
+        choco install graphviz
+
+    - name: Run Pytest with --profile-svg
+      shell: bash
+      run: |
+        pytest ${{inputs.arguments}} --profile-svg
+
+    - name: Check combined.prof existence
+      id: check_combined_prof
+      uses: andstor/file-existence-action@v2
+      with:
+        files: prof/combined.prof
+
+    - name: Run svg generation for Windows
+      shell: bash
+      if: ${{always() && runner.os == 'Windows' && steps.check_combined_prof.outputs.files_exists == 'true'}}
+      run: |
+        gprof2dot -f pstats prof/combined.prof > prof/tmp
+        dot -Tsvg -o prof/combined.svg prof/tmp
+
+    - uses: actions/upload-artifact@v3
+      if: ${{always()}}
+      with:
+        name: ${{runner.os}}_${{inputs.artifact_name}}
+        path: ./prof/combined.svg
+        retention-days: 1

.github/workflows/!PR.yml

@@ -80,20 +80,23 @@ jobs:
 
   # PR is Ready only
   pytest_nix:
+    needs: changes
     if: ${{needs.changes.outputs.src == 'true' && !github.event.pull_request.draft}}
     uses: ./.github/workflows/pytest.yml
     with:
       python-version: 3.8
       matrix: '{"os": ["macos-latest", "ubuntu-latest"]}'
 
   guitest_nix:
+    needs: changes
     if: ${{needs.changes.outputs.src == 'true' && !github.event.pull_request.draft}}
     uses: ./.github/workflows/guitest.yml
     with:
       python-version: 3.8
       matrix: '{"os": ["macos-latest", "ubuntu-latest"]}'
 
   ubuntu:
+    needs: changes
     if: ${{needs.changes.outputs.build == 'true' && !github.event.pull_request.draft}}
     uses: ./.github/workflows/build_ubuntu.yml
     with:
@@ -102,6 +105,7 @@ jobs:
       python-version: 3.8
 
   windows:
+    needs: changes
     if: ${{needs.changes.outputs.build == 'true' && !github.event.pull_request.draft}}
     uses: ./.github/workflows/build_windows.yml
     with:
@@ -110,6 +114,7 @@ jobs:
       python-version: 3.8
 
   mac:
+    needs: changes
     if: ${{needs.changes.outputs.build == 'true' && !github.event.pull_request.draft}}
     uses: ./.github/workflows/build_mac.yml
     with:
@@ -118,6 +123,7 @@ jobs:
       python-version: 3.8
 
   documentation:
+    needs: changes
     if: ${{needs.changes.outputs.doc == 'true' && !github.event.pull_request.draft}}
     uses: ./.github/workflows/documentation.yml
     with:

.github/workflows/guitest.yml

@@ -9,17 +9,27 @@ on:
         required: false
 
       matrix:
-        default: '{"os":["windows-latest"]}'
+        default: '{"os":["windows-latest", "macos-latest", "ubuntu-latest"]}'
         type: string
         required: false
 
+      enable-profiling:
+        default: true
+        type: boolean
+        required: false
+
 jobs:
   run:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix: ${{fromJson(inputs.matrix)}}
 
+    timeout-minutes: 10
+
+    env:
+      PYTEST_ARGUMENTS: ./src/tribler/gui --guitests -v --randomly-seed=1 --exitfirst --disable-warnings
+
     steps:
       - uses: actions/checkout@v3
 
@@ -42,10 +52,22 @@ jobs:
         if: runner.os == 'Linux'
         uses: pyvista/setup-headless-display-action@v1
 
+      - name: Add --looptime
+        if: runner.os != 'Windows'
+        run: |
+          echo "PYTEST_ARGUMENTS=$PYTEST_ARGUMENTS --looptime" >> $GITHUB_ENV
+
       - name: Run GUI tests
-        timeout-minutes: 5
+        if: ${{!inputs.enable-profiling}}
         run: |
-          pytest ./src/tribler/gui --guitests -v --randomly-seed=1 --timeout=300
+          pytest ${PYTEST_ARGUMENTS}
+
+      - name: Run GUI tests (Profiler)
+        if: ${{inputs.enable-profiling}}
+        uses: ./.github/actions/profile
+        with:
+          artifact_name: guitests_prof.svg
+          arguments: ${PYTEST_ARGUMENTS}
 
       - uses: actions/upload-artifact@v3
         if: always()

.github/workflows/pytest.yml

@@ -9,17 +9,34 @@ on:
         required: false
 
       matrix:
-        default: '{"os":["ubuntu-latest"]}'
+        default: '{"os":["windows-latest", "macos-latest", "ubuntu-latest"]}'
         type: string
         required: false
 
+      enable_profiling:
+        default: true
+        type: boolean
+        required: false
+
 jobs:
   run:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix: ${{fromJson(inputs.matrix)}}
 
+    defaults:
+      run:
+        shell: bash
+
+    timeout-minutes: 10
+
+    env:
+      PYTEST_CORE_ARGUMENTS: ./src/tribler/core
+      PYTEST_TUNNELS_ARGUMENTS: ./src/tribler/core/components/tunnel/tests/test_full_session --tunneltests
+
+      PYTEST_COMMON_ARGUMENTS: --exitfirst --disable-warnings
+
     steps:
       - uses: actions/checkout@v3
 
@@ -29,22 +46,35 @@ jobs:
           python-version: ${{inputs.python-version}}
           requirements: requirements-test.txt
 
-      - name: Install windows dependencies
+      - name: Install dependencies (Windows)
         if: runner.os == 'Windows'
         uses: ./.github/actions/windows_dependencies
 
-      - name: Run Pytest (nix)
+      - name: Add --looptime
         if: runner.os != 'Windows'
-        timeout-minutes: 5
         run: |
-          pytest ./src/tribler/core --exitfirst --disable-warnings --looptime
+          echo "PYTEST_COMMON_ARGUMENTS=$PYTEST_COMMON_ARGUMENTS --looptime" >> $GITHUB_ENV
 
-      - name: Run Pytest (win)
-        if: runner.os == 'Windows'
-        timeout-minutes: 5
+      - name: Run Pytest
+        if: ${{!inputs.enable_profiling}}
         run: |
-          pytest ./src/tribler/core --exitfirst --disable-warnings
-          
+          pytest ${PYTEST_CORE_ARGUMENTS} ${PYTEST_COMMON_ARGUMENTS}
+
+      - name: Run Pytest (Profiler)
+        if: ${{inputs.enable_profiling}}
+        uses: ./.github/actions/profile
+        with:
+          artifact_name: pytest_prof.svg
+          arguments: ${PYTEST_CORE_ARGUMENTS} ${PYTEST_COMMON_ARGUMENTS}
+
       - name: Run Tunnels Tests
+        if: ${{!inputs.enable_profiling}}
         run: |
-          pytest ./src/tribler/core/components/tunnel/tests/test_full_session --tunneltests --exitfirst --disable-warnings
+          pytest ${PYTEST_TUNNELS_ARGUMENTS} ${PYTEST_COMMON_ARGUMENTS}
+
+      - name: Run Tunnel Tests (Profiler)
+        if: ${{inputs.enable_profiling}}
+        uses: ./.github/actions/profile
+        with:
+          artifact_name: tunneltest_prof.svg
+          arguments: ${PYTEST_TUNNELS_ARGUMENTS} ${PYTEST_COMMON_ARGUMENTS}

requirements-test.txt

@@ -13,4 +13,6 @@ looptime==0.2
 
 asynctest==0.13.0 # this library has to be installed to properly work with ipv8 TestBase.
 
-scipy==1.8.0
+scipy==1.8.0
+
+pytest-profiling==1.7.0 # for pytest profiling