test: Speedup azure pipelines workflows

.azure-pipelines/linux_libs.yaml

@@ -0,0 +1,7 @@
+steps:
+  - script: |
+      sudo apt-get install -y libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 \
+                              libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
+      /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile \
+                              --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX
+    displayName: "install libs and run Xvfb"

.azure-pipelines/pip_cache.yaml

@@ -0,0 +1,13 @@
+parameters:
+- name: key # name of the parameter; required
+  type: string
+- name: path # name of the parameter; required
+  type: string
+
+steps:
+  - bash: echo "##vso[task.setvariable variable=PY]$(python -VV)"
+  - task: CacheBeta@1
+    displayName: Cache pip
+    inputs:
+      key: ${{ parameters.key }}
+      path: ${{ parameters.path }}

.azure-pipelines/pyinstaller.yaml

@@ -0,0 +1,28 @@
+parameters:
+- name: test_path # name of the parameter; required
+  type: string # data type of the parameter; required
+- name: cache_dir # name of the parameter; required
+  type: string
+
+steps:
+  - {task: UsePythonVersion@0, inputs: {versionSpec: '3.9', architecture: x64}}
+  - template: pip_cache.yaml
+    parameters:
+      key: pyinstaller | requirements/requirements_pyinstaller.txt | "$(Agent.OS)" | "$(PY)"
+      path:  ${{ parameters.cache_dir }}
+  - bash: |
+      python -m pip install -U pip wheel setuptools
+      python -m pip install -r requirements/requirements_pyinstaller.txt
+    displayName: install libs
+  - bash: python -m pip install .
+    displayName: install partseg
+  - bash: |
+      python  build_utils/create_and_pack_executable.py
+    displayName: build
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathToPublish: dist2
+      artifactName: execs
+  - script: ${{ parameters.test_path }}
+    displayName: TestBuild

.azure-pipelines/tests.yaml

@@ -1,12 +1,22 @@
+parameters:
+- name: cache_dir # name of the parameter; required
+  type: string
+
 steps:
-  - {task: UsePythonVersion@0, inputs: {versionSpec: '3.8'}}
+  - {task: UsePythonVersion@0, inputs: {versionSpec: '3.9'}}
+  - template: pip_cache.yaml
+    parameters:
+      key: test | requirements/requirements_pyinstaller.txt | "$(Agent.OS)" | "$(PY)"
+      path: ${{ parameters.cache_dir }}
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: TestData
       path: $(Build.SourcesDirectory)/test_data
-  - script: pip install -r requirements/requirements_dev.txt
+  - script: |
+      pip install -U pip wheel setuptools virtualenv
+      pip install -r requirements/requirements_dev.txt
     displayName: "Install deps"
-  - script: tox -e py38-PyQt5-azure
+  - script: tox -e py39-PyQt5-azure
     displayName: "Run Tox"
     env:
       CODECOV_TOKEN: $(codecov_token_secret)

.github/workflows/base_test_workflow.yml

@@ -74,7 +74,6 @@ jobs:
           python -m pip install -r requirements/requirements_dev.txt tox-gh-actions
 
       - name: Test with tox
-        # run tests using pip install --pre
         uses: GabrielBB/xvfb-action@v1
         timeout-minutes: 60
         with:

azure-pipelines.yml

@@ -23,7 +23,7 @@ stages:
           displayName: "download data"
         - task: UsePythonVersion@0
           inputs:
-            versionSpec: '3.8'
+            versionSpec: '3.9'
           displayName: 'Use Python $(python.version)'
         - script: python build_utils/cut_changelog.py changelog_cut.md
           displayName: "Cut changelog"
@@ -32,13 +32,21 @@ stages:
         - publish: changelog_cut.md
           artifact: ReleaseInfo
 
-  - stage: formating_check
+  - stage: formatting_check
     dependsOn: []
     jobs:
       - job: check_formating
-        pool: {vmImage: 'Ubuntu-20.04'}
+        pool:
+          vmImage: ubuntu-latest
+        variables:
+          PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache
         steps:
           - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}}
+          - script: echo "##vso[task.setvariable variable=PY]$(python -VV)"
+          - task: Cache@2
+            inputs:
+              key: pre-commit | .pre-commit-config.yaml | "$(PY)"
+              path: $(PRE_COMMIT_HOME)
           - bash: |
               python -m pip install pre-commit
             displayName: InstallDeps
@@ -49,11 +57,11 @@ stages:
     dependsOn: []
     jobs:
       - job: manifest_check
-        pool: {vmImage: 'Ubuntu-18.04'}
+        pool: {vmImage: 'Ubuntu-latest'}
         steps:
-          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.8', architecture: x64}}
+          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.10', architecture: x64}}
           - bash: |
-              python -m pip install check-manifest==0.42 numpy==1.19.1
+              python -m pip install check-manifest==0.48 numpy==1.22.3
             displayName: InstallDeps
           - bash: check-manifest
             displayName: check-manifest
@@ -62,9 +70,15 @@ stages:
     dependsOn: []
     jobs:
       - job: help
-        pool: {vmImage: 'Ubuntu-18.04'}
+        pool: {vmImage: 'ubuntu-latest'}
+        variables:
+          pip_cache_dir: $(Pipeline.Workspace)/.pip
         steps:
-          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.8', architecture: x64}}
+          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.9', architecture: x64}}
+          - template: .azure-pipelines/pip_cache.yaml
+            parameters:
+              key: doc | requirements/requirements_pyinstaller.txt | "$(PY)"
+              path: $(pip_cache_dir)
           - bash: |
               python -m pip install tox
             displayName: Install tox
@@ -81,18 +95,42 @@ stages:
         continueOnError: true
         variables:
           DATA_PATH: typy_neuronow2
+          pip_cache_dir: $(Pipeline.Workspace)/.pip
         steps:
-          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.8', architecture: x64}}
+          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.9', architecture: x64}}
+          - template: .azure-pipelines/pip_cache.yaml
+            parameters:
+              key: notebook | requirements/requirements_pyinstaller.txt | "$(PY)"
+              path: $(pip_cache_dir)
           - bash: |
               python -m pip install tox
             displayName: Test notebook
           - bash: tox -e jupyter
             displayName: "Run Notebook"
 
-  - stage: Tests
-    dependsOn: [GetTestData, formating_check]
+  - stage: Tests_linux
+    dependsOn: [GetTestData, formatting_check]
     variables:
       AZURE_PIPELINES: 1
+      pip_cache_dir: $(Pipeline.Workspace)/.pip
+    jobs:
+    - job: test_linux
+      pool: { vmImage: 'ubuntu-20.04' }
+      variables:
+        AZURE_PIPELINES: 1
+        DISPLAY: ':99.0'
+        pip_cache_dir: $(Pipeline.Workspace)/.pip
+      steps:
+        - template: .azure-pipelines/linux_libs.yaml
+        - template: .azure-pipelines/tests.yaml
+          parameters:
+            cache_dir: $(pip_cache_dir)
+
+  - stage: Tests_other
+    dependsOn: [GetTestData, formatting_check]
+    variables:
+      AZURE_PIPELINES: 1
+      pip_cache_dir: $(Pipeline.Workspace)/.pip
     jobs:
     - job: test
       condition: not(startsWith(variables['Build.SourceBranch'], 'refs/heads/feature_prepare'))
@@ -105,20 +143,11 @@ stages:
       pool: {vmImage: $(imageName)}
       steps:
         - template: .azure-pipelines/tests.yaml
-    - job: test_linux
-      pool: { vmImage: 'ubuntu-20.04' }
-      variables:
-        AZURE_PIPELINES: 1
-        DISPLAY: ':99.0'
-      steps:
-        - script: |
-            sudo apt-get install -y libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
-            /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX
-          displayName: "install libs"
-        - template: .azure-pipelines/tests.yaml
+          parameters:
+            cache_dir: $(pip_cache_dir)
 
   - stage: Builds
-    dependsOn: Tests
+    dependsOn: Tests_linux
     jobs:
       - job: sdist
         pool: {vmImage: 'Ubuntu-18.04'}
@@ -137,28 +166,14 @@ stages:
         variables:
           test_path: dist/PartSeg/PartSeg _test
           DISPLAY: ':99.0'
+          pip_cache_dir: $(Pipeline.Workspace)/.pip
         pool: { vmImage: 'ubuntu-18.04' }
         steps:
-          - { task: UsePythonVersion@0, inputs: { versionSpec: '3.8', architecture: x64 } }
-          - script: |
-              sudo apt-get install -y libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
-              /sbin/start-stop-daemon --start --quiet \
-                          --pidfile /tmp/custom_xvfb_99.pid --make-pidfile \
-                          --background --exec /usr/bin/Xvfb \
-                          -- :99 -screen 0 1920x1200x24 -ac +extension GLX
-            displayName: "install libs"
-
-          - bash: |
-              python -m pip install -r requirements/requirements_pyinstaller.txt
-              python -m pip install -e .
-              python  build_utils/create_and_pack_executable.py
-
-          - task: PublishBuildArtifacts@1
-            inputs:
-              pathToPublish: dist2
-              artifactName: execs
-          - script: $(test_path)
-            displayName: TestBuild
+          - template: .azure-pipelines/linux_libs.yaml
+          - template: .azure-pipelines/pyinstaller.yaml
+            parameters:
+              test_path: $(test_path)
+              cache_dir: $(pip_cache_dir)
 
       - job: pyinstaller
         condition: not(startsWith(variables['Build.SourceBranch'], 'refs/heads/feature_prepare'))
@@ -170,21 +185,11 @@ stages:
             windows:
               imageName: 'windows-2019'
               test_path: dist\PartSeg\PartSeg _test
+        variables:
+          pip_cache_dir: $(Pipeline.Workspace)/.pip
         pool: {vmImage: $(imageName)}
         steps:
-          - {task: UsePythonVersion@0, inputs: {versionSpec: '3.8', architecture: x64}}
-          - bash: |
-              python -m pip install -U pip
-              python -m pip install -r requirements/requirements_pyinstaller.txt
-              python -m pip install .
-            displayName: install libs
-          - bash: |
-              python  build_utils/create_and_pack_executable.py
-            displayName: build
-
-          - task: PublishBuildArtifacts@1
-            inputs:
-              pathToPublish: dist2
-              artifactName: execs
-          - script: $(test_path)
-            displayName: TestBuild
+          - template: .azure-pipelines/pyinstaller.yaml
+            parameters:
+              test_path: $(test_path)
+              cache_dir: $(pip_cache_dir)

requirements/requirements_pyinstaller.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements_pyinstaller.in
@@ -30,13 +30,13 @@ certifi==2021.10.8
     #   sentry-sdk
 charset-normalizer==2.0.12
     # via requests
-click==8.1.2
+click==8.1.3
     # via typer
 cloudpickle==2.0.0
     # via dask
 czifile==2019.7.2
     # via -r requirements_pyinstaller.in
-dask[array]==2022.4.1
+dask[array]==2022.4.2
     # via
     #   -r requirements_pyinstaller.in
     #   napari
@@ -46,7 +46,7 @@ decorator==5.1.1
     # via ipython
 defusedxml==0.7.1
     # via -r requirements_pyinstaller.in
-docstring-parser==0.14
+docstring-parser==0.14.1
     # via magicgui
 docutils==0.17.1
     # via sphinx
@@ -70,29 +70,27 @@ idna==3.3
     # via requests
 imagecodecs==2022.2.22
     # via -r requirements_pyinstaller.in
-imageio==2.18.0
+imageio==2.19.0
     # via
     #   napari
     #   napari-svg
 imagesize==1.3.0
     # via sphinx
 importlib-metadata==4.11.3
     # via sphinx
-importlib-resources==5.7.1
-    # via jsonschema
 ipykernel==6.13.0
     # via
     #   napari-console
     #   qtconsole
-ipython==8.2.0
+ipython==8.3.0
     # via
     #   ipykernel
     #   napari-console
 ipython-genutils==0.2.0
     # via qtconsole
 jedi==0.18.1
     # via ipython
-jinja2==3.0.3
+jinja2==3.1.2
     # via
     #   numpydoc
     #   sphinx
@@ -162,7 +160,7 @@ numpy==1.22.3
     #   scipy
     #   tifffile
     #   vispy
-numpydoc==1.2.1
+numpydoc==1.3.1
     # via napari
 oiffile==2022.2.2
     # via -r requirements_pyinstaller.in
@@ -315,7 +313,7 @@ superqt==0.3.1
     #   napari
 sympy==1.10.1
     # via -r requirements_pyinstaller.in
-tifffile==2022.4.26
+tifffile==2022.4.28
     # via
     #   -r requirements_pyinstaller.in
     #   czifile
@@ -362,16 +360,14 @@ vispy==0.9.6
     #   napari-svg
 wcwidth==0.2.5
     # via prompt-toolkit
-wrapt==1.14.0
+wrapt==1.14.1
     # via napari
 xlrd==2.0.1
     # via -r requirements_pyinstaller.in
 xlsxwriter==3.0.3
     # via -r requirements_pyinstaller.in
 zipp==3.8.0
-    # via
-    #   importlib-metadata
-    #   importlib-resources
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

setup.cfg

@@ -128,6 +128,8 @@ filterwarnings =
     ignore:divide by zero encountered in true_divide
     ignore:the imp module is deprecated in favour of importlib
     ignore:distutils Version classes are deprecated.
+    ignore:Please import PackageMetadata from 'npe2' or from 'npe2.manifest'
+    ignore:Please use `zoom` from the `scipy.ndimage` namespace
 markers =
     enablethread: Allow to use thread in test
     enabledialog: Allow to use dialog in test