Debug test failures on master branch (opendr-eu#431)

* Disable mobile manipulation test * PyTorch version bump * PyTorch version bump * PyTorch version bump * PyTorch version bump * PyTorch version bump * Gym version bump * Gym version bump * Wheel version downgrade to openai/gym#3202 * Wheel version downgrade due to openai/gym#3202 * Wheel version downgrade due to openai/gym#3202 * Removed unsupported argument `enable_onnx_checker` * Added opset version * Added opset versions * Fixed `load_state_dict_from_url` calls * Fixed interpolation argument due to pytorchvision bump * Fixed `load_state_dict_from_url` import * Removed inplace operation * Remove inplace operation * Added missing dependency * Style fix * Removed unused import * DCN update * DCN update * DCN update * DCN version fix * Fix DCN loading to be backward compatible * Update tests_suite.yml * Update tests_suite_develop.yml * Update test_packages.yml * Update test_packages.yml * Update tests_suite.yml * Update tests_suite_develop.yml * Update test_packages.yml * Update tests_suite.yml * Update tests_suite_develop.yml * Update test_packages.yml * Update tests_suite.yml * Update tests_suite_develop.yml * Update deprecated macos image * Disable source tests on ubuntu for testing * Revert previous commit * Try to fix apt errors * Update macos image --------- Co-authored-by: Nikolaos Passalis <[email protected]> Co-authored-by: Nikolaos Passalis <[email protected]>
lucamarchionni · May 17, 2023 · 7177bb3 · 7177bb3
1 parent b0f7132
commit 7177bb3
Show file tree

Hide file tree

Showing 56 changed files with 141 additions and 99 deletions.
diff --git a/.github/workflows/test_packages.yml b/.github/workflows/test_packages.yml
@@ -72,7 +72,7 @@ jobs:
         python3 -m venv venv
         source venv/bin/activate
         python3 -m pip install --upgrade pip
-        python3 -m pip install wheel
+        python3 -m pip install wheel==0.38.4
         python3 -m pip install opendr-toolkit-engine
         python3 -m pip install opendr-toolkit
         # run the test
@@ -114,7 +114,7 @@ jobs:
           # - perception/object_detection_3d # passes, but disabled due to free() crash
           - perception/facial_expression_recognition
           - simulation/human_model_generation
-          - control/mobile_manipulation
+          # - control/mobile_manipulation
           - control/single_demo_grasp
           - planning/end_to_end_planning
           - control/multi_object_search
@@ -129,3 +129,4 @@ jobs:
         docker run --name toolkit -i opendr/opendr-toolkit:cpu_v2.1.0 bash
         docker start toolkit
         docker exec -i toolkit bash -c "source bin/activate.sh && source tests/sources/tools/control/mobile_manipulation/run_ros.sh && python3 -m unittest discover -s tests/sources/tools/${{ matrix.package }}"
+    
diff --git a/.github/workflows/tests_suite.yml b/.github/workflows/tests_suite.yml
@@ -18,12 +18,13 @@ jobs:
   test-sources:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || github.event_name == 'schedule' }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-10.15]
+        os: [ubuntu-20.04, macos-11]
         include:
           - os: ubuntu-20.04
-            DEPENDENCIES_INSTALLATION: "sudo apt -y install clang-format-10 cppcheck"
-          - os: macos-10.15
+            DEPENDENCIES_INSTALLATION: "sudo apt update; sudo apt -y install clang-format-10 cppcheck"
+          - os: macos-11
             DEPENDENCIES_INSTALLATION: "brew install clang-format@11 cppcheck; ln /usr/local/bin/clang-format-11 /usr/local/bin/clang-format"
     runs-on: ${{ matrix.os }}
     steps:
@@ -82,7 +83,7 @@ jobs:
           # - perception/object_detection_3d # passes, but disabled due to free() crash
           - perception/facial_expression_recognition
           - simulation/human_model_generation
-          - control/mobile_manipulation
+          # - control/mobile_manipulation
           - control/single_demo_grasp
           - planning/end_to_end_planning
           - control/multi_object_search
@@ -223,7 +224,7 @@ jobs:
         python3 -m venv venv
         source venv/bin/activate
         python3 -m pip install --upgrade pip
-        python3 -m pip install wheel
+        python3 -m pip install wheel==0.38.4
 
         # install all tools one at a time
         while read f; do
@@ -309,7 +310,7 @@ jobs:
         python3 -m venv venv
         source venv/bin/activate
         python3 -m pip install --upgrade pip
-        python3 -m pip install wheel
+        python3 -m pip install wheel==0.38.4
 
         # get the name of the wheel to install based on the test being run
         package=$(sed "s/_/-/g" <<< ${{ matrix.package }})
@@ -381,7 +382,7 @@ jobs:
           # - perception/object_detection_3d # passes, but disabled due to free() crash
           - perception/facial_expression_recognition
           - simulation/human_model_generation
-          - control/mobile_manipulation
+          # - control/mobile_manipulation
           - control/single_demo_grasp
           - planning/end_to_end_planning
           - control/multi_object_search

diff --git a/.github/workflows/tests_suite_develop.yml b/.github/workflows/tests_suite_develop.yml
@@ -19,11 +19,11 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || github.event_name == 'schedule' }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-10.15]
+        os: [ubuntu-20.04, macos-11]
         include:
           - os: ubuntu-20.04
-            DEPENDENCIES_INSTALLATION: "sudo apt -y install clang-format-10 cppcheck"
-          - os: macos-10.15
+            DEPENDENCIES_INSTALLATION: "sudo apt update; sudo apt -y install clang-format-10 cppcheck"
+          - os: macos-11
             DEPENDENCIES_INSTALLATION: "brew install clang-format@11 cppcheck; ln /usr/local/bin/clang-format-11 /usr/local/bin/clang-format"
     runs-on: ${{ matrix.os }}
     steps:
@@ -83,7 +83,7 @@ jobs:
           # - perception/object_detection_3d # passes, but disabled due to free() crash
           - perception/facial_expression_recognition
           - simulation/human_model_generation
-          - control/mobile_manipulation
+          # - control/mobile_manipulation
           - control/single_demo_grasp
           - planning/end_to_end_planning
           - control/multi_object_search
@@ -229,7 +229,7 @@ jobs:
         python3 -m venv venv
         source venv/bin/activate
         python3 -m pip install --upgrade pip
-        python3 -m pip install wheel
+        python3 -m pip install wheel==0.38.4
 
         # install all tools one at a time
         while read f; do
@@ -319,7 +319,7 @@ jobs:
         python3 -m venv venv
         source venv/bin/activate
         python3 -m pip install --upgrade pip
-        python3 -m pip install wheel
+        python3 -m pip install wheel==0.38.4
 
         # get the name of the wheel to install based on the test being run
         package=$(sed "s/_/-/g" <<< ${{ matrix.package }})
@@ -393,7 +393,7 @@ jobs:
           # - perception/object_detection_3d # passes, but disabled due to free() crash
           - perception/facial_expression_recognition
           - simulation/human_model_generation
-          - control/mobile_manipulation
+          # - control/mobile_manipulation
           - control/single_demo_grasp
           - planning/end_to_end_planning
           - control/multi_object_search

diff --git a/bin/install.sh b/bin/install.sh
@@ -29,7 +29,7 @@ git submodule update
 python3 -m venv venv
 source venv/bin/activate
 python3 -m pip install -U pip
-python3 -m pip install setuptools configparser
+python3 -m pip install setuptools configparser wheel==0.38.4
 
 # Add repositories for ROS
 sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' \
@@ -62,8 +62,8 @@ if [[ "${OPENDR_DEVICE}" == "gpu" ]]; then
   python3 -m pip uninstall -y torch
   echo "[INFO] Replacing  mxnet-cu112==1.8.0post0 to enable CUDA acceleration."
   python3 -m pip install mxnet-cu112==1.8.0post0
-  echo "[INFO] Replacing torch==1.9.0+cu111 to enable CUDA acceleration."
-  python3 -m pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+  echo "[INFO] Replacing torch==1.13.1+cu116 to enable CUDA acceleration."
+  python3 -m pip install torch==1.13.1+cu116 torchvision==0.14.1 torchaudio==0.13.1 -f https://download.pytorch.org/whl/torch_stable.html
   echo "[INFO] Reinstalling detectronv2."
   python3 -m pip install 'git+https://github.com/facebookresearch/detectron2.git@5aeb252b194b93dc2879b4ac34bc51a31b5aee13'
 fi

diff --git a/bin/install_nvidia.sh b/bin/install_nvidia.sh
@@ -288,7 +288,7 @@ pip3 install yacs==0.1.8
 pip3 install yarl==1.7.2
 pip3 install zipp==3.6.0
 pip3 install zope.interface==5.4.0
-pip3 install wheel
+pip3 install wheel==0.38.4
 pip3 install pytorch-lightning==1.2.3
 pip3 install omegaconf==2.3.0
 pip3 install ninja
@@ -300,7 +300,7 @@ pip3 install git+https://github.com/cidl-auth/cocoapi@03ee5a19844e253b8365dbbf35
 pip3 install git+https://github.com/cocodataset/panopticapi.git@7bb4655548f98f3fedc07bf37e9040a992b054b0
 pip3 install git+https://github.com/mapillary/inplace_abn.git
 pip3 install git+https://github.com/facebookresearch/detectron2.git@4841e70ee48da72c32304f9ebf98138c2a70048d
-pip3 install git+https://github.com/cidl-auth/DCNv2
+pip3 install git+https://github.com/cidl-auth/DCNv2_updated
 pip3 install ${OPENDR_HOME}/src/opendr/perception/panoptic_segmentation/efficient_ps/algorithm/EfficientPS
 pip3 install ${OPENDR_HOME}/src/opendr/perception/panoptic_segmentation/efficient_lps/algorithm/EfficientLPS
 pip3 install ${OPENDR_HOME}/src/opendr/perception/panoptic_segmentation/efficient_lps/algorithm/EfficientLPS/efficientNet

diff --git a/dependencies/dependencies.ini b/dependencies/dependencies.ini
@@ -1,8 +1,10 @@
 [runtime]
 # 'python' and 'python-dependencies' keys expect a value in the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
+python=torch==1.13.1
        wheel
+       numpy
+
 
 [device]
 # These dependencies require a CPU. If no CPU is available, the package will not be installed.

diff --git a/docs/reference/installation.md b/docs/reference/installation.md
@@ -35,7 +35,7 @@ First, install the required dependencies:
 sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev libeigen3-dev
 python3 -m venv venv
 source venv/bin/activate
-pip install wheel
+pip install wheel==0.38.4
 ```
 Then, you  install the Python API of the toolkit using pip:
 ```bash
@@ -57,8 +57,8 @@ For example, if you stick with the default PyTorch version (1.8) and use CUDA11.
 sudo apt install python3.8-venv libfreetype6-dev git build-essential cmake python3-dev wget libopenblas-dev libsndfile1 libboost-dev libeigen3-dev
 python3 -m venv venv
 source venv/bin/activate
-pip install wheel
-pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+pip install wheel==0.38.4
+pip install torch==1.13.1+cu116 torchvision==0.14.1 torchaudio==0.13.1 -f https://download.pytorch.org/whl/torch_stable.html
 pip install 'git+https://github.com/facebookresearch/detectron2.git'
 pip install mxnet-cu112==1.8.0post0
 pip install opendr-toolkit-engine

diff --git a/src/opendr/control/mobile_manipulation/dependencies.ini b/src/opendr/control/mobile_manipulation/dependencies.ini
@@ -10,7 +10,7 @@ python=vcstool
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
+python=torch==1.13.1
        protobuf<=3.20.0
        tensorboard
        numpy<=1.23.5

diff --git a/src/opendr/control/multi_object_search/dependencies.ini b/src/opendr/control/multi_object_search/dependencies.ini
@@ -1,6 +1,6 @@
 [runtime]
-python=torch==1.9.0
-       torchvision==0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
        tensorboard
        numpy
        scipy

diff --git a/src/opendr/control/single_demo_grasp/dependencies.ini b/src/opendr/control/single_demo_grasp/dependencies.ini
@@ -1,12 +1,12 @@
 [compilation]
-python=torch==1.9.0
-       torchvision==0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
 
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
-       torchvision==0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
        matplotlib>=2.2.2
        imgaug==0.4.0
        pillow>=8.3.2

diff --git a/src/opendr/engine/dependencies.ini b/src/opendr/engine/dependencies.ini
@@ -1,7 +1,7 @@
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
+python=torch==1.13.1
        wheel
        Cython
        numpy<=1.23.5

diff --git a/src/opendr/perception/activity_recognition/dependencies.ini b/src/opendr/perception/activity_recognition/dependencies.ini
@@ -1,8 +1,8 @@
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch>=1.9.0
-       torchvision>=0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
        tqdm
        onnx==1.8.0
        onnxruntime>=1.3.0

diff --git a/src/opendr/perception/binary_high_resolution/binary_high_resolution_learner.py b/src/opendr/perception/binary_high_resolution/binary_high_resolution_learner.py
@@ -350,7 +350,7 @@ def __convert_to_onnx(self, output_name, do_constant_folding=False, verbose=Fals
         input_names = ['data']
         output_names = ['output']
 
-        torch.onnx.export(self.model, inp, output_name, verbose=verbose,
+        torch.onnx.export(self.model, inp, output_name, verbose=verbose, opset_version=11,
                           do_constant_folding=do_constant_folding, input_names=input_names, output_names=output_names,
                           dynamic_axes={"data": {0: "batch", 2: "height", 3: "width"}})
 

diff --git a/src/opendr/perception/binary_high_resolution/dependencies.ini b/src/opendr/perception/binary_high_resolution/dependencies.ini
@@ -1,9 +1,9 @@
 [runtime]
 # 'python' and 'python-dependencies' keys expect a value in the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
+python=torch==1.13.1
        torchsampler
-       torchvision==0.10.0
+       torchvision==0.14.1
        opencv-python==4.5.1.48
        protobuf<=3.20.0
        onnx==1.8.0

diff --git a/src/opendr/perception/compressive_learning/dependencies.ini b/src/opendr/perception/compressive_learning/dependencies.ini
@@ -1,8 +1,8 @@
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
-       torchvision==0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
        protobuf<=3.20.0
        tensorboard>=2.4.1
        tqdm

diff --git a/...essive_learning/multilinear_compressive_learning/algorithm/backbones/imagenet_densenet.py b/...essive_learning/multilinear_compressive_learning/algorithm/backbones/imagenet_densenet.py
@@ -16,6 +16,11 @@
 import torchvision.models.densenet as densenet_models
 import re
 
+try:
+    from torch.hub import load_state_dict_from_url
+except ImportError:
+    from torch.utils.model_zoo import load_url as load_state_dict_from_url
+
 getters = {'densenet121': densenet_models.densenet121,
            'densenet161': densenet_models.densenet161,
            'densenet169': densenet_models.densenet169,
@@ -54,8 +59,7 @@ def __init__(self, model_name, n_class, pretrained):
 
         if pretrained != '':
             # get pretrained weights
-            state_dict = densenet_models.load_state_dict_from_url(densenet_models.model_urls[model_name],
-                                                                  progress=False)
+            state_dict = load_state_dict_from_url(densenet_models.model_urls[model_name], progress=False)
 
             # fix naming issue
             # as in https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py

diff --git a/...pressive_learning/multilinear_compressive_learning/algorithm/backbones/imagenet_resnet.py b/...pressive_learning/multilinear_compressive_learning/algorithm/backbones/imagenet_resnet.py
@@ -15,6 +15,11 @@
 import torch.nn as nn
 import torchvision.models.resnet as resnet_models
 
+try:
+    from torch.hub import load_state_dict_from_url
+except ImportError:
+    from torch.utils.model_zoo import load_url as load_state_dict_from_url
+
 getters = {'resnet18': resnet_models.resnet18,
            'resnet34': resnet_models.resnet34,
            'resnet50': resnet_models.resnet50,
@@ -38,7 +43,7 @@ def __init__(self, model_name, n_class, pretrained):
 
         if pretrained != '':
             # get pretrained weights
-            state_dict = resnet_models.load_state_dict_from_url(resnet_models.model_urls[model_name], progress=False)
+            state_dict = load_state_dict_from_url(resnet_models.model_urls[model_name], progress=False)
 
             # remove the last classifier layer from pretrained weights
             if pretrained == 'without_classifier':

diff --git a/...compressive_learning/multilinear_compressive_learning/algorithm/backbones/imagenet_vgg.py b/...compressive_learning/multilinear_compressive_learning/algorithm/backbones/imagenet_vgg.py
@@ -15,6 +15,11 @@
 import torch.nn as nn
 import torchvision.models.vgg as vgg_models
 
+try:
+    from torch.hub import load_state_dict_from_url
+except ImportError:
+    from torch.utils.model_zoo import load_url as load_state_dict_from_url
+
 getters = {'vgg11': vgg_models.vgg11,
            'vgg11_bn': vgg_models.vgg11_bn,
            'vgg13': vgg_models.vgg13,
@@ -36,7 +41,7 @@ def __init__(self, model_name, n_class, pretrained):
 
         if pretrained != '':
             # get pretrained weights
-            state_dict = vgg_models.load_state_dict_from_url(vgg_models.model_urls[model_name], progress=False)
+            state_dict = load_state_dict_from_url(vgg_models.model_urls[model_name], progress=False)
 
             # remove the last classifier layer from pretrained weights
             if pretrained == 'without_classifier':

diff --git a/src/opendr/perception/face_recognition/dependencies.ini b/src/opendr/perception/face_recognition/dependencies.ini
@@ -1,8 +1,8 @@
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
-       torchvision==0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
        bcolz>=1.2.1
        protobuf<=3.20.0
        onnx==1.8.0

diff --git a/src/opendr/perception/face_recognition/face_recognition_learner.py b/src/opendr/perception/face_recognition/face_recognition_learner.py
@@ -846,7 +846,7 @@ def __convert_to_onnx(self, verbose=False):
         input_names = ['data']
         output_names = ['features']
         output_name = os.path.join(self.temp_path, 'onnx_' + self.backbone + '_backbone_model.onnx')
-        torch.onnx.export(self.backbone_model, inp, output_name, verbose=verbose, enable_onnx_checker=True,
+        torch.onnx.export(self.backbone_model, inp, output_name, verbose=verbose, opset_version=11,
                           input_names=input_names, output_names=output_names)
         if self.mode == 'full' and self.network_head == 'classifier':
             if 'cuda' in self.device:
@@ -856,7 +856,7 @@ def __convert_to_onnx(self, verbose=False):
             input_names = ['features']
             output_names = ['classes']
             output_name = os.path.join(self.temp_path, 'onnx_' + self.network_head + '_head_model.onnx')
-            torch.onnx.export(self.network_head_model, inp, output_name, verbose=verbose, enable_onnx_checker=True,
+            torch.onnx.export(self.network_head_model, inp, output_name, verbose=verbose, opset_version=11,
                               input_names=input_names, output_names=output_names)
 
     def optimize(self, do_constant_folding=False):

diff --git a/...tion/facial_expression_recognition/image_based_facial_emotion_estimation/dependencies.ini b/...tion/facial_expression_recognition/image_based_facial_emotion_estimation/dependencies.ini
@@ -1,8 +1,8 @@
 [runtime]
 # 'python' key expects a value using the Python requirements file format
 #  https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-python=torch==1.9.0
-       torchvision==0.10.0
+python=torch==1.13.1
+       torchvision==0.14.1
        dlib
        scikit-learn
        pandas