Test-tools improvement (opendr-eu#456)

* Applied latest change from tests_suite.yml

* Add echo message for identification

* Moved identification at the beginning

* Removed identification

* Attempting to split the installation

* Split installation in another job

* Added activation of environment in test tools

* Trying out some caching

* Trying out separate installation job with cache

* Fix cache version in test tools and keep only engine test for debugging

* Trying out caching venv

* Trying out some additional tests

* Re-enabled all tests for full run

* Attempting to install libopenblas etc

* Added more system dependencies and disabled most tests to lessen load

* Attempting to install Cython again

* Attempting different Cython installation

* Attempting retinaface make

* Re-enable all other tests for ful lrun

* Attempting install dependency

* Added missing repositories for ROS

* Attempting to remedy the timeouts on cache restore

* Attempting to update apt after adding keys

* Attempting to run make libopendr as it seems to be used in regular install.sh after ROS installation (presumably builds ROS workspaces?)

* Temporary fix to nms

* Temporary fix for nms

* Temp fix seq2seq nms

* Remove non-existing options

* Update test_seq2seq_nms.py

* Attempting to install mobile manipulation before running

* Attempting make mobile manip

* Removed make libopendr

* Attempting to run mobile manip stuff without if

* Added some echos

* Update tests_suite.yml

* Update tests_suite.yml

* Readded if for mobile manipulation

* Removed if altogether

* Readded accidentally removed cd

* Changed up the if to include multi object search for installing ROS deps

* Attempting make libopendr again

* Added opendr device

* Modified ifs for end to end planning and mobile manip

* Determining whether egl and dependencies are installed

* Attempting installation of libegl1 and some attempted fixes on mobile manip if

* Added if f or libegl installation and attempting to source setup.bash for end-to-end-planning

* Removed ifs between mobile manip and and end-to-end, also enabled all tests for full run

* Added ffmpeg system install and hashing dependencies.inis for cache key

* Removed cache restore key and added test with and without cache

* Testing if cache-hit

* Added missing cache step id

* Added some echos, added regular installation on cache miss, added timeout workaround for downloading

* Fix

* Add dummy dependency to test cache behaviour

* Dummy code change

* Removed dummy code change

* Removed dummy dependency

* Added additional hash name for cache

* Fix actions versions

* Fixed cache restoring

* Increased timeout mins 3->5

* Removed ffmpeg system dependency

* Added apt update

.github/workflows/tests_suite.yml

@@ -14,6 +14,10 @@ defaults:
   run:
     shell: bash
 
+env:
+    # XXX: Workaround for https://github.com/actions/cache/issues/1141
+    SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
 jobs:
   test-sources:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test sources') || github.event_name == 'schedule' }}
@@ -45,7 +49,43 @@ jobs:
         export OPENDR_DEVICE=cpu
         pip install -r tests/requirements.txt
         python -m unittest discover -s tests
+
+  install-toolkit:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'test tools') || github.event_name == 'schedule' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+    - name: Cache Python dependencies and virtual environment
+      id: cache-python-deps
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pip
+          venv
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dependencies.ini') }}-${{ hashFiles('**/install.sh') }}
+        # restore-keys: |
+        #   ${{ runner.os }}-pip-
+    - name: Install toolkit and dependencies
+      if: steps.cache-python-deps.outputs.cache-hit != 'true'
+      run: |
+        export ROS_DISTRO=noetic
+        
+        ./bin/install.sh
+        source bin/activate.sh
+        python3 -m pip install -r tests/requirements.txt
+
   test-tools:
+    needs: install-toolkit
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test tools') || github.event_name == 'schedule' }}
     strategy:
       fail-fast: false
@@ -92,17 +132,30 @@ jobs:
           - control/multi_object_search
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
-    - name: Test Tools
+    - name: Restore Python dependencies and virtual environment cache
+      id: cache-python-deps
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pip
+          venv
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dependencies.ini') }}-${{ hashFiles('**/install.sh') }}
+
+    - name: Run toolkit tests without cache
+      if: steps.cache-python-deps.outputs.cache-hit != 'true'
       run: |
+        echo "Restore cache failed, running regular installation and testing..."
         export ROS_DISTRO=noetic
         export OPENDR_HOME=$PWD
+        
+        echo "Running full OpenDR installation"
         ./bin/install.sh
         source bin/activate.sh
         python3 -m pip install -r tests/requirements.txt
@@ -113,8 +166,84 @@ jobs:
           if [[ ${{ matrix.package }} == "control/mobile_manipulation" || ${{ matrix.package }} == "planning/end_to_end_planning" ]]; then
             source tests/sources/tools/control/mobile_manipulation/run_ros.sh
           fi
+
+          echo "Running unittest discover for ${{ matrix.package }}"
           python3 -m unittest discover -s tests/sources/tools/${{ matrix.package }}
         fi
+
+    - name: Run toolkit tests with cache
+      if: steps.cache-python-deps.outputs.cache-hit == 'true'
+      run: |
+        export ROS_DISTRO=noetic
+        export OPENDR_HOME=$PWD
+        export OPENDR_DEVICE=cpu
+
+        # Reinstall some system dependencies
+        echo "Installing system dependencies"
+        sudo apt-get update
+        sudo apt-get install --yes unzip libfreetype6-dev lsb-release git python3-pip curl wget python3.8-venv ffmpeg
+        sudo apt-get install --yes libopenblas-dev libsndfile1 libboost-dev
+        
+        source bin/activate.sh
+
+        echo "OpenDR env activated, running retinaface make"
+        # This should probably run conditionally for tools that need it
+        cd $OPENDR_HOME/src/opendr/perception/object_detection_2d/retinaface
+        make
+        cd $OPENDR_HOME
+        
+        if [ ${{ matrix.package }} = "ctests" ]; then
+          make ctests
+        else
+          if [[ ${{ matrix.package }} == "control/mobile_manipulation" || ${{ matrix.package }} == "planning/end_to_end_planning"  || ${{ matrix.package }} == "control/multi_object_search" ]]; then
+            echo "ROS $ROS_DISTRO Installation"
+
+            echo "Adding ROS repositories keys"
+            sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' \
+              && curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | sudo apt-key add -
+            
+            sudo apt update
+            
+            # ROS package dependencies
+            if [[ ${ROS_DISTRO} == "noetic" || ${ROS_DISTRO} == "melodic" ]]; then
+              echo "Installing ROS dependencies"
+              sudo apt-get -y install ros-$ROS_DISTRO-vision-msgs ros-$ROS_DISTRO-geometry-msgs ros-$ROS_DISTRO-sensor-msgs ros-$ROS_DISTRO-audio-common-msgs ros-$ROS_DISTRO-hri-msgs ros-$ROS_DISTRO-usb-cam ros-$ROS_DISTRO-webots-ros
+            fi
+            
+            # ROS2 package dependencies
+            if [[ ${ROS_DISTRO} == "foxy" || ${ROS_DISTRO} == "humble" ]]; then
+              echo "Installing ROS2 dependencies"
+              sudo apt-get -y install python3-lark ros-$ROS_DISTRO-usb-cam ros-$ROS_DISTRO-webots-ros2 python3-colcon-common-extensions ros-$ROS_DISTRO-vision-msgs ros-$ROS_DISTRO-sensor-msgs-py
+              LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/ros/$ROS_DISTRO/lib/controller
+              cd $OPENDR_HOME/projects/opendr_ws_2/
+              git clone --depth 1 --branch ros2 https://github.com/ros-drivers/audio_common src/audio_common
+              rosdep install -i --from-path src/audio_common --rosdistro $ROS_DISTRO -y
+              cd $OPENDR_HOME
+            fi
+
+            # Install libegl required by multi-object search
+            if [ ${{ matrix.package}} == "control/multi_object_search" ]; then
+              echo "Installing libegl1 for multi-object search"
+              sudo apt install libegl1-mesa-dev
+            fi
+
+            # Make and run mobile manipulation test
+            # End-to-end planning requires mobile manipulation to run
+            if [[ ${{ matrix.package }} == "control/mobile_manipulation" || ${{ matrix.package }} == "planning/end_to_end_planning" ]]; then
+              echo "Installing mobile manipulation"
+              cd src/opendr/control/mobile_manipulation/
+              make
+              cd ../../../..
+
+              echo "Running mobile manipulation test"
+              source tests/sources/tools/control/mobile_manipulation/run_ros.sh
+            fi
+          fi
+
+          echo "Running unittest discover for ${{ matrix.package }}"
+          python3 -m unittest discover -s tests/sources/tools/${{ matrix.package }}
+        fi
+
   build-wheel:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test release') || github.event_name == 'schedule' }}
     runs-on: ubuntu-20.04
@@ -123,7 +252,7 @@ jobs:
       with:
         submodules: true
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
     - name: Install prerequisites
@@ -139,6 +268,7 @@ jobs:
         name: wheel-artifact
         path:
           dist/*.tar.gz
+
   build-docker:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test release') || github.event_name == 'schedule' }}
     runs-on: ubuntu-20.04
@@ -161,6 +291,7 @@ jobs:
         name: docker-artifact
         path:
           cpu_test.zip
+
   test-wheel:
     needs: build-wheel
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test release') || github.event_name == 'schedule' }}
@@ -253,6 +384,7 @@ jobs:
 
         # run the test
         python3 -m unittest discover -s tests/sources/tools/${{ matrix.package }}
+
   test-wheel-separate:
     needs: build-wheel
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test release') || github.event_name == 'schedule' }}
@@ -364,6 +496,7 @@ jobs:
 
         # run the test
         python3 -m unittest discover -s tests/sources/tools/${{ matrix.package }}
+
   test-docker:
     needs: build-docker
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test release') || github.event_name == 'schedule' }}
@@ -428,6 +561,7 @@ jobs:
         else
           docker exec -i toolkit bash -c "source bin/activate.sh && python3 -m unittest discover -s tests/sources/tools/${{ matrix.package }}"
         fi
+
   delete-docker-artifacts:
     needs: [build-docker, test-docker]
     if: ${{ always() }}
@@ -440,6 +574,7 @@ jobs:
       uses: geekyeggo/delete-artifact@v1
       with:
         name: docker-artifact
+
   delete-wheel-artifacts:
     needs: [build-wheel, test-wheel, test-wheel-separate]
     if: ${{ always() }}

.github/workflows/tests_suite_develop.yml

@@ -107,7 +107,8 @@ jobs:
         export OPENDR_HOME=$PWD
         ./bin/install.sh
         source bin/activate.sh
-        python3 -m pip install -r tests/sources/requirements.txt
+        python3 -m pip install -r tests/requirements.txt
+        
         if [ ${{ matrix.package }} = "ctests" ]; then
           make ctests
         else