Add robot-log-visualizer package, its dependencies and rob_sup_pure_python_ycm_ep_helper CMake function to add pure Python packages

.ci/install_debian.sh

@@ -16,7 +16,7 @@ SCRIPT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; cd -P "$(dirname "$(readlink "$BASH
 source ${SCRIPT_DIR}/../scripts/install_apt_dependencies.sh
 
 # Python
-apt-get install -y python3-dev python3-numpy python3-pybind11 pybind11-dev
+source ${SCRIPT_DIR}/../scripts/install_apt_python_dependencies.sh
 
 # Octave
 apt-get install -y liboctave-dev

.github/workflows/ci.yml

@@ -79,7 +79,7 @@ jobs:
         # Actual dependencies
         mamba install ace asio assimp boost eigen gazebo glew glfw graphviz gsl ipopt irrlicht libjpeg-turbo libmatio libode libxml2 nlohmann_json opencv pkg-config portaudio qt=5.12.9=*_4 sdl sdl2 sqlite tinyxml spdlog lua soxr
         # Python 
-        mamba install numpy swig pybind11
+        mamba install python numpy swig pybind11 pyqt matplotlib h5py tornado u-msgpack-python pyzmq ipython
 
     # Additional dependencies useful only on Linux
     - name: Dependencies [Conda/Linux]

.github/workflows/generate-conda-packages.yaml

@@ -149,6 +149,8 @@ jobs:
             # see https://github.com/mamba-org/boa/issues/117
             # We also manually delete the recipe directory to avoid that it is built again, 
             # specifically because we can't use --skip-existing as these packages are part of the robotology channel
+            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robot-log-visualizer
+            rm -rf robot-log-visualizer
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml yarp
             rm -rf yarp
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml gazebo-yarp-plugins

CHANGELOG.md

@@ -5,6 +5,9 @@ The format of this document is based on [Keep a Changelog](https://keepachangelo
 
 ## [Unreleased]
 
+### Added
+- Added the [`robot-log-visualizer` package](https://github.com/ami-iit/robot-log-visualizer) and its dependencies to the robotology-superbuild, if the option `ROBOTOLOGY_USES_PYTHON` and `ROBOTOLOGY_ENABLE_DYNAMICS_FULL_DEPS` are enabled. As `robot-log-visualizer` is a pure Python package, the PR also adds a `rob_sup_pure_python_ycm_ep_helper` CMake helper function to simply the process or writing `Build<package>.cmake` scripts for pure Python packages (https://github.com/robotology/robotology-superbuild/pull/1069).
+
 ## [2022.02.1] - 2022-04-01
 
 ### Fixed

apt-python.txt

@@ -0,0 +1,14 @@
+python3-dev
+python3-numpy
+python3-pybind11
+pybind11-dev
+python3-pyqt5
+python3-pyqt5.qtmultimedia
+python3-matplotlib
+python3-pyqt5.qtwebengine
+python3-qtpy
+python3-h5py
+python3-u-msgpack
+python3-tornado
+python3-zmq
+python3-ipython

cmake/Buildmeshcat-python.cmake

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2022 Fondazione Istituto Italiano di Tecnologia
+# SPDX-License-Identifier: BSD-3-Clause
+
+include(RobSupPurePythonYCMEPHelper)
+
+rob_sup_pure_python_ycm_ep_helper(meshcat-python
+                                  REPOSITORY rdeits/meshcat-python.git
+                                  TAG master
+                                  COMPONENT dynamics
+                                  FOLDER src)
+
+
+set(meshcat-python_CONDA_PKG_NAME meshcat-python)
+set(meshcat-python_CONDA_PKG_CONDA_FORGE_OVERRIDE ON)

cmake/Buildpyqtconsole.cmake

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2022 Fondazione Istituto Italiano di Tecnologia
+# SPDX-License-Identifier: BSD-3-Clause
+
+include(RobSupPurePythonYCMEPHelper)
+
+rob_sup_pure_python_ycm_ep_helper(pyqtconsole
+                                  REPOSITORY pyqtconsole/pyqtconsole.git
+                                  TAG master
+                                  COMPONENT dynamics
+                                  FOLDER src)
+
+set(pyqtconsole_CONDA_PKG_NAME pyqtconsole)
+set(pyqtconsole_CONDA_PKG_CONDA_FORGE_OVERRIDE ON)

cmake/Buildrobot-log-visualizer.cmake

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2022 Fondazione Istituto Italiano di Tecnologia
+# SPDX-License-Identifier: BSD-3-Clause
+
+include(RobSupPurePythonYCMEPHelper)
+
+find_or_build_package(icub-models QUIET)
+find_or_build_package(pyqtconsole QUIET)
+find_or_build_package(meshcat-python QUIET)
+find_or_build_package(iDynTree QUIET)
+
+rob_sup_pure_python_ycm_ep_helper(robot-log-visualizer
+                                  REPOSITORY ami-iit/robot-log-visualizer.git
+                                  DEPENDS meshcat-python
+                                          pyqtconsole
+                                          icub-models
+                                          iDynTree
+                                  TAG main
+                                  COMPONENT dynamics
+                                  FOLDER src)
+
+set(robot-log-visualizer_CONDA_DEPENDENCIES numpy pyqt matplotlib h5py)

cmake/RobSupPurePythonYCMEPHelper.cmake

@@ -0,0 +1,54 @@
+#[=======================================================================[.rst:
+RobSupPurePythonYCMEPHelper
+-----------
+
+A wrapper for ycm_ep_helper for pure Python projects in the robotology-superbuold::
+
+ rob_sup_pure_python_ycm_ep_helper(<name>
+   [COMPONENT <component>] (default = "external")
+   [FOLDER <folder> (default = "<component>")
+   [REPOSITORY <repo>]
+  #--Git and Hg only arguments-----------
+   [TAG <tag>]
+   )
+
+#]=======================================================================]
+
+
+if(COMMAND rob_sup_pure_python_ycm_ep_helper)
+  return()
+endif()
+
+include(YCMEPHelper)
+
+function(ROB_SUP_PURE_PYTHON_YCM_EP_HELPER _name)
+  # Dependencies
+  find_package(Python3 COMPONENTS Interpreter REQUIRED)
+
+  execute_process(COMMAND ${Python3_EXECUTABLE}
+                  -c "from distutils import sysconfig; print(sysconfig.get_python_lib(1,0,prefix=''))"
+                  OUTPUT_VARIABLE _PYTHON_INSTDIR)
+  string(STRIP ${_PYTHON_INSTDIR} ROBSUB_PYTHON_INSTALL_DIR)
+
+  # Check arguments
+  set(_options)
+  set(_oneValueArgs COMPONENT
+                    FOLDER
+                    REPOSITORY
+                    TAG)
+  set(_multiValueArgs DEPENDS)
+
+  cmake_parse_arguments(_PYH_${_name} "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" "${ARGN}")
+
+  ycm_ep_helper(${_name} TYPE GIT
+                         STYLE GITHUB
+                         REPOSITORY ${_PYH_${_name}_REPOSITORY}
+                         DEPENDS ${_PYH_${_name}_DEPENDS}
+                         TAG ${_PYH_${_name}_TAG}
+                         COMPONENT ${_PYH_${_name}_COMPONENT}
+                         FOLDER ${_PYH_${_name}_FOLDER}
+                         # Workaround for https://github.com/robotology/robotology-superbuild/pull/1069#issuecomment-1099446237
+                         CONFIGURE_COMMAND ${CMAKE_COMMAND} --version
+                         BUILD_COMMAND ${CMAKE_COMMAND} --version
+                         INSTALL_COMMAND ${Python3_EXECUTABLE} -m pip install --upgrade --no-deps --target=${YCM_EP_INSTALL_DIR}/${ROBSUB_PYTHON_INSTALL_DIR} -VV <SOURCE_DIR>)
+endfunction()

cmake/RobotologySuperbuildLogic.cmake

@@ -65,6 +65,9 @@ if(ROBOTOLOGY_ENABLE_DYNAMICS)
     if(ROBOTOLOGY_USES_MATLAB)
       find_or_build_package(casadi-matlab-bindings)
     endif()
+    if(ROBOTOLOGY_USES_PYTHON)
+      find_or_build_package(robot-log-visualizer)
+    endif()
   endif()
 endif()
 

cmake/template/addPathsToUserEnvVariables.ps1.in

@@ -74,6 +74,8 @@ Add-ValueToUserEnvVariable YARP_DATA_DIRS $ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX"
 @cmakeif ROBOTOLOGY_USES_PYTHON
 # Add the python bindings directory to the PYTHON_PATH
 Add-ValueToUserEnvVariable PYTHONPATH $ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX"@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR@";
+# Add to the PATH where pure python packages (such as robot-log-visualizer) install the scripts
+Add-ValueToUserEnvVariable PATH $ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX"@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR@\bin";
 @endcmakeif ROBOTOLOGY_USES_PYTHON
 
 @cmakeif ROBOTOLOGY_USES_MATLAB

cmake/template/removePathsFromUserEnvVariables.ps1.in

@@ -77,6 +77,8 @@ Remove-ValueFromUserEnvVariable YARP_DATA_DIRS $ROBOTOLOGY_SUPERBUILD_INSTALL_PR
 @cmakeif ROBOTOLOGY_USES_PYTHON
 # Cleanup the python bindings directory from the PYTHON_PATH
 Remove-ValueFromUserEnvVariable PYTHONPATH $ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX"\@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR@";
+# Remove from the PATH where pure python packages (such as robot-log-visualizer) install the scripts
+Remove-ValueFromUserEnvVariable PATH $ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX"@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR@\bin";
 @endcmakeif ROBOTOLOGY_USES_PYTHON
 
 @cmakeif ROBOTOLOGY_USES_MATLAB

cmake/template/setup.bat.in

@@ -41,6 +41,8 @@ set "YARP_DATA_DIRS=%YARP_DATA_DIRS%;%ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX%\shar
 @cmakeif ROBOTOLOGY_USES_PYTHON
 rem Add the python bindings directory to the PYTHON_PATH
 set "PYTHONPATH=%PYTHONPATH%;%ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX%\@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR@"
+rem Add to the PATH where pure python packages (such as robot-log-visualizer) install the scripts
+set "PATH=%PATH%;%ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX%\@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR@\bin"
 @endcmakeif ROBOTOLOGY_USES_PYTHON
 
 @cmakeif ROBOTOLOGY_USES_MATLAB

cmake/template/setup.sh.in

@@ -44,6 +44,8 @@ export YARP_DATA_DIRS=$YARP_DATA_DIRS:${ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX}/sh
 @cmakeif ROBOTOLOGY_USES_PYTHON
 # Add the python bindings directory to the PYTHON_PATH
 export PYTHONPATH=${PYTHONPATH:+${PYTHONPATH}:}${ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX}/@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR_SETUP_SH@
+# Add to the PATH where pure python packages (such as robot-log-visualizer) install the scripts
+export PATH=$PATH:${ROBOTOLOGY_SUPERBUILD_INSTALL_PREFIX}/@ROBOTOLOGY_SUPERBUILD_PYTHON_INSTALL_DIR_SETUP_SH@/bin
 @endcmakeif ROBOTOLOGY_USES_PYTHON
 
 @cmakeif ROBOTOLOGY_USES_MATLAB

conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake

@@ -1,9 +1,12 @@
 # Copyright (C) Fondazione Istituto Italiano di Tecnologia
 # CopyPolicy: Released under the terms of the LGPLv2.1 or later, see LGPL.TXT
 
-# Redefine find_or_build_package and ycm_ep_helper
-# functions to extract metadata necessary for conda recipe
-# generation from the RobotologySuperbuildLogic file
+# Redefine find_or_build_package, ycm_ep_helper and rob_sup_pure_python_ycm_ep_helper
+# as macros to extract metadata necessary for conda recipe
+# generation from the RobotologySuperbuildLogic file.
+# Note that these were originally functions, but they are re-defined as macros so that
+# all the variables that they create are placed in the global scope, and in this script
+# we can access variables such as ${_YH_${_cmake_pkg}_CMAKE_ARGS}
 macro(ycm_ep_helper _name)
   # Check arguments
   set(_options )
@@ -44,6 +47,30 @@ macro(ycm_ep_helper _name)
   set_property(GLOBAL PROPERTY YCM_PROJECTS ${_projects})
 endmacro()
 
+macro(ROB_SUP_PURE_PYTHON_YCM_EP_HELPER _name)
+  # Check arguments
+  set(_options)
+  set(_oneValueArgs COMPONENT
+                    FOLDER
+                    REPOSITORY
+                    TAG)
+  set(_multiValueArgs DEPENDS)
+
+  cmake_parse_arguments(_PYH_${_name} "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" "${ARGN}")
+
+  ycm_ep_helper(${_name} TYPE GIT
+                         STYLE GITHUB
+                         REPOSITORY ${_PYH_${_name}_REPOSITORY}
+                         DEPENDS ${_PYH_${_name}_DEPENDS}
+                         TAG ${_PYH_${_name}_TAG}
+                         COMPONENT ${_PYH_${_name}_COMPONENT}
+                         FOLDER ${_PYH_${_name}_FOLDER})
+
+  # Set this variable so that RobotologySuperbuildGenerateCondaRecipes.cmake pass this information to the
+  # Python scripts that generates the conda recipes
+  set(${_name}_CONDA_BUILD_TYPE "pure_python")
+endmacro()
+
 macro(find_or_build_package _pkg)
   get_property(_superbuild_pkgs GLOBAL PROPERTY YCM_PROJECTS)
   if(NOT ${_pkg} IN_LIST _superbuild_pkgs)
@@ -74,6 +101,11 @@ macro(generate_metametadata_file)
       set(${_cmake_pkg}_CONDA_VERSION ${${_cmake_pkg}_CONDA_TAG})
     endif()
 
+    # If the build_type is not defined, it is assumed to be cmake
+    if(NOT DEFINED ${_cmake_pkg}_CONDA_BUILD_TYPE)
+      set(${_cmake_pkg}_CONDA_BUILD_TYPE "cmake")
+    endif()
+
 
     # If a package is already available in conda-forge, we use
     # that one by defining appropriately the <_cmake_pkg>_CONDA_PACKAGE_NAME
@@ -113,11 +145,13 @@ macro(generate_metametadata_file)
     string(APPEND metametadata_file_contents "    github_repo: ${${_cmake_pkg}_CONDA_GIHUB_REPO}\n")
     string(APPEND metametadata_file_contents "    github_tag: ${${_cmake_pkg}_CONDA_TAG}\n")
     string(APPEND metametadata_file_contents "    conda_build_number: ${CONDA_BUILD_NUMBER}\n")
+    string(APPEND metametadata_file_contents "    build_type: ${${_cmake_pkg}_CONDA_BUILD_TYPE}\n")
 
     if(_YH_${_cmake_pkg}_SOURCE_SUBDIR)
       string(APPEND metametadata_file_contents "    source_subdir: ${_YH_${_cmake_pkg}_SOURCE_SUBDIR}\n")
     endif()
 
+
     if(NOT "${${_cmake_pkg}_CONDA_CMAKE_ARGS}" STREQUAL "")
       string(APPEND metametadata_file_contents "    cmake_args:\n")
       foreach(_cmake_arg IN LISTS ${_cmake_pkg}_CONDA_CMAKE_ARGS)
@@ -216,7 +250,6 @@ macro(generate_conda_recipes)
     set(python_generation_script_additional_options "")
   endif()
   execute_process(COMMAND python ${python_generation_script} -i ${metametadata_file} -o ${generated_conda_recipes_dir} ${python_generation_script_additional_options} RESULT_VARIABLE CONDA_GENERATION_SCRIPT_RETURN_VALUE)
-  message(STATUS "CONDA_GENERATION_SCRIPT_RETURN_VALUE: ${CONDA_GENERATION_SCRIPT_RETURN_VALUE}")
   if(CONDA_GENERATION_SCRIPT_RETURN_VALUE STREQUAL "0")
     message(STATUS "conda recipes correctly generated in ${generated_conda_recipes_dir}.")
     message(STATUS "To build the generated conda recipes, navigate to the directory and run conda build . in it.")

conda/pure_python_recipe_template/meta.yaml

@@ -0,0 +1,37 @@
+{{ '{% set name =' }} "{{ name }}" {{ '%}' }}
+
+package:
+{% raw %}  name: {{ name }} {% endraw %}
+  version: "{{ version.replace("v","") }}"
+
+source:
+  git_url: https://github.com/{{ github_repo }}.git
+  git_rev: {{ github_tag }}
+
+build:
+  # Due to robot-log-visualizer, drop when conda-forge does not uses Python 3.7 anymore
+  skip: True  # [py<38]
+  number: {{ conda_build_number }}
+{% raw %}  script: {{ PYTHON }} -m pip install . --no-deps -vv {% endraw %}
+
+requirements:
+  build:
+
+  host:
+    - python
+    - pip
+{% for dep in dependencies %}    - {{ dep }}
+{% endfor %}
+  run:
+    - python
+{% for dep in dependencies %}    - {{ dep }}
+{% endfor %}
+
+test:
+  commands:
+    - pip check
+  requires:
+    - pip
+
+about:
+  home: https://github.com/{{ github_repo }}

conda/python/generate_conda_recipes_from_metametadata.py

@@ -126,17 +126,26 @@ def main():
     parser.add_argument('--generate_distro_metapackages', action='store_true', help="if passed also generates the recipes for the robotology-distro and robotology-distro-all metapackages ")
     args = parser.parse_args()
 
-    # Get recipe templates
-    recipe_template_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../recipe_template");
-    recipe_template_files = [f for f in os.listdir(recipe_template_dir) if os.path.isfile(os.path.join(recipe_template_dir, f))]
+    # Get cmake recipe template fles
+    cmake_recipe_template_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../cmake_recipe_template");
+    cmake_recipe_template_files = [f for f in os.listdir(cmake_recipe_template_dir) if os.path.isfile(os.path.join(cmake_recipe_template_dir, f))]
+    cmake_recipe_template_file_loader = jinja2.FileSystemLoader(cmake_recipe_template_dir)
+    cmake_recipe_template_env = jinja2.Environment(loader=cmake_recipe_template_file_loader)
+
+    # Get pure_python recipe template fles
+    pure_python_recipe_template_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../pure_python_recipe_template");
+    pure_python_recipe_template_files = [f for f in os.listdir(pure_python_recipe_template_dir) if os.path.isfile(os.path.join(pure_python_recipe_template_dir, f))]
+    pure_python_recipe_template_file_loader = jinja2.FileSystemLoader(pure_python_recipe_template_dir)
+    pure_python_recipe_template_env = jinja2.Environment(loader=pure_python_recipe_template_file_loader)
 
     # Get multisheller scripts
     multisheller_scripts_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../multisheller");
     multisheller_scripts = [f for f in os.listdir(multisheller_scripts_dir) if os.path.isfile(os.path.join(multisheller_scripts_dir, f))]
 
-    # Prepare Jinja templates
-    file_loader = jinja2.FileSystemLoader(recipe_template_dir)
-    env = jinja2.Environment(loader=file_loader)
+
+
+
+
     # Load metametadata
     metametadata = yaml.load(open(args.metametadata), Loader=yaml.FullLoader)
 
@@ -165,13 +174,22 @@ def main():
         else:
             pkg_info['copy_activation_scripts'] = False
 
-        # Generate recipe
-        for template_file in recipe_template_files:
-            template = env.get_template(template_file)
-            template_output = template.render(pkg_info)
-            with open(os.path.join(recipe_dir, template_file), 'w') as f:
-                f.write(template_output)
-
+        # Generate recipe if the package is installed via cmake
+        if pkg_info['build_type'] == "cmake":
+            for template_file in cmake_recipe_template_files:
+                template = cmake_recipe_template_env.get_template(template_file)
+                template_output = template.render(pkg_info)
+                with open(os.path.join(recipe_dir, template_file), 'w') as f:
+                    f.write(template_output)
+
+        # Generate recipe if the package is installed via cmake
+        if pkg_info['build_type'] == "pure_python":
+            for template_file in pure_python_recipe_template_files:
+                template = pure_python_recipe_template_env.get_template(template_file)
+                template_output = template.render(pkg_info)
+                with open(os.path.join(recipe_dir, template_file), 'w') as f:
+                    f.write(template_output)
+
     # If requested also generate recipes for distro metapackages
     if args.generate_distro_metapackages:
         recipe_metapackages_template_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../metapackages_recipes_template");