From d520cb283a887bdecf18dd481cf17b2bc749e0ce Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 9 Mar 2021 15:07:40 +0100 Subject: [PATCH] Add automatic generation of conda binary packages for Windows, macOS and Linux in robotology channel --- .github/workflows/ci.yml | 6 - .../workflows/generate-conda-packages.yaml | 112 ++++++++++++++++++ CMakeLists.txt | 10 ++ cmake/BuildCppAD.cmake | 3 + cmake/BuildGazeboYARPPlugins.cmake | 2 + cmake/BuildICUB.cmake | 14 +++ cmake/BuildLieGroupControllers.cmake | 2 + cmake/BuildOsqpEigen.cmake | 2 + cmake/BuildUnicyclePlanner.cmake | 2 + cmake/BuildWBToolbox.cmake | 2 + cmake/BuildYARP.cmake | 10 +- cmake/Buildbipedal-locomotion-framework.cmake | 2 + cmake/Buildblocktest-yarp-plugins.cmake | 3 + cmake/Buildblocktestcore.cmake | 3 + cmake/Buildcasadi.cmake | 3 + cmake/Buildhuman-dynamics-estimation.cmake | 2 + cmake/BuildiDynTree.cmake | 2 + cmake/Buildicub-basic-demos.cmake | 2 + cmake/Buildicub_firmware_shared.cmake | 5 +- cmake/Buildmanif.cmake | 3 + cmake/Buildmatio-cpp.cmake | 2 + cmake/Buildosqp.cmake | 3 + cmake/Buildqhull.cmake | 3 + cmake/BuildqpOASES.cmake | 2 + cmake/Buildwalking-controllers.cmake | 4 +- cmake/Buildwalking-teleoperation.cmake | 3 + cmake/Buildwhole-body-estimators.cmake | 2 + cmake/RobotologySuperbuildOptions.cmake | 10 +- doc/conda-forge.md | 64 ++++++++-- doc/conda-recipe-generation.md | 58 +++++++++ doc/developers-faqs.md | 17 +++ releases/latest.releases.yaml | 2 +- 32 files changed, 331 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/generate-conda-packages.yaml create mode 100644 doc/conda-recipe-generation.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ed97b23..95f02420d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,6 @@ name: C++ CI Workflow on: - push: - branches: - - master - - 'releases/**' - tags: - - v* pull_request: release: types: [published] diff --git a/.github/workflows/generate-conda-packages.yaml b/.github/workflows/generate-conda-packages.yaml new file mode 100644 index 000000000..06a5304dc --- /dev/null +++ b/.github/workflows/generate-conda-packages.yaml @@ -0,0 +1,112 @@ +name: Generate conda packages +# This action automatically generate conda packages for the packages in the robotology-superbuild +# Check doc/conda-recipe-generation.md for more info + +on: + workflow_dispatch: + schedule: + # Run the job once a week + - cron: '0 0 * * 2' + +jobs: + generate-conda-packages: + name: "Generate conda packages @${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + conda_platform: linux-64 + - os: macos-latest + conda_platform: osx-64 + - os: windows-2019 + conda_platform: win-64 + + steps: + - uses: actions/checkout@v2 + + - uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "*" + channels: conda-forge + channel-priority: true + python-version: "3.8" + + - name: Install files to enable compilation of mex files [Conda/Linux] + if: contains(matrix.os, 'ubuntu') + run: | + curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020b_mexa64.zip + unzip msdk_R2020b_mexa64.zip + rm msdk_R2020b_mexa64.zip + echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020b_mexa64" >> $GITHUB_ENV + echo "GHA_Matlab_MEX_EXTENSION=mexa64" >> $GITHUB_ENV + + - name: Install files to enable compilation of mex files [Conda/macOS] + if: contains(matrix.os, 'macos') + run: | + curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexmaci64.zip + unzip msdk_R2020a_mexmaci64.zip + rm msdk_R2020a_mexmaci64.zip + echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexmaci64" >> $GITHUB_ENV + echo "GHA_Matlab_MEX_EXTENSION=mexmaci64" >> $GITHUB_ENV + + - name: Install files to enable compilation of mex files [Conda/Windows] + if: contains(matrix.os, 'windows') + shell: bash + run: | + curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexw64.zip + unzip msdk_R2020a_mexw64.zip + rm msdk_R2020a_mexw64.zip + echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexw64" >> $GITHUB_ENV + echo "GHA_Matlab_MEX_EXTENSION=mexw64" >> $GITHUB_ENV + + # Python 3.8 is required by https://github.com/Anaconda-Platform/anaconda-client/pull/551 + - name: Dependencies for conda recipes generation and upload + shell: bash -l {0} + run: | + mamba install pyyaml jinja2 conda-build ninja anaconda-client + python -m pip install git+https://github.com/wolfv/multisheller.git@0cc03c68d0c68d2f9cf7b07ddb68afa531419a6d + + - name: Generate recipes [Linux&macOS] + if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu') + shell: bash -l {0} + run: | + mkdir build + cd build + cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON .. + + - name: Generate recipes [Windows] + if: contains(matrix.os, 'windows') + shell: bash -l {0} + run: | + mkdir build + cd build + cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON .. + + # Disable options not tested on Conda for now + # Reference issue: https://github.com/robotology/robotology-superbuild/issues/563 + - name: Disable options not supported in conda + shell: bash -l {0} + run: | + # Cleanup recipes + rm -rf build/conda/generated_recipes + # Re-generate + cd build + cmake -DROBOTOLOGY_USES_OCTAVE:BOOL=OFF -DROBOTOLOGY_USES_PYTHON:BOOL=OFF . + + - name: Build conda packages + shell: bash -l {0} + run: | + cd build/conda/generated_recipes + conda build -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . + - name: Upload conda packages + shell: bash -l {0} + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + run: | + cd ${CONDA_PREFIX}/conda-bld/${{ matrix.conda_platform}}/ + ls *.tar.bz2 + anaconda upload --skip-existing *.tar.bz2 + + diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b0a64b26..07092d090 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,15 @@ include(RobotologySuperbuildOptions) set(YCM_FOLDER src) set(YCM_COMPONENT core) set(YCM_MINIMUM_VERSION 0.11.1) + +# Include logic for generating Conda recipes (by default it is disabled) and exit +if(ROBOTOLOGY_GENERATE_CONDA_RECIPES) + message(STATUS "ROBOTOLOGY_GENERATE_CONDA_RECIPES CMake option enabled, generating conda recipes.") + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/conda/cmake") + include(RobotologySuperbuildGenerateCondaRecipes) + return() +endif() + include(YCMBootstrap) include(FindOrBuildPackage) @@ -44,6 +53,7 @@ include(YCMEPHelper) # depending on which profiles are enabled include(RobotologySuperbuildLogic) + if(ROBOTOLOGY_USES_MATLAB) # The following line is to properly configure the matlab and simulink software installed by the superbuild configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/template/startup_robotology_superbuild.m.in ${CMAKE_BINARY_DIR}/startup_robotology_superbuild.m) diff --git a/cmake/BuildCppAD.cmake b/cmake/BuildCppAD.cmake index b0cb87cad..e2cf05e99 100644 --- a/cmake/BuildCppAD.cmake +++ b/cmake/BuildCppAD.cmake @@ -10,3 +10,6 @@ ycm_ep_helper(CppAD TYPE GIT TAG master COMPONENT external FOLDER src) + +set(CppAD_CONDA_PKG_NAME cppad) +set(CppAD_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) diff --git a/cmake/BuildGazeboYARPPlugins.cmake b/cmake/BuildGazeboYARPPlugins.cmake index 09f23606d..811c4f565 100644 --- a/cmake/BuildGazeboYARPPlugins.cmake +++ b/cmake/BuildGazeboYARPPlugins.cmake @@ -40,3 +40,5 @@ ycm_ep_helper(GazeboYARPPlugins TYPE GIT DEPENDS YARP gazebo CMAKE_ARGS -DGAZEBO_YARP_PLUGINS_HAS_OPENCV:BOOL=ON) + +set(GazeboYARPPlugins_CONDA_DEPENDENCIES opencv gazebo) diff --git a/cmake/BuildICUB.cmake b/cmake/BuildICUB.cmake index ef72fe983..21607fb54 100644 --- a/cmake/BuildICUB.cmake +++ b/cmake/BuildICUB.cmake @@ -8,6 +8,7 @@ include(FindOrBuildPackage) find_or_build_package(YARP QUIET) set(ICUB_DEPENDS "") +list(APPEND ICUB_DEPENDS YCM) list(APPEND ICUB_DEPENDS YARP) if(ROBOTOLOGY_ENABLE_ICUB_HEAD) @@ -81,3 +82,16 @@ ycm_ep_helper(ICUB TYPE GIT -DENABLE_icubmod_socketcan:BOOL=${ENABLE_icubmod_socketcan} -DICUB_USE_icub_firmware_shared:BOOL=${ROBOTOLOGY_ENABLE_ICUB_HEAD} -DICUBMAIN_COMPILE_SIMULATORS:BOOL=${ICUBMAIN_COMPILE_SIMULATORS}) + +# Options related to generation of conda binary packages +set(ICUB_CONDA_DEPENDENCIES ace opencv gsl ipopt libode qt sdl) +if(NOT (APPLE OR WIN32)) + list(APPEND ICUB_CONDA_DEPENDENCIES libdc1394) +endif() + +if(NOT APPLE) + list(APPEND ICUB_CONDA_DEPENDENCIES freeglut) +endif() +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND ICUB_CONDA_DEPENDENCIES libglu) +endif() diff --git a/cmake/BuildLieGroupControllers.cmake b/cmake/BuildLieGroupControllers.cmake index e89154dc0..9dba63d61 100644 --- a/cmake/BuildLieGroupControllers.cmake +++ b/cmake/BuildLieGroupControllers.cmake @@ -13,3 +13,5 @@ ycm_ep_helper(LieGroupControllers TYPE GIT COMPONENT dynamics FOLDER src DEPENDS manif) + +set(LieGroupControllers_CONDA_DEPENDENCIES eigen) diff --git a/cmake/BuildOsqpEigen.cmake b/cmake/BuildOsqpEigen.cmake index fc2e74d15..a8efcacc3 100644 --- a/cmake/BuildOsqpEigen.cmake +++ b/cmake/BuildOsqpEigen.cmake @@ -14,3 +14,5 @@ ycm_ep_helper(OsqpEigen TYPE GIT FOLDER src CMAKE_ARGS -DBUILD_TESTING:BOOL=OFF DEPENDS osqp) + +set(OsqpEigen_CONDA_DEPENDENCIES "eigen") \ No newline at end of file diff --git a/cmake/BuildUnicyclePlanner.cmake b/cmake/BuildUnicyclePlanner.cmake index 0489b028e..4152ee094 100644 --- a/cmake/BuildUnicyclePlanner.cmake +++ b/cmake/BuildUnicyclePlanner.cmake @@ -14,3 +14,5 @@ ycm_ep_helper(UnicyclePlanner TYPE GIT COMPONENT dynamics FOLDER src DEPENDS iDynTree) + +set(UnicyclePlanner_CONDA_DEPENDENCIES eigen) diff --git a/cmake/BuildWBToolbox.cmake b/cmake/BuildWBToolbox.cmake index f17f96516..237fdbbb6 100644 --- a/cmake/BuildWBToolbox.cmake +++ b/cmake/BuildWBToolbox.cmake @@ -23,3 +23,5 @@ ycm_ep_helper(WBToolbox TYPE GIT iDynTree qpOASES BlockFactory) + +set(WBToolbox_CONDA_DEPENDENCIES eigen) diff --git a/cmake/BuildYARP.cmake b/cmake/BuildYARP.cmake index 04e1da2a2..84331bb46 100644 --- a/cmake/BuildYARP.cmake +++ b/cmake/BuildYARP.cmake @@ -5,10 +5,6 @@ include(YCMEPHelper) include(FindOrBuildPackage) -find_package(ACE QUIET) -find_package(SQLite QUIET) -find_package(Eigen3 QUIET) - set(YARP_OPTIONAL_DEPS "") if(ROBOTOLOGY_ENABLE_ROBOT_TESTING) find_or_build_package(RobotTestingFramework QUIET) @@ -17,8 +13,6 @@ endif() # Workaround for https://github.com/robotology/robotology-superbuild/issues/377 if(NOT APPLE) - find_package(SQLite QUIET) - list(APPEND YARP_OPTIONAL_DEPS SQLite) set(YARP_OPTIONAL_CMAKE_ARGS "") else() set(YARP_OPTIONAL_CMAKE_ARGS "-DYARP_USE_SYSTEM_SQLite:BOOL=OFF") @@ -51,8 +45,6 @@ ycm_ep_helper(YARP TYPE GIT COMPONENT core FOLDER src DEPENDS YCM - ACE - Eigen3 ${YARP_OPTIONAL_DEPS} CMAKE_ARGS -DYARP_COMPILE_GUIS:BOOL=ON -DYARP_USE_SYSTEM_SQLite:BOOL=ON @@ -89,3 +81,5 @@ ycm_ep_helper(YARP TYPE GIT -DCREATE_LUA:BOOL=${ROBOTOLOGY_USES_LUA} -DENABLE_yarpmod_usbCamera:BOOL=${ENABLE_USBCAMERA} ${YARP_OPTIONAL_CMAKE_ARGS}) + +set(YARP_CONDA_DEPENDENCIES ace opencv tinyxml qt eigen sdl sdl2 sqlite libjpeg-turbo) diff --git a/cmake/Buildbipedal-locomotion-framework.cmake b/cmake/Buildbipedal-locomotion-framework.cmake index 8be4b39cf..4d8ed2e41 100644 --- a/cmake/Buildbipedal-locomotion-framework.cmake +++ b/cmake/Buildbipedal-locomotion-framework.cmake @@ -45,3 +45,5 @@ ycm_ep_helper(bipedal-locomotion-framework TYPE GIT -DFRAMEWORK_USE_casadi:BOOL=${ROBOTOLOGY_ENABLE_DYNAMICS_FULL_DEPS} -DFRAMEWORK_USE_LieGroupControllers:BOOL=${ROBOTOLOGY_ENABLE_DYNAMICS_FULL_DEPS} DEPENDS ${bipedal-locomotion-framework_DEPENDS}) + +set(bipedal-locomotion-framework_CONDA_DEPENDENCIES eigen) diff --git a/cmake/Buildblocktest-yarp-plugins.cmake b/cmake/Buildblocktest-yarp-plugins.cmake index 5a6052f81..1b2b2db75 100644 --- a/cmake/Buildblocktest-yarp-plugins.cmake +++ b/cmake/Buildblocktest-yarp-plugins.cmake @@ -17,3 +17,6 @@ ycm_ep_helper(blocktest-yarp-plugins TYPE GIT DEPENDS YARP blocktestcore CMAKE_CACHE_ARGS -DENABLE_MSVC_WARNINGS:BOOL=OFF) + +set(blocktest-yarp-plugins_CONDA_DEPENDENCIES boost-cpp) +set(blocktest-yarp-plugins_CONDA_VERSION "1.1.0.1") diff --git a/cmake/Buildblocktestcore.cmake b/cmake/Buildblocktestcore.cmake index 75f70c23f..cbfe189b7 100644 --- a/cmake/Buildblocktestcore.cmake +++ b/cmake/Buildblocktestcore.cmake @@ -14,3 +14,6 @@ ycm_ep_helper(blocktestcore TYPE GIT FOLDER src CMAKE_CACHE_ARGS -DENABLE_MSVC_WARNINGS:BOOL=OFF DEPENDS YCM) + +set(blocktestcore_CONDA_DEPENDENCIES qt boost-cpp) +set(blocktestcore_CONDA_VERSION "2.3.0.1") diff --git a/cmake/Buildcasadi.cmake b/cmake/Buildcasadi.cmake index a7b34cd7c..8192997b5 100644 --- a/cmake/Buildcasadi.cmake +++ b/cmake/Buildcasadi.cmake @@ -21,3 +21,6 @@ ycm_ep_helper(casadi TYPE GIT -DLIB_PREFIX:PATH=lib -DBIN_PREFIX:PATH=bin DEPENDS osqp) + +set(casadi_CONDA_PKG_NAME casadi) +set(casadi_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) diff --git a/cmake/Buildhuman-dynamics-estimation.cmake b/cmake/Buildhuman-dynamics-estimation.cmake index a71e0f310..d13cf1c06 100644 --- a/cmake/Buildhuman-dynamics-estimation.cmake +++ b/cmake/Buildhuman-dynamics-estimation.cmake @@ -21,3 +21,5 @@ ycm_ep_helper(human-dynamics-estimation TYPE GIT wearables osqp OsqpEigen) + +set(human-dynamics-estimation_CONDA_DEPENDENCIES eigen) diff --git a/cmake/BuildiDynTree.cmake b/cmake/BuildiDynTree.cmake index dcbc3521a..3ef998445 100644 --- a/cmake/BuildiDynTree.cmake +++ b/cmake/BuildiDynTree.cmake @@ -29,3 +29,5 @@ ycm_ep_helper(iDynTree TYPE GIT -DIDYNTREE_USES_PYTHON:BOOL=${ROBOTOLOGY_USES_PYTHON} -DIDYNTREE_USES_OCTAVE:BOOL=${ROBOTOLOGY_USES_OCTAVE} DEPENDS ${iDynTree_DEPENDS}) + +set(iDynTree_CONDA_DEPENDENCIES libxml2 ipopt eigen qt irrlicht) diff --git a/cmake/Buildicub-basic-demos.cmake b/cmake/Buildicub-basic-demos.cmake index 6f48540bd..7082535fc 100644 --- a/cmake/Buildicub-basic-demos.cmake +++ b/cmake/Buildicub-basic-demos.cmake @@ -16,3 +16,5 @@ ycm_ep_helper(icub-basic-demos TYPE GIT DEPENDS YARP ICUB ICUBcontrib) + +set(icub-basic-demos_CONDA_DEPENDENCIES opencv qt) diff --git a/cmake/Buildicub_firmware_shared.cmake b/cmake/Buildicub_firmware_shared.cmake index 791e4c7a9..908c3b4fb 100644 --- a/cmake/Buildicub_firmware_shared.cmake +++ b/cmake/Buildicub_firmware_shared.cmake @@ -5,8 +5,11 @@ include(YCMEPHelper) include(FindOrBuildPackage) +find_or_build_package(YCM QUIET) + ycm_ep_helper(icub_firmware_shared TYPE GIT STYLE GITHUB REPOSITORY robotology/icub-firmware-shared.git COMPONENT iCub - FOLDER src) + FOLDER src + DEPENDS YCM) diff --git a/cmake/Buildmanif.cmake b/cmake/Buildmanif.cmake index cbda4c12c..1c85a2073 100644 --- a/cmake/Buildmanif.cmake +++ b/cmake/Buildmanif.cmake @@ -11,3 +11,6 @@ ycm_ep_helper(manif TYPE GIT COMPONENT external FOLDER src CMAKE_ARGS -DBUILD_TESTING:BOOL=OFF -DBUILD_EXAMPLES:BOOL=OFF) + +set(manif_CONDA_PKG_NAME manif) +set(manif_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) diff --git a/cmake/Buildmatio-cpp.cmake b/cmake/Buildmatio-cpp.cmake index 33e621ef5..d9351d546 100644 --- a/cmake/Buildmatio-cpp.cmake +++ b/cmake/Buildmatio-cpp.cmake @@ -10,3 +10,5 @@ ycm_ep_helper(matio-cpp TYPE GIT COMPONENT dynamics FOLDER src CMAKE_ARGS -DBUILD_TESTING:BOOL=OFF) + +set(matio-cpp_CONDA_DEPENDENCIES "libmatio") diff --git a/cmake/Buildosqp.cmake b/cmake/Buildosqp.cmake index be69044e7..f39e7cdc5 100644 --- a/cmake/Buildosqp.cmake +++ b/cmake/Buildosqp.cmake @@ -11,3 +11,6 @@ ycm_ep_helper(osqp TYPE GIT COMPONENT external FOLDER src CMAKE_ARGS -DUNITTESTS:BOOL=OFF) + +set(osqp_CONDA_PKG_NAME libosqp) +set(osqp_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) diff --git a/cmake/Buildqhull.cmake b/cmake/Buildqhull.cmake index 47a47c902..8831f69f5 100644 --- a/cmake/Buildqhull.cmake +++ b/cmake/Buildqhull.cmake @@ -11,3 +11,6 @@ ycm_ep_helper(qhull TYPE GIT COMPONENT external FOLDER src CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON) + +set(qhull_CONDA_PKG_NAME qhull) +set(qhull_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) diff --git a/cmake/BuildqpOASES.cmake b/cmake/BuildqpOASES.cmake index f4640f57b..78b8a648d 100644 --- a/cmake/BuildqpOASES.cmake +++ b/cmake/BuildqpOASES.cmake @@ -28,3 +28,5 @@ ycm_ep_helper(qpOASES TYPE GIT COMPONENT external FOLDER src CMAKE_ARGS -DQPOASES_BUILD_BINDINGS_MATLAB:BOOL=OFF) + +set(qpOASES_CONDA_PKG_NAME "qpoases") diff --git a/cmake/Buildwalking-controllers.cmake b/cmake/Buildwalking-controllers.cmake index e6321aacd..e7bf3c143 100644 --- a/cmake/Buildwalking-controllers.cmake +++ b/cmake/Buildwalking-controllers.cmake @@ -24,8 +24,10 @@ list(APPEND walking-controllers_DEPENDS ICUBcontrib) ycm_ep_helper(walking-controllers TYPE GIT STYLE GITHUB - REPOSITORY robotology/walking-controllers + REPOSITORY robotology/walking-controllers.git TAG master COMPONENT dynamics FOLDER src DEPENDS ${walking-controllers_DEPENDS}) + +set(walking-controllers_CONDA_DEPENDENCIES "eigen") \ No newline at end of file diff --git a/cmake/Buildwalking-teleoperation.cmake b/cmake/Buildwalking-teleoperation.cmake index 503c13efc..c7e6b772c 100644 --- a/cmake/Buildwalking-teleoperation.cmake +++ b/cmake/Buildwalking-teleoperation.cmake @@ -17,3 +17,6 @@ ycm_ep_helper(walking-teleoperation TYPE GIT DEPENDS iDynTree ICUB YARP) + +set(walking-teleoperation_CONDA_DEPENDENCIES "eigen") +set(walking-teleoperation_CONDA_VERSION "0.0.1") diff --git a/cmake/Buildwhole-body-estimators.cmake b/cmake/Buildwhole-body-estimators.cmake index d082f7128..dfa117b86 100644 --- a/cmake/Buildwhole-body-estimators.cmake +++ b/cmake/Buildwhole-body-estimators.cmake @@ -22,3 +22,5 @@ ycm_ep_helper(whole-body-estimators TYPE GIT COMPONENT dynamics FOLDER src DEPENDS ${whole-body-estimators_DEPENDS}) + +set(whole-body-estimators_CONDA_DEPENDENCIES "eigen") diff --git a/cmake/RobotologySuperbuildOptions.cmake b/cmake/RobotologySuperbuildOptions.cmake index 4d4f20c24..d994c39dd 100644 --- a/cmake/RobotologySuperbuildOptions.cmake +++ b/cmake/RobotologySuperbuildOptions.cmake @@ -67,16 +67,19 @@ if(NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS ${ROBOTOLOGY_BUILD_TYPES}) endif() -set(ROBOTOLOGY_PROJECT_TAGS "Stable" CACHE STRING "The tags to be used for the robotology projects: Stable, Unstable or Custom. This can be changed only before the first configuration.") +set(ROBOTOLOGY_PROJECT_TAGS "Stable" CACHE STRING "The tags to be used for the robotology projects: Stable, Unstable, LatestRelease or Custom. This can be changed only before the first configuration.") mark_as_advanced(ROBOTOLOGY_PROJECT_TAGS) set(ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE CACHE FILEPATH "If ROBOTOLOGY_PROJECT_TAGS is custom, this file will be loaded to specify the tags of the projects to use.") mark_as_advanced(ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE) -set_property(CACHE ROBOTOLOGY_PROJECT_TAGS PROPERTY STRINGS "Stable" "Unstable" "Custom") +set_property(CACHE ROBOTOLOGY_PROJECT_TAGS PROPERTY STRINGS "Stable" "Unstable" "LatestRelease" "Custom") if(ROBOTOLOGY_PROJECT_TAGS STREQUAL "Stable") include(ProjectsTagsStable) elseif(ROBOTOLOGY_PROJECT_TAGS STREQUAL "Unstable") include(ProjectsTagsUnstable) +elseif(ROBOTOLOGY_PROJECT_TAGS STREQUAL "LatestRelease") + include(YCMLoadVcsYamlInfo) + ycm_load_vcs_yaml_info(YAML_FILE ${PROJECT_SOURCE_DIR}/releases/latest.releases.yaml VERBOSE) elseif(ROBOTOLOGY_PROJECT_TAGS STREQUAL "Custom") if(ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE MATCHES ".yaml$" OR ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE MATCHES ".repos$") @@ -88,3 +91,6 @@ elseif(ROBOTOLOGY_PROJECT_TAGS STREQUAL "Custom") else() message(FATAL_ERROR "The ROBOTOLOGY_PROJECT_TAGS variable can be Stable, Unstable or Custom. ${ROBOTOLOGY_PROJECT_TAGS} value is not supported.") endif() + +option(ROBOTOLOGY_GENERATE_CONDA_RECIPES "If enabled, generate conda recipes instead of building the superbuild. This should not be used by end users." OFF) +mark_as_advanced(ROBOTOLOGY_GENERATE_CONDA_RECIPES) diff --git a/doc/conda-forge.md b/doc/conda-forge.md index af9cfed61..1bbf65332 100644 --- a/doc/conda-forge.md +++ b/doc/conda-forge.md @@ -4,6 +4,11 @@ [`conda-forge`](https://conda-forge.org/) is a community-mantained channel for the `conda` package manager that provides many dependencies useful for scientific software, in particular all the one that are required by the robotology-superbuild . +## Table of Contents +* [conda-forge overview](#conda-forge-overview) +* [Binary installation](#binary-installation) +* [Source installation](#source-installation) + ## conda-forge overview The **advantages** of the use of conda and conda-forge are: @@ -18,23 +23,62 @@ The **advantages** of the use of conda and conda-forge are: The **disadvantages** of conda and conda-forge are: * **Not compatible with dependencies installed by other package managers:** As conda-forge provides all the software it installs without using almost anything from the system, you cannot mix libraries installed by other package managers (such as `apt` or `homebrew`). If you have a dependency that is only provided by `apt` and is not available in `conda-forge`, then you should install all the dependencies that you want to use from `apt`. -* **No Debug version of C++ libraries on Windows:** as other package managers such as `apt` or `homebrew`, a part from some small exceptions conda-forge only ships the version of the libraries compiled as shared library and in `Release` compilation mode. This is not a big problem on Linux or macOS, where executables compiled in `Debug` can link libraries compiled in `Release`, but is more problematic on Windows, where this is not possible. When you are using conda-forge libraries on Windows, you can't compile your executable in `Debug` mode, but only in `Release` or `RelWithDebInfo` mode. If you stricly need to also have the `Debug` version of the libraries, the [`vcpkg`](https://github.com/microsoft/vcpkg) package manager may be more useful for oyu. +* **No Debug version of C++ libraries on Windows:** as other package managers such as `apt` or `homebrew`, a part from some small exceptions conda-forge only ships the version of the libraries compiled as shared library and in `Release` compilation mode. This is not a big problem on Linux or macOS, where executables compiled in `Debug` can link libraries compiled in `Release`, but is more problematic on Windows, where this is not possible. When you are using conda-forge libraries on Windows, you can't compile your executable in `Debug` mode, but only in `Release` or `RelWithDebInfo` mode. If you stricly need to also have the `Debug` version of the libraries, the [`vcpkg`](https://github.com/microsoft/vcpkg) package manager may be more useful for you. -## Source installation +## Binary installation -This section describes how to compile and install the robotology-superbuild with conda-forge provided dependencies on Windows, macOS and Linux. +This section describes how to compile and install the binary packages built from the robotology-superbuild on conda on Windows, macOS and Linux. -### Install a conda distribution -If you do not have a conda distribution on your system, we suggest to use the minimal -[`miniforge3`](https://github.com/conda-forge/miniforge#miniforge3) distribution, that uses `conda-forge` packages by default. +The binary packages are hosted in the [`robotology` conda channel](https://anaconda.org/robotology) . Only packages that are built as part of the profiles and options that are supported on Conda (see [documentation on profiles](profiles.md)). + +### Create an environment +Differently from `apt` and `homebrew`, the `conda` package manager is an `environment`-oriented package manager, meaning that packages are not +installed in some global location, but rather you install packages in an `environment` (that is just a directory in your filesystem), so that you +can easily have multiple different environments with different packages installed on your system. To read more about this, check https://docs.conda.io/projects/conda/en/4.6.1/user-guide/tasks/manage-environments.html . -After you installed it, make sure that you can execute the `conda info` package from your terminal, and its output is similar to: +For this reason, to use the robotology conda packages it is suggested to first create a conda environment, and then install in it all the packages you want to use. To create a new environment called `robotologyenv`, execute the following command: +~~~ +conda create -n robotologyenv +~~~ + +Once you created the `robotologyenv` environment, you can "activate" it for the current terminal (i.e. make sure that the installed packages can be found) by the command: +~~~ +conda activate robotologyenv ~~~ -$ conda info - active environment : None -[...] +**IMPORTANT: if you open a new terminal, you need to manually activate the environment also there.** + +### Install robotology packages + +Once you are in an activated environment, you can install robotology packages by just running the command: ~~~ +conda install -c robotology +~~~ + +The list of available packages is available at https://anaconda.org/robotology/repo . + +For example, if you want to install yarp and icub-main, you simple need to install: +~~~ +conda install -c robotology yarp icub-main +~~~ + +While if you want to simulate the iCub in Gazebo, you can install `icub-models` and `gazebo-yarp-plugins`: +~~~ +conda install -c robotology gazebo-yarp-plugins icub-models +~~~ + +If you want to develop some C++ code on the top of this libraries, it is recommended to also install the necessary compiler and development tools directly in the same environment: +~~~ +conda installl compilers cmake pkg-config make ninja +~~~ + +## Source installation + +This section describes how to compile and install the robotology-superbuild with conda-forge provided dependencies on Windows, macOS and Linux. + +### Install a conda distribution +If you do not have a conda distribution on your system, we suggest to use the minimal +[`miniforge3`](https://github.com/conda-forge/miniforge#miniforge3) distribution, that uses `conda-forge` packages by default, following the instructions in our [`install-miniforge`](install-miniforge.md) documentation. Also other conda distributions should be supported, you just will need to manually specify to use the `conda-forge` packages by adding `-c conda-forge` to all `conda install` commands. diff --git a/doc/conda-recipe-generation.md b/doc/conda-recipe-generation.md new file mode 100644 index 000000000..7606af096 --- /dev/null +++ b/doc/conda-recipe-generation.md @@ -0,0 +1,58 @@ +Conda Recipe Generation +======================= + +## Rationale + +See the [documentation on conda-forge and the robotology-superbuild](conda-forge.md) for a general discussion on how the robotology-superbuild and conda can work together. +This document describes how to + +## How to generate recipes + +To generate the conda recipes for a given configuration of the `robotology-superbuild`, configure the `robotology-superbuild` in a conda workspace (TODO: docs on this, for now check the GitHub Action CI). As the conda recipes are intendend to build release version of software, specify an option to get a specific release. + +After that, install the additional dependencies required for the recipe generation: +~~~ +conda install pyyaml jinja2 conda-build ninja anaconda-client +python -m pip install git+https://github.com/wolfv/multisheller.git@0cc03c68d0c68d2f9cf7b07ddb68afa531419a6d +~~~ + +Then, set the `ROBOTOLOGY_GENERATE_CONDA_RECIPES` CMake option to `ON` in the `robotology-superbuild` build. This will **deactivate** the usual logic +for building projects of the `robotology-superbuild`, and instead will generate (at CMake configure time) conda recipes in `/conda/generated_recipes` for all the projects of the superbuild. If the configuration was concluded correctly, i.e. CMake configuration ended with: +~~~ +-- To build the generated conda recipes, navigate to the directory and run conda build . in it. +-- Configuring done +-- Generating done +-- Build files have been written to: C:/src/robotology-superbuild/build-conda +~~~ +you can then build the generated recipes by moving in the `/conda/generated_recipes` directory and running either [`conda build .`](https://github.com/conda/conda-build) or [`boa .`](https://github.com/mamba-org/boa). + +Note that the generated recipes will depend on the specific configuration of the robotology-superbuild used, so enabling additional profiles will generate recipes +for the packages contained in those profiles. For this reason, the generated recipes in general are not cross-platform. For example the recipes generated on `Linux` could contain Linux-specific CMake options passed to the projects, so may not be usable on Windows. +If you want to obtain recipe that can be built on a given operating system, please generate them with the robotology-superbuild on that operating system. + +## Internals + +All the files necessary to generate the conda recipes are contained in the `conda` directory of the root source dir. + +In particular: + +### `conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake` +This CMake scripts contain the logic to generate the recipes. The generation is articulated as follows. + +The `cmake/RobotologySuperbuildLogic.cmake` file, that contains the logic of which projects should be built, +is run after redefining the `ycm_ep_helper` and `find_or_build_package` YCM functions, to extract the informations on name, +versions, dependency and cmake options of subpackages. This information are combined with the conda-specific metadata specified +in `conda/cmake/CondaGenerationOptions.cmake` (that contains information such as conda dependencies external to the `robotology-superbuild`) +to generate the `/conda/robotology-superbuild-conda-metametadata.yaml` file. + +After that, the data in `/conda/robotology-superbuild-conda-metametadata.yaml` is used via the `conda/python/generate_conda_recipes_from_metadatadata.py` +python script to generate the conda recipes. + +### `conda/recipe_template` + +This directory contains `Jinja2` templates for the files `meta.yaml`, `bld.bat` and `build.sh` that will be part of each generated recipe. + +### `conda/python/generate_conda_recipes_from_metadatadata.py` + +Python script that takes the `metametadata` file generated by the `generate_metametadata_file` CMake macro, and uses it to configure the +`Jinja2` templates `conda/recipe_template` to generate the recipe for each specific package. diff --git a/doc/developers-faqs.md b/doc/developers-faqs.md index b3cbb1491..80c8f6105 100644 --- a/doc/developers-faqs.md +++ b/doc/developers-faqs.md @@ -57,3 +57,20 @@ you can disable the automatical update of the tags by adding its CMake name in t * Once the release is ready to be made, create a `releases/yyyy.mm` branch from `master` * On the branch `releases/yyyy.mm` modify the default value of the `ROBOTOLOGY_PROJECT_TAGS` CMake option to be `Custom` and of the `ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE` to the point to the `yyyy.mm.yaml` file. * Create a new tag and release `vyyyy.mm` on the `releases/yyyy.mm` branch + + +## How to ensure that binary packages are correctly generated for a new package +* If the package is already available in [`conda-forge`](https://conda-forge.org), then no binary should be create and the `conda-forge` version should be used. This is done by setting in the `Build.cmake` file the `_CONDA_PKG_NAME` variable to the name of the package in `conda-forge`, and setting to `ON` the `_CONDA_PKG_CONDA_FORGE_OVERRIDE` variable. For an example of such package, see [Buildosqp.cmake](../cmake/Buildosqp.cmake). +* If instead the package is not part of conda-forge, then it is appropriate to generate a binary package as part of the `robotology` channel, providing the following CMake options in the `Build.cmake` file: + +| Variable | Meaning | Default Value | +|:--------:|:-------:|:--------------:| +| `_CONDA_PKG_NAME` | The name that will be used for the conda package name. The convention is to use lowercase names separated by dashes. | The name of the github repo of the package. | +| `_CONDA_DEPENDENCIES` | The list of conda-forge dependencies required by this package. Note that dependencies managed by the robotology-superbuild should not be listed, as those are handled automatically. | The default value is empty. | + +For any doubt, double check the existing `Build.cmake` files for inspiration. +* If your package needs to set or modify some environment variables to work correctly, it should provide a pair [multisheller](https://github.com/wolfv/multisheller) scripts name `_activate.msh` and `_activate.msh` in the `conda/multisheller` directory to describe how the environment should be modified, see the existing scripts for more details. + + +## How ofter are conda binary packages generated? +* The conda binaries hosted in the [robotology conda channel](https://anaconda.org/robotology) are re-generated weekly by [`generate-conda-packages`](../.github/workflows/generate-conda-packages.yaml) GitHub Action CI job. At each new re-build, remember to bump the `CONDA_BUILD_NUMBER` in the [`conda/cmake/CondaGenerationOptions.cmake`](../conda/cmake/CondaGenerationOptions.cmake) file. The latest released version of the packages, as specified in the [`releases/latest.releases.yaml`](../releases/latest.releases.yaml) file is used to generate the binaries, that is in turn updated weekly automatically by the [`update-latest-releases`](../.github/workflows/update-latest-releases.yml) GitHub Action CI job. If you need to quickly generate a new binary for a new release, feel free to [open an issue to request that](https://github.com/robotology/robotology-superbuild/issues/new). diff --git a/releases/latest.releases.yaml b/releases/latest.releases.yaml index 85dc519fe..f8cac882a 100644 --- a/releases/latest.releases.yaml +++ b/releases/latest.releases.yaml @@ -54,7 +54,7 @@ repositories: yarp-matlab-bindings: type: git url: https://github.com/robotology/yarp-matlab-bindings.git - version: v3.4.1 + version: v3.4.2 RobotTestingFramework: type: git url: https://github.com/robotology/robot-testing-framework.git