Add generation of conda metapackages robotology-distro and robotology…

…-distro-all for each new distro release (#1030)

.github/workflows/generate-conda-packages.yaml

@@ -12,11 +12,39 @@ on:
   schedule:
   # Run the job once a week
   - cron: '0 0 * * 2'
+  release:
+    types: [published]
 
 jobs:
+    # Regardless of the branch on which this action is trigged, 
+    # the CONDA_BUILD_NUMBER CMake option needs to be read from the 
+    # master branch, to avoid that a conda package generation from the 
+    # master branch and one from a releases/YYYY.MM branch use the same
+    # CONDA_BUILD_NUMBER
+    get-conda-build-number:
+        name: "Read Conda Build number from master branch"
+        runs-on: ubuntu-latest
+        # Define outputs (see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-defining-outputs-for-a-job)
+        outputs:
+          conda_build_number: ${{ steps.step1.outputs.conda_build_number }}
+
+        steps:
+        - uses: actions/checkout@v2
+          with:
+            ref: 'master'
+
+        - id: step1
+          name: Get CONDA_BUILD_NUMBER and set it as output
+          shell: bash
+          run: |
+            # Get CONDA_BUILD_NUMBER via grep and set it to an environment variable 
+            export CONDA_BUILD_NUMBER=`grep "CONDA_BUILD_NUMBER" ./conda/cmake/CondaGenerationOptions.cmake | grep -oe '\([0-9.]*\)'` 
+            echo "::set-output name=conda_build_number::${CONDA_BUILD_NUMBER}"
+
     generate-conda-packages:
         name: "Generate conda packages @${{ matrix.os }}"
         runs-on:  ${{ matrix.os }}
+        needs: get-conda-build-number
         strategy:
             fail-fast: false
             matrix:
@@ -88,15 +116,22 @@ jobs:
           run: |
             mkdir build
             cd build
-            cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON ..
+            cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
 
         - name: Generate recipes [Windows]
           if: contains(matrix.os, 'windows')
           shell: bash -l {0}
           run: |
             mkdir build
             cd build
-            cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON ..
+            cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake  -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}}  ..
+
+        - name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages
+          if: github.event_name == 'release'
+          shell: bash -l {0}
+          run: |
+            cd build
+            cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON .
 
         - name: Build conda packages
           shell: bash -l {0}
@@ -155,12 +190,22 @@ jobs:
             rm -rf blocktest
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml human-dynamics-estimation
             rm -rf human-dynamics-estimation
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . 
+            conda build -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . 
+
+        - name: Build conda metapackages (if necessary
+          if: github.event_name == 'release'
+          shell: bash -l {0}
+          run: |
+            cd build/conda/generated_recipes_metapackages
+             # Debug generated recipes
+            cat */meta.yaml
+            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro
+            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all
 
         - name: Upload conda packages
           shell: bash -l {0}
           # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
-          if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
+          if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
           env:
             ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
           run: |
@@ -170,22 +215,26 @@ jobs:
 
     # If the  generate-conda-packages completed correctly and binaries are uploaded,
     # bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake
-    # for future builds
+    # of master branch for future builds
+    # the master branch is always used in case the action is triggered by a release on 
+    # a release/vYYYY.MM branch
     bump-conda-build-number:
         name: "Bump Conda Build number for future builds"
         runs-on: ubuntu-latest
         needs: generate-conda-packages
-        if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
+        if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
 
         steps:
         - uses: actions/checkout@v2
+          with:
+            ref: 'master'
 
         - name: Bump Conda Build number for future builds
           shell: bash
           run: |
             sh ./scripts/robotologyBumpCondaBuildNumber.sh 
       
-        - uses: EndBug/add-and-commit@v7.0.0
+        - uses: EndBug/add-and-commit@v8.0.1
           with:
             default_author: github_actions
             message: 'Bump CONDA_BUILD_NUMBER after successful Conda packages build and upload'

.github/workflows/release.yml

@@ -40,6 +40,9 @@ jobs:
         sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION ${{ github.event.inputs.superbuild_year_month_prefix }}.0)/g' ./packaging/windows/CMakeLists.txt
         git add .
         git commit -m "Updating packaging/windows/CMakeLists.txt"
+        sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION ${{ github.event.inputs.superbuild_year_month_prefix }}.0)/g' ./conda/cmake/CondaGenerationOptions.cmake
+        git add .
+        git commit -m "Updating conda/cmake/CondaGenerationOptions.cmake"
         git push --set-upstream origin releases/${{ github.event.inputs.superbuild_year_month_prefix }}
 
     - name: Create and push the ${{ github.event.inputs.superbuild_year_month_prefix }}.0 tag

CHANGELOG.md

@@ -7,6 +7,7 @@ The format of this document is based on [Keep a Changelog](https://keepachangelo
 
 ### Added
 - Added dependency on `graphviz` to compile `yarpviz` YARP tool (https://github.com/robotology/robotology-superbuild/pull/988).
+- Added generation of `robotology-distro` and `robotology-distro-all` conda metapackages on releases (https://github.com/robotology/robotology-superbuild/pull/1030). 
 
 ### Changed
 - On Windows the option `ROBOTOLOGY_USES_ESDCAN` is now enabled when generating conda packages (https://github.com/robotology/robotology-superbuild/pull/935).

cmake/Buildcasadi.cmake

@@ -35,3 +35,9 @@ ycm_ep_helper(casadi TYPE GIT
 
 set(casadi_CONDA_PKG_NAME casadi)
 set(casadi_CONDA_PKG_CONDA_FORGE_OVERRIDE ON)
+# This is a small hack. To avoid incompatibilities between the version tagged in the ami-iit fork
+# (something like 3.5.5.x) and the version available in conda-forge when generating conda metapackages
+# such as robotology-distro and robotology-distro-all, we override the conda package version of casadi
+# here. This needs to be removed as soon as we stop use our fork in the superbuild 
+set(casadi_CONDA_VERSION 3.5.5)
+

cmake/Buildmanif.cmake

@@ -21,3 +21,8 @@ ycm_ep_helper(manif TYPE GIT
 
 set(manif_CONDA_PKG_NAME manif)
 set(manif_CONDA_PKG_CONDA_FORGE_OVERRIDE ON)
+# This is a small hack. To avoid incompatibilities between the version tagged in the ami-iit fork
+# (something like 0.0.4.x) and the version available in conda-forge when generating conda metapackages
+# such as robotology-distro and robotology-distro-all, we override the conda package version of manif
+# here. This needs to be removed as soon as we stop using our fork in the superbuild 
+set(manif_CONDA_VERSION 0.0.4)

conda/cmake/CondaGenerationOptions.cmake

@@ -1,7 +1,18 @@
 # This number needs to be increased at each full rebuild,
 # to ensure that binaries belonging to different rebuilds
 # can be distinguished even if the version number is the same
-set(CONDA_BUILD_NUMBER 43)
+if(NOT CONDA_BUILD_NUMBER)
+  set(CONDA_BUILD_NUMBER 43)
+endif()
+
+# This option is enabled only when the metapackages robotology-distro and robotology-distro-all are
+# generated, so only in case a robotology-superbuild distro is released
+option(CONDA_GENERATE_ROBOTOLOGY_METAPACKAGES "If on, generate recipes for robotology-distro and robotology-distro-all conda metapackages." OFF)
+
+# This variable is automatically set to contain the robotology-superbuild version in case of releases
+if(NOT CONDA_ROBOTOLOGY_SUPERBUILD_VERSION)
+  set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")
+endif()
 
 # For more conda generation options, check the specific project Build<CMakeProject>.cmake
 # file for variables that start with `<CMakeProject>_CONDA`

conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake

@@ -60,15 +60,10 @@ macro(generate_metametadata_file)
 
   get_property(_superbuild_pkgs GLOBAL PROPERTY YCM_PROJECTS)
   foreach(_cmake_pkg IN LISTS _superbuild_pkgs)
-    # If a package is already available in conda-forge, we use
-    # that one by defining appropriately the <_cmake_pkg>_CONDA_PACKAGE_NAME
-    # and <_cmake_pkg>_CONDA_PKG_CONDA_FORGE_OVERRIDE variables
-    if(DEFINED ${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE AND
-       "${${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE}")
-      continue()
-    endif()
-
     # Compute conda version
+    # We do it for all packages as this is also necessary
+    # for packages for which <_cmake_pkg>_CONDA_PKG_CONDA_FORGE_OVERRIDE
+    # is defined when generating the metapackages
     if(DEFINED ${_cmake_pkg}_TAG)
      set(${_cmake_pkg}_CONDA_TAG ${${_cmake_pkg}_TAG})
     else()
@@ -79,6 +74,15 @@ macro(generate_metametadata_file)
       set(${_cmake_pkg}_CONDA_VERSION ${${_cmake_pkg}_CONDA_TAG})
     endif()
 
+
+    # If a package is already available in conda-forge, we use
+    # that one by defining appropriately the <_cmake_pkg>_CONDA_PACKAGE_NAME
+    # and <_cmake_pkg>_CONDA_PKG_CONDA_FORGE_OVERRIDE variables
+    if(DEFINED ${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE AND
+       "${${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE}")
+      continue()
+    endif()
+
     # Compute conda CMake options
     set(${_cmake_pkg}_CONDA_CMAKE_ARGS ${_YH_${_cmake_pkg}_CMAKE_ARGS})
     list(APPEND ${_cmake_pkg}_CONDA_CMAKE_ARGS ${_YH_${_cmake_pkg}_CMAKE_CACHE_ARGS})
@@ -178,10 +182,26 @@ macro(generate_metametadata_file)
       string(APPEND metametadata_file_contents "    add_numpy_runtime_dep: true\n")
     endif()
 
-
+    string(APPEND metametadata_file_contents "\n")
+
     string(APPEND metametadata_file_contents "\n")
   endforeach()
 
+  # If we generate robotology-distro and robotology-distro-all metapackages, we need also to add the
+  # conda-metapackages-metametadata: section that will be used to generate the metapackages recipes
+  # To the people from the future: I am really sorry about the amount of "meta" in these names
+  if(CONDA_GENERATE_ROBOTOLOGY_METAPACKAGES)
+    string(APPEND metametadata_file_contents "conda-metapackages-metametadata:\n")
+    string(APPEND metametadata_file_contents "  robotology_superbuild_version: ${CONDA_ROBOTOLOGY_SUPERBUILD_VERSION}\n")
+    string(APPEND metametadata_file_contents "  conda_build_number: ${CONDA_BUILD_NUMBER}\n")
+    string(APPEND metametadata_file_contents "  robotology_all_packages: \n")
+    foreach(_cmake_pkg IN LISTS _superbuild_pkgs)
+      string(APPEND metametadata_file_contents "    - name: ${${_cmake_pkg}_CONDA_PKG_NAME}\n")
+      string(APPEND metametadata_file_contents "      version: \"${${_cmake_pkg}_CONDA_VERSION}\"\n")
+    endforeach()
+
+  endif()
+
   file(WRITE ${metametadata_file} ${metametadata_file_contents})
   message(STATUS "Saved metametadata in ${metametadata_file}")
 endmacro()
@@ -190,7 +210,12 @@ macro(generate_conda_recipes)
   set(python_generation_script "${CMAKE_CURRENT_SOURCE_DIR}/conda/python/generate_conda_recipes_from_metametadata.py")
   set(generated_conda_recipes_dir "${CMAKE_CURRENT_BINARY_DIR}/conda/generated_recipes")
   file(MAKE_DIRECTORY ${generated_conda_recipes_dir})
-  execute_process(COMMAND python ${python_generation_script} -i ${metametadata_file} -o ${generated_conda_recipes_dir} RESULT_VARIABLE CONDA_GENERATION_SCRIPT_RETURN_VALUE)
+  if(CONDA_GENERATE_ROBOTOLOGY_METAPACKAGES)
+    set(python_generation_script_additional_options "--generate_distro_metapackages")
+  else()
+    set(python_generation_script_additional_options "")
+  endif()
+  execute_process(COMMAND python ${python_generation_script} -i ${metametadata_file} -o ${generated_conda_recipes_dir} ${python_generation_script_additional_options} RESULT_VARIABLE CONDA_GENERATION_SCRIPT_RETURN_VALUE)
   message(STATUS "CONDA_GENERATION_SCRIPT_RETURN_VALUE: ${CONDA_GENERATION_SCRIPT_RETURN_VALUE}")
   if(CONDA_GENERATION_SCRIPT_RETURN_VALUE STREQUAL "0")
     message(STATUS "conda recipes correctly generated in ${generated_conda_recipes_dir}.")

conda/metapackages_recipes_template/robotology-distro-all.yaml

@@ -0,0 +1,16 @@
+package:
+  name: robotology-distro-all
+  version: {{ robotology_superbuild_version }}
+
+build:
+  number: {{ conda_build_number }}
+
+requirements:
+  # We use run as installing robotology-distro-all should install all robotology packages
+  run: {# List all packages and the version dependency. #}
+{% for pkg in robotology_all_packages %}    - {{ pkg.name }}={{ pkg.version.replace("v","") }}
+{% endfor %}
+
+about:
+  home: https://github.com/robotology/robotology-superbuild
+

conda/metapackages_recipes_template/robotology-distro.yaml

@@ -0,0 +1,18 @@
+package:
+  name: robotology-distro
+  version: {{ robotology_superbuild_version }}
+
+build:
+  number: {{ conda_build_number }}
+
+requirements:
+  # We used run_constrained to ensure that a specific version of 
+  # the specified package is used if the package is installed, but
+  # we do not depend explicitly on the packages
+  run_constrained: {# List all packages and the version dependency. #}
+{% for pkg in robotology_all_packages %}    - {{ pkg.name }}={{ pkg.version.replace("v","") }}
+{% endfor %}
+
+about:
+  home: https://github.com/robotology/robotology-superbuild
+

conda/python/generate_conda_recipes_from_metametadata.py

@@ -123,6 +123,7 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-i", "--metametadata", type=str, help="metametadata .yaml file")
     parser.add_argument("-o", "--recipes_dir", type=dir_path, help="directory of generated recipes directory")
+    parser.add_argument('--generate_distro_metapackages', action='store_true', help="if passed also generates the recipes for the robotology-distro and robotology-distro-all metapackages ")
     args = parser.parse_args()
 
     # Get recipe templates
@@ -170,6 +171,34 @@ def main():
             template_output = template.render(pkg_info)
             with open(os.path.join(recipe_dir, template_file), 'w') as f:
                 f.write(template_output)
-
+
+    # If requested also generate recipes for distro metapackages
+    if args.generate_distro_metapackages:
+        recipe_metapackages_template_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../metapackages_recipes_template");
+        recipe_metapackages_template_files = [f for f in os.listdir(recipe_metapackages_template_dir) if os.path.isfile(os.path.join(recipe_metapackages_template_dir, f))]
+
+        # Prepare Jinja templates
+        file_loader_metapackages = jinja2.FileSystemLoader(recipe_metapackages_template_dir)
+        jinja_env_meta = jinja2.Environment(loader=file_loader_metapackages)
+
+        for template_file_metapackage in recipe_metapackages_template_files:
+            # Extract metapackage name from the template file
+            metapackage_name = os.path.splitext(template_file_metapackage)[0]
+
+            # Prepare directory for metapackage recipe
+            recipe_metapackages_dir = os.path.realpath(args.recipes_dir + "/../generated_recipes_metapackages")
+            if not  os.path.isdir(recipe_metapackages_dir):
+                os.mkdir(recipe_metapackages_dir)
+            recipe_dir = os.path.join(recipe_metapackages_dir, metapackage_name)
+            shutil.rmtree(recipe_dir, ignore_errors=True)
+            os.mkdir(recipe_dir)
+
+    	    # Generate meta.yaml file in the created recipe directory
+            template_metapackage = jinja_env_meta.get_template(template_file_metapackage)
+            print(metametadata['conda-metapackages-metametadata'])
+            template_output = template_metapackage.render(metametadata['conda-metapackages-metametadata'])
+            with open(os.path.join(recipe_dir, "meta.yaml"), 'w') as f:
+                f.write(template_output)
+
 if __name__ == '__main__':
     main()

scripts/robotologyBumpCondaBuildNumber.sh

@@ -11,4 +11,4 @@
 
 # Inspired from https://superuser.com/questions/198143/increment-one-value-in-a-text-line-using-script
 cp ./conda/cmake/CondaGenerationOptions.cmake /tmp/CondaGenerationOptions.cmake
-awk '{if ($1 == "set(CONDA_BUILD_NUMBER") printf("%s %d)\n", $1, $2 + 1); else print $0;}' /tmp/CondaGenerationOptions.cmake > ./conda/cmake/CondaGenerationOptions.cmake
+awk '{if ($1 == "set(CONDA_BUILD_NUMBER") printf("  %s %d)\n", $1, $2 + 1); else print $0;}' /tmp/CondaGenerationOptions.cmake > ./conda/cmake/CondaGenerationOptions.cmake