diff --git a/.github/workflows/generate-conda-packages.yaml b/.github/workflows/generate-conda-packages.yaml index 55277d17a..a38f38fd8 100644 --- a/.github/workflows/generate-conda-packages.yaml +++ b/.github/workflows/generate-conda-packages.yaml @@ -12,11 +12,39 @@ on: schedule: # Run the job once a week - cron: '0 0 * * 2' + release: + types: [published] jobs: + # Regardless of the branch on which this action is trigged, + # the CONDA_BUILD_NUMBER CMake option needs to be read from the + # master branch, to avoid that a conda package generation from the + # master branch and one from a releases/YYYY.MM branch use the same + # CONDA_BUILD_NUMBER + get-conda-build-number: + name: "Read Conda Build number from master branch" + runs-on: ubuntu-latest + # Define outputs (see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-defining-outputs-for-a-job) + outputs: + conda_build_number: ${{ steps.step1.outputs.conda_build_number }} + + steps: + - uses: actions/checkout@v2 + with: + ref: 'master' + + - id: step1 + name: Get CONDA_BUILD_NUMBER and set it as output + shell: bash + run: | + # Get CONDA_BUILD_NUMBER via grep and set it to an environment variable + export CONDA_BUILD_NUMBER=`grep "CONDA_BUILD_NUMBER" ./conda/cmake/CondaGenerationOptions.cmake | grep -oe '\([0-9.]*\)'` + echo "::set-output name=conda_build_number::${CONDA_BUILD_NUMBER}" + generate-conda-packages: name: "Generate conda packages @${{ matrix.os }}" runs-on: ${{ matrix.os }} + needs: get-conda-build-number strategy: fail-fast: false matrix: @@ -88,7 +116,7 @@ jobs: run: | mkdir build cd build - cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON .. + cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} .. - name: Generate recipes [Windows] if: contains(matrix.os, 'windows') @@ -96,7 +124,14 @@ jobs: run: | mkdir build cd build - cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON .. + cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} .. + + - name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages + if: github.event_name == 'release' + shell: bash -l {0} + run: | + cd build + cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON . - name: Build conda packages shell: bash -l {0} @@ -155,12 +190,22 @@ jobs: rm -rf blocktest conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml human-dynamics-estimation rm -rf human-dynamics-estimation - conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . + conda build -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . + + - name: Build conda metapackages (if necessary + if: github.event_name == 'release' + shell: bash -l {0} + run: | + cd build/conda/generated_recipes_metapackages + # Debug generated recipes + cat */meta.yaml + conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro + conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all - name: Upload conda packages shell: bash -l {0} # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true' - if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') + if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') env: ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} run: | @@ -170,22 +215,26 @@ jobs: # If the generate-conda-packages completed correctly and binaries are uploaded, # bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake - # for future builds + # of master branch for future builds + # the master branch is always used in case the action is triggered by a release on + # a release/vYYYY.MM branch bump-conda-build-number: name: "Bump Conda Build number for future builds" runs-on: ubuntu-latest needs: generate-conda-packages - if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') + if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') steps: - uses: actions/checkout@v2 + with: + ref: 'master' - name: Bump Conda Build number for future builds shell: bash run: | sh ./scripts/robotologyBumpCondaBuildNumber.sh - - uses: EndBug/add-and-commit@v7.0.0 + - uses: EndBug/add-and-commit@v8.0.1 with: default_author: github_actions message: 'Bump CONDA_BUILD_NUMBER after successful Conda packages build and upload' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f52a0b6c1..64042a181 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,9 @@ jobs: sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION ${{ github.event.inputs.superbuild_year_month_prefix }}.0)/g' ./packaging/windows/CMakeLists.txt git add . git commit -m "Updating packaging/windows/CMakeLists.txt" + sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION ${{ github.event.inputs.superbuild_year_month_prefix }}.0)/g' ./conda/cmake/CondaGenerationOptions.cmake + git add . + git commit -m "Updating conda/cmake/CondaGenerationOptions.cmake" git push --set-upstream origin releases/${{ github.event.inputs.superbuild_year_month_prefix }} - name: Create and push the ${{ github.event.inputs.superbuild_year_month_prefix }}.0 tag diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cdc3d5c3..fb0e679d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format of this document is based on [Keep a Changelog](https://keepachangelo ### Added - Added dependency on `graphviz` to compile `yarpviz` YARP tool (https://github.com/robotology/robotology-superbuild/pull/988). +- Added generation of `robotology-distro` and `robotology-distro-all` conda metapackages on releases (https://github.com/robotology/robotology-superbuild/pull/1030). ### Changed - On Windows the option `ROBOTOLOGY_USES_ESDCAN` is now enabled when generating conda packages (https://github.com/robotology/robotology-superbuild/pull/935). diff --git a/cmake/Buildcasadi.cmake b/cmake/Buildcasadi.cmake index 39d82fe2f..0bf4be48f 100644 --- a/cmake/Buildcasadi.cmake +++ b/cmake/Buildcasadi.cmake @@ -35,3 +35,9 @@ ycm_ep_helper(casadi TYPE GIT set(casadi_CONDA_PKG_NAME casadi) set(casadi_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) +# This is a small hack. To avoid incompatibilities between the version tagged in the ami-iit fork +# (something like 3.5.5.x) and the version available in conda-forge when generating conda metapackages +# such as robotology-distro and robotology-distro-all, we override the conda package version of casadi +# here. This needs to be removed as soon as we stop use our fork in the superbuild +set(casadi_CONDA_VERSION 3.5.5) + diff --git a/cmake/Buildmanif.cmake b/cmake/Buildmanif.cmake index 4b9d61e80..d8067eba2 100644 --- a/cmake/Buildmanif.cmake +++ b/cmake/Buildmanif.cmake @@ -21,3 +21,8 @@ ycm_ep_helper(manif TYPE GIT set(manif_CONDA_PKG_NAME manif) set(manif_CONDA_PKG_CONDA_FORGE_OVERRIDE ON) +# This is a small hack. To avoid incompatibilities between the version tagged in the ami-iit fork +# (something like 0.0.4.x) and the version available in conda-forge when generating conda metapackages +# such as robotology-distro and robotology-distro-all, we override the conda package version of manif +# here. This needs to be removed as soon as we stop using our fork in the superbuild +set(manif_CONDA_VERSION 0.0.4) diff --git a/conda/cmake/CondaGenerationOptions.cmake b/conda/cmake/CondaGenerationOptions.cmake index d632f5f5c..be2c6ab31 100644 --- a/conda/cmake/CondaGenerationOptions.cmake +++ b/conda/cmake/CondaGenerationOptions.cmake @@ -1,7 +1,18 @@ # This number needs to be increased at each full rebuild, # to ensure that binaries belonging to different rebuilds # can be distinguished even if the version number is the same -set(CONDA_BUILD_NUMBER 43) +if(NOT CONDA_BUILD_NUMBER) + set(CONDA_BUILD_NUMBER 43) +endif() + +# This option is enabled only when the metapackages robotology-distro and robotology-distro-all are +# generated, so only in case a robotology-superbuild distro is released +option(CONDA_GENERATE_ROBOTOLOGY_METAPACKAGES "If on, generate recipes for robotology-distro and robotology-distro-all conda metapackages." OFF) + +# This variable is automatically set to contain the robotology-superbuild version in case of releases +if(NOT CONDA_ROBOTOLOGY_SUPERBUILD_VERSION) + set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "") +endif() # For more conda generation options, check the specific project Build.cmake # file for variables that start with `_CONDA` diff --git a/conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake b/conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake index 6ba2895bb..032505239 100644 --- a/conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake +++ b/conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake @@ -60,15 +60,10 @@ macro(generate_metametadata_file) get_property(_superbuild_pkgs GLOBAL PROPERTY YCM_PROJECTS) foreach(_cmake_pkg IN LISTS _superbuild_pkgs) - # If a package is already available in conda-forge, we use - # that one by defining appropriately the <_cmake_pkg>_CONDA_PACKAGE_NAME - # and <_cmake_pkg>_CONDA_PKG_CONDA_FORGE_OVERRIDE variables - if(DEFINED ${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE AND - "${${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE}") - continue() - endif() - # Compute conda version + # We do it for all packages as this is also necessary + # for packages for which <_cmake_pkg>_CONDA_PKG_CONDA_FORGE_OVERRIDE + # is defined when generating the metapackages if(DEFINED ${_cmake_pkg}_TAG) set(${_cmake_pkg}_CONDA_TAG ${${_cmake_pkg}_TAG}) else() @@ -79,6 +74,15 @@ macro(generate_metametadata_file) set(${_cmake_pkg}_CONDA_VERSION ${${_cmake_pkg}_CONDA_TAG}) endif() + + # If a package is already available in conda-forge, we use + # that one by defining appropriately the <_cmake_pkg>_CONDA_PACKAGE_NAME + # and <_cmake_pkg>_CONDA_PKG_CONDA_FORGE_OVERRIDE variables + if(DEFINED ${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE AND + "${${_cmake_pkg}_CONDA_PKG_CONDA_FORGE_OVERRIDE}") + continue() + endif() + # Compute conda CMake options set(${_cmake_pkg}_CONDA_CMAKE_ARGS ${_YH_${_cmake_pkg}_CMAKE_ARGS}) list(APPEND ${_cmake_pkg}_CONDA_CMAKE_ARGS ${_YH_${_cmake_pkg}_CMAKE_CACHE_ARGS}) @@ -178,10 +182,26 @@ macro(generate_metametadata_file) string(APPEND metametadata_file_contents " add_numpy_runtime_dep: true\n") endif() - + string(APPEND metametadata_file_contents "\n") + string(APPEND metametadata_file_contents "\n") endforeach() + # If we generate robotology-distro and robotology-distro-all metapackages, we need also to add the + # conda-metapackages-metametadata: section that will be used to generate the metapackages recipes + # To the people from the future: I am really sorry about the amount of "meta" in these names + if(CONDA_GENERATE_ROBOTOLOGY_METAPACKAGES) + string(APPEND metametadata_file_contents "conda-metapackages-metametadata:\n") + string(APPEND metametadata_file_contents " robotology_superbuild_version: ${CONDA_ROBOTOLOGY_SUPERBUILD_VERSION}\n") + string(APPEND metametadata_file_contents " conda_build_number: ${CONDA_BUILD_NUMBER}\n") + string(APPEND metametadata_file_contents " robotology_all_packages: \n") + foreach(_cmake_pkg IN LISTS _superbuild_pkgs) + string(APPEND metametadata_file_contents " - name: ${${_cmake_pkg}_CONDA_PKG_NAME}\n") + string(APPEND metametadata_file_contents " version: \"${${_cmake_pkg}_CONDA_VERSION}\"\n") + endforeach() + + endif() + file(WRITE ${metametadata_file} ${metametadata_file_contents}) message(STATUS "Saved metametadata in ${metametadata_file}") endmacro() @@ -190,7 +210,12 @@ macro(generate_conda_recipes) set(python_generation_script "${CMAKE_CURRENT_SOURCE_DIR}/conda/python/generate_conda_recipes_from_metametadata.py") set(generated_conda_recipes_dir "${CMAKE_CURRENT_BINARY_DIR}/conda/generated_recipes") file(MAKE_DIRECTORY ${generated_conda_recipes_dir}) - execute_process(COMMAND python ${python_generation_script} -i ${metametadata_file} -o ${generated_conda_recipes_dir} RESULT_VARIABLE CONDA_GENERATION_SCRIPT_RETURN_VALUE) + if(CONDA_GENERATE_ROBOTOLOGY_METAPACKAGES) + set(python_generation_script_additional_options "--generate_distro_metapackages") + else() + set(python_generation_script_additional_options "") + endif() + execute_process(COMMAND python ${python_generation_script} -i ${metametadata_file} -o ${generated_conda_recipes_dir} ${python_generation_script_additional_options} RESULT_VARIABLE CONDA_GENERATION_SCRIPT_RETURN_VALUE) message(STATUS "CONDA_GENERATION_SCRIPT_RETURN_VALUE: ${CONDA_GENERATION_SCRIPT_RETURN_VALUE}") if(CONDA_GENERATION_SCRIPT_RETURN_VALUE STREQUAL "0") message(STATUS "conda recipes correctly generated in ${generated_conda_recipes_dir}.") diff --git a/conda/metapackages_recipes_template/robotology-distro-all.yaml b/conda/metapackages_recipes_template/robotology-distro-all.yaml new file mode 100644 index 000000000..0070e1e13 --- /dev/null +++ b/conda/metapackages_recipes_template/robotology-distro-all.yaml @@ -0,0 +1,16 @@ +package: + name: robotology-distro-all + version: {{ robotology_superbuild_version }} + +build: + number: {{ conda_build_number }} + +requirements: + # We use run as installing robotology-distro-all should install all robotology packages + run: {# List all packages and the version dependency. #} +{% for pkg in robotology_all_packages %} - {{ pkg.name }}={{ pkg.version.replace("v","") }} +{% endfor %} + +about: + home: https://github.com/robotology/robotology-superbuild + diff --git a/conda/metapackages_recipes_template/robotology-distro.yaml b/conda/metapackages_recipes_template/robotology-distro.yaml new file mode 100644 index 000000000..877f2d301 --- /dev/null +++ b/conda/metapackages_recipes_template/robotology-distro.yaml @@ -0,0 +1,18 @@ +package: + name: robotology-distro + version: {{ robotology_superbuild_version }} + +build: + number: {{ conda_build_number }} + +requirements: + # We used run_constrained to ensure that a specific version of + # the specified package is used if the package is installed, but + # we do not depend explicitly on the packages + run_constrained: {# List all packages and the version dependency. #} +{% for pkg in robotology_all_packages %} - {{ pkg.name }}={{ pkg.version.replace("v","") }} +{% endfor %} + +about: + home: https://github.com/robotology/robotology-superbuild + diff --git a/conda/python/generate_conda_recipes_from_metametadata.py b/conda/python/generate_conda_recipes_from_metametadata.py index af8a258aa..dc6c62a3f 100644 --- a/conda/python/generate_conda_recipes_from_metametadata.py +++ b/conda/python/generate_conda_recipes_from_metametadata.py @@ -123,6 +123,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("-i", "--metametadata", type=str, help="metametadata .yaml file") parser.add_argument("-o", "--recipes_dir", type=dir_path, help="directory of generated recipes directory") + parser.add_argument('--generate_distro_metapackages', action='store_true', help="if passed also generates the recipes for the robotology-distro and robotology-distro-all metapackages ") args = parser.parse_args() # Get recipe templates @@ -170,6 +171,34 @@ def main(): template_output = template.render(pkg_info) with open(os.path.join(recipe_dir, template_file), 'w') as f: f.write(template_output) - + + # If requested also generate recipes for distro metapackages + if args.generate_distro_metapackages: + recipe_metapackages_template_dir = os.path.realpath(os.path.dirname(os.path.abspath(__file__)) + "/../metapackages_recipes_template"); + recipe_metapackages_template_files = [f for f in os.listdir(recipe_metapackages_template_dir) if os.path.isfile(os.path.join(recipe_metapackages_template_dir, f))] + + # Prepare Jinja templates + file_loader_metapackages = jinja2.FileSystemLoader(recipe_metapackages_template_dir) + jinja_env_meta = jinja2.Environment(loader=file_loader_metapackages) + + for template_file_metapackage in recipe_metapackages_template_files: + # Extract metapackage name from the template file + metapackage_name = os.path.splitext(template_file_metapackage)[0] + + # Prepare directory for metapackage recipe + recipe_metapackages_dir = os.path.realpath(args.recipes_dir + "/../generated_recipes_metapackages") + if not os.path.isdir(recipe_metapackages_dir): + os.mkdir(recipe_metapackages_dir) + recipe_dir = os.path.join(recipe_metapackages_dir, metapackage_name) + shutil.rmtree(recipe_dir, ignore_errors=True) + os.mkdir(recipe_dir) + + # Generate meta.yaml file in the created recipe directory + template_metapackage = jinja_env_meta.get_template(template_file_metapackage) + print(metametadata['conda-metapackages-metametadata']) + template_output = template_metapackage.render(metametadata['conda-metapackages-metametadata']) + with open(os.path.join(recipe_dir, "meta.yaml"), 'w') as f: + f.write(template_output) + if __name__ == '__main__': main() diff --git a/scripts/robotologyBumpCondaBuildNumber.sh b/scripts/robotologyBumpCondaBuildNumber.sh index 9c0f45769..2f6c5b0d4 100755 --- a/scripts/robotologyBumpCondaBuildNumber.sh +++ b/scripts/robotologyBumpCondaBuildNumber.sh @@ -11,4 +11,4 @@ # Inspired from https://superuser.com/questions/198143/increment-one-value-in-a-text-line-using-script cp ./conda/cmake/CondaGenerationOptions.cmake /tmp/CondaGenerationOptions.cmake -awk '{if ($1 == "set(CONDA_BUILD_NUMBER") printf("%s %d)\n", $1, $2 + 1); else print $0;}' /tmp/CondaGenerationOptions.cmake > ./conda/cmake/CondaGenerationOptions.cmake \ No newline at end of file +awk '{if ($1 == "set(CONDA_BUILD_NUMBER") printf(" %s %d)\n", $1, $2 + 1); else print $0;}' /tmp/CondaGenerationOptions.cmake > ./conda/cmake/CondaGenerationOptions.cmake \ No newline at end of file