From 4aab71a597a04160bd5c0a1041e014cf402f7532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 9 Aug 2023 09:33:25 +0200 Subject: [PATCH 01/11] feat: add lint that enforces every recipe to have a run_exports section --- bioconda_utils/lint/check_build_help.py | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index eae342ea85..e1ed5bb5f9 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -136,3 +136,35 @@ class cython_needs_compiler(LintCheck): def check_deps(self, deps): if 'cython' in deps and 'compiler_c' not in deps: self.message() + + +class missing_run_exports(LintCheck): + """Every recipe should have a run_export statement that ensures + that the package is automatically pinned to a compatible version if + it is used as a dependency in another recipe. + This is a conservative strategy to avoid breakaged. We came to the + conclusion that it is better to require this little overhead instead + of trying to fix things when they break later on. + This holds for compiled packages (in particular those with shared + libraries) but also for e.g. Python packages, as those might also + introduce breaking changes in their APIs or command line interfaces. + + + Add run_exports to the recipe like this:: + + build: + run_exports: + - {{ pin_subpackage('myrecipe') }} + + with ``myrecipe`` being the name of the recipe. This will by default pin the + package to ``>=x.x.x, Date: Wed, 9 Aug 2023 09:37:51 +0200 Subject: [PATCH 02/11] docs --- bioconda_utils/lint/check_build_help.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index e1ed5bb5f9..e9154cb0fe 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -160,9 +160,21 @@ class missing_run_exports(LintCheck): package to ``>=x.x.x, Date: Wed, 9 Aug 2023 09:41:26 +0200 Subject: [PATCH 03/11] add test case --- test/lint_cases.yaml | 673 ++++++++++++++++++++++--------------------- 1 file changed, 341 insertions(+), 332 deletions(-) diff --git a/test/lint_cases.yaml b/test/lint_cases.yaml index a1ec41df54..62dd2f9d61 100644 --- a/test/lint_cases.yaml +++ b/test/lint_cases.yaml @@ -12,6 +12,8 @@ setup: build: noarch: True number: 0 + run_exports: + - { { pin_subpackage('one') } } test: commands: - do nothing @@ -33,6 +35,8 @@ setup: build: noarch: True number: 0 + run_exports: + - { { pin_subpackage('one') } } test: commands: - do nothing @@ -54,336 +58,341 @@ setup: R: - version: 1.0 - tests: -- name: minimal_recipe -- name: missing_meta_yaml - move_files: - meta.yaml: '' - expect: missing_meta_yaml -- name: meta_yml - move_files: - meta.yaml: meta.yml - expect: missing_meta_yaml -- name: empty_build_section - remove: build - expect: [should_be_noarch_generic, missing_build_number] -- name: missing_home - remove: about/home - expect: missing_home -- name: missing_home_with_lint_skip - remove: about/home - add: - extra: - skip-lints: - - missing_home -- name: passing_only_package - remove: - - source - - build - - test - - about - add: - extra: - skip-lints: - - missing_home - - missing_license - - missing_build_number - - missing_summary - - missing_tests - - no_tests - - gpl_requires_license_distributed - - should_be_noarch_generic -- name: missing_home_empty_string - add: { about: { home: '' } } - expect: missing_home -- name: missing_summary - remove: about/summary - expect: missing_summary -- name: missing_summary_empty_string - add: { about: { summary: '' } } - expect: missing_summary -- name: missing_license - remove: about/license - expect: missing_license -- name: missing_license_empty_string - add: { about: { license: '' } } - expect: missing_license -- name: missing_tests - remove: test - expect: missing_tests -- name: missing_tests_empty_section - remove: - - test/commands - expect: missing_tests -- name: missing_tests_but_runtest_py - remove: test - add_files: - run_test.py: '' -- name: missing_tests_but_runtest_sh - remove: test - add_files: - run_test.sh: '' -- name: missing_tests_but_runtest_pl - remove: test - add_files: - run_test.pl: '' -- name: missing_tests_but_runtst_sh - remove: test - add_files: - run_tst.sh: '' - expect: missing_tests -- name: missing_hash - remove: source/sha256 - expect: missing_hash -- name: md5_hash - remove: source/sha256 - add: { source: { md5: abc } } -- name: sha1_hash - remove: source/sha256 - add: { source: { sha1: abc } } -- name: no_source - remove: source -- name: uses_git_url - expect: uses_vcs_url - remove: source/url - add: { source: { git_url: someurl } } -- name: uses_svn_url - expect: uses_vcs_url - remove: source/url - add: { source: { svn_url: someurl } } -- name: uses_hg_url - expect: uses_vcs_url - remove: source/url - add: { source: { hg_url: someurl } } -- name: uses_perl_threaded_in_build - add: { requirements: { build: [perl-threaded] } } - expect: uses_perl_threaded -- name: uses_perl_threaded_in_run - add: { requirements: { run: [perl-threaded] } } - expect: uses_perl_threaded -- name: uses_perl_threaded_in_host - add: { requirements: { host: [perl-threaded] } } - expect: uses_perl_threaded -- name: uses_perl_threaded_in_run_and_build - add: { requirements: { run: [perl-threaded], host: [perl-threaded] } } - expect: uses_perl_threaded -- name: uses_openjdk_in_build - add: { requirements: { build: [openjdk] } } -- name: uses_openjdk_in_build_and_run - add: { requirements: { run: [openjdk], build: [openjdk] } } -- name: uses_javajdk_in_build - add: { requirements: { build: [java-jdk] } } - expect: uses_javajdk -- name: uses_javajdk_in_run - add: { requirements: { run: [java-jdk] } } - expect: uses_javajdk -- name: uses_openjdk_in_build_and_run - add: { requirements: { run: [java-jdk], build: [java-jdk] } } - expect: uses_javajdk -- name: uses_matplotlib_in_run - add: { requirements: { run: [matplotlib] } } - expect: uses_matplotlib -- name: uses_setuptools_in_build - add: { requirements: { build: [setuptools] } } -- name: uses_setuptools_in_run - add: { requirements: { run: [setuptools] } } - expect: uses_setuptools -- name: uses_setuptools_in_run_and_build - add: { requirements: { run: [setuptools], build: [setuptools] } } - expect: uses_setuptools -- name: has_windows_bat_file - add_files: - build.bat: '' - expect: has_windows_bat_file -- name: has_windows_bat_file - add_files: - any.bat: '' - expect: has_windows_bat_file -# FIXME: should_be_noarch without python -- name: should_be_noarch_python - add: { requirements: { build: [python], run: [python] } } - expect: should_be_noarch_python -- name: should_be_noarch_python_good - add: { build: { noarch: python }, requirements: { build: [python], run: [python] } } -- name: should_be_noarch_python_good_skip - add: - build: { noarch: python, skip: false } - requirements: { host: [python], run: [python] } -- name: should_not_be_noarch_compiler - add: - build: { noarch: python } - requirements: { build: [compiler_gcc]} - expect: should_not_be_noarch_compiler -- name: should_not_be_noarch_skip - add: - build: - noarch: python - skip: True # [osx] - expect: should_not_be_noarch_skip -- name: should_not_be_noarch_source - expect: should_not_be_noarch_source - remove: source - add: - source: - - url: https://somewhere # [linux] - sha256: 123 # [linux] - - url: https://elsewhere # [osx] - sha256: 123 # [osx] -- name: selector_in_source_with_compiler - remove: - - source - - build/noarch - add: - source: - - url: https://somewhere # [linux] - sha256: 123 # [linux] - - url: https://elsewhere # [osx] - sha256: 123 # [osx] - requirements: { build: ['{{compiler("c")}}'] } -- name: noarch_java_with_python_wrapper - add: - build: { noarch: generic } - requirements: { run: [openjdk, python] } -- name: setup_py_only_install_build_sh - add_files: - build.sh: | - $PYTHON setup.py install -- name: setup_py_only_install_script - add: - build: - script: $PYTHON setup.py install -- name: setup_py_correct_build_sh - add: { requirements: { build: [setuptools] } } - add_files: - build.sh: | - $PYTHON setup install --single-version-externally-managed --report=a.txt -- name: setup_py_correct_build_sh_wrapped - add: { requirements: { build: [setuptools] } } - add_files: - build.sh: | - $PYTHON setup install \\ - --single-version-externally-managed --report=a.txt -- name: setup_py_correct_build_sh_no_setuptools - add_files: - build.sh: | - $PYTHON setup install --single-version-externally-managed --report=a.txt -- name: setup_py_correct_script - add: { requirements: { build: [setuptools] } } - add_files: - build.sh: | - $PYTHON setup install --single-version-externally-managed --report=a.txt -- name: setup_py_incorrect_script - expect: setup_py_install_args - add: - requirements: { build: [setuptools] } - build: - script: $PYTHON setup.py install -- name: setup_py_incorrect_build_sh - expect: setup_py_install_args - add: { requirements: { build: [setuptools] } } - add_files: - build.sh: | - $PYTHON setup.py install -- name: version_constraints_whitespace_ok - add: { requirements: { run: ['one >1', 'two >=1', 'three >1,<2'] } } -- name: version_constraints_whitespace_missing - expect: version_constraints_missing_whitespace - add: { requirements: { run: ['one>1'] } } -- name: extra_identifiers_ok - add: { extra: { identifiers: ['doi:123'] } } -- name: extra_identifiers_not_list - expect: extra_identifiers_not_list - add: { extra: { identifiers: 'doi:123' } } -- name: extra_identifiers_not_list - expect: extra_identifiers_not_list - add: { extra: { identifiers: { doi: 123 } } } -- name: extra_identifiers_not_string - expect: extra_identifiers_not_string - add: { extra: { identifiers: [ { doi: 123} ] } } -- name: extra_identifiers_missing_colon - expect: extra_identifiers_missing_colon - add: { extra: { identifiers: [ '123' ] } } -- name: numpy_ok - add: { requirements: { build: [numpy], run: [numpy]} } -- name: numpy_deprecated - expect: deprecated_numpy_spec - add: { requirements: { build: ['numpy x.x'], run: ['numpy x.x']} } -- name: compiler_ok - remove: build/noarch - add: { requirements: { build: ['{{compiler("c")}}'] } } -- name: compiler_old_1 - expect: should_use_compilers - add: { requirements: { build: ['gcc # [linux]'] } } -- name: compiler_old_2 - expect: should_use_compilers - add: { requirements: { build: ['libgcc # [linux]'] } } -- name: compiler_old_3 - expect: should_use_compilers - add: { requirements: { build: ['llvm # [osx]'] } } -- name: compiler_in_host - expect: compilers_must_be_in_build - remove: build/noarch - add: { requirements: { host: ['{{compiler("c")}}'] } } -- name: compiler_in_run - expect: compilers_must_be_in_build - remove: build/noarch - add: { requirements: { run: ['{{compiler("c")}}'] } } -- name: use_fn - expect: should_not_use_fn - add: { source: { fn: abc } } -- name: folder_and_package_name_mismatch - expect: folder_and_package_name_must_match - add: { package: { name: somethingelse } } -- name: gpl_without_license_file - expect: gpl_requires_license_distributed - add: { about: { license: GPLv3 } } -- name: build_number_needs_reset - expect: build_number_needs_reset - add: { build: { number: 1 } } -- name: build_number_needs_bump - expect: build_number_needs_bump - add: { package: { version: 0.0.1 } } -- name: version_starts_with_v - expect: version_starts_with_v - add: { package: { version: "v0.1" } } -- name: in_other_channels - expect: in_other_channels - repodata: { conda-forge: { one: [{version: 0.1}], two: [version: 0.1] } } -- name: long_summary - expect: long_summary - add: - about: - summary: | - 123456789012345678901234567890123456789012345678901234567890 - 123456789012345678901234567890123456789012345678901234567890 -- name: blacklisted - expect: recipe_is_blacklisted - add_root_files: - blacklist: | - recipes/one - recipes/two - config: - blacklists: - - blacklist -- name: cran_packages_to_conda_forge - expect: cran_packages_to_conda_forge - add: { requirements: { run: [R, three] } } -- name: cran_packages_to_conda_forge_bioconda_dep - add: { requirements: { run: [R, four] } } -- name: cython_in_run_no_compiler - expect: [cython_must_be_in_host, cython_needs_compiler] - add: { requirements: { run: [cython] } } -- name: cython_in_run - expect: cython_must_be_in_host - add: - requirements: { run: [cython], build: ['{{compiler("c")}}'] } - build: { noarch: False } -- name: cython_in_host_no_compiler - expect: cython_needs_compiler - add: { requirements: { host: [cython] } } -- name: cython_in_host - add: - requirements: { host: [cython], build: ['{{compiler("c")}}'] } - build: { noarch: False } - + - name: minimal_recipe + - name: missing_meta_yaml + move_files: + meta.yaml: "" + expect: missing_meta_yaml + - name: meta_yml + move_files: + meta.yaml: meta.yml + expect: missing_meta_yaml + - name: empty_build_section + remove: build + expect: [should_be_noarch_generic, missing_build_number] + - name: missing_home + remove: about/home + expect: missing_home + - name: missing_home_with_lint_skip + remove: about/home + add: + extra: + skip-lints: + - missing_home + - name: passing_only_package + remove: + - source + - build + - test + - about + add: + extra: + skip-lints: + - missing_home + - missing_license + - missing_build_number + - missing_summary + - missing_tests + - no_tests + - gpl_requires_license_distributed + - should_be_noarch_generic + - name: missing_home_empty_string + add: { about: { home: "" } } + expect: missing_home + - name: missing_summary + remove: about/summary + expect: missing_summary + - name: missing_summary_empty_string + add: { about: { summary: "" } } + expect: missing_summary + - name: missing_license + remove: about/license + expect: missing_license + - name: missing_license_empty_string + add: { about: { license: "" } } + expect: missing_license + - name: missing_tests + remove: test + expect: missing_tests + - name: missing_tests_empty_section + remove: + - test/commands + expect: missing_tests + - name: missing_tests_but_runtest_py + remove: test + add_files: + run_test.py: "" + - name: missing_tests_but_runtest_sh + remove: test + add_files: + run_test.sh: "" + - name: missing_tests_but_runtest_pl + remove: test + add_files: + run_test.pl: "" + - name: missing_tests_but_runtst_sh + remove: test + add_files: + run_tst.sh: "" + expect: missing_tests + - name: missing_hash + remove: source/sha256 + expect: missing_hash + - name: md5_hash + remove: source/sha256 + add: { source: { md5: abc } } + - name: sha1_hash + remove: source/sha256 + add: { source: { sha1: abc } } + - name: no_source + remove: source + - name: uses_git_url + expect: uses_vcs_url + remove: source/url + add: { source: { git_url: someurl } } + - name: uses_svn_url + expect: uses_vcs_url + remove: source/url + add: { source: { svn_url: someurl } } + - name: uses_hg_url + expect: uses_vcs_url + remove: source/url + add: { source: { hg_url: someurl } } + - name: uses_perl_threaded_in_build + add: { requirements: { build: [perl-threaded] } } + expect: uses_perl_threaded + - name: uses_perl_threaded_in_run + add: { requirements: { run: [perl-threaded] } } + expect: uses_perl_threaded + - name: uses_perl_threaded_in_host + add: { requirements: { host: [perl-threaded] } } + expect: uses_perl_threaded + - name: uses_perl_threaded_in_run_and_build + add: { requirements: { run: [perl-threaded], host: [perl-threaded] } } + expect: uses_perl_threaded + - name: uses_openjdk_in_build + add: { requirements: { build: [openjdk] } } + - name: uses_openjdk_in_build_and_run + add: { requirements: { run: [openjdk], build: [openjdk] } } + - name: uses_javajdk_in_build + add: { requirements: { build: [java-jdk] } } + expect: uses_javajdk + - name: uses_javajdk_in_run + add: { requirements: { run: [java-jdk] } } + expect: uses_javajdk + - name: uses_openjdk_in_build_and_run + add: { requirements: { run: [java-jdk], build: [java-jdk] } } + expect: uses_javajdk + - name: uses_matplotlib_in_run + add: { requirements: { run: [matplotlib] } } + expect: uses_matplotlib + - name: uses_setuptools_in_build + add: { requirements: { build: [setuptools] } } + - name: uses_setuptools_in_run + add: { requirements: { run: [setuptools] } } + expect: uses_setuptools + - name: uses_setuptools_in_run_and_build + add: { requirements: { run: [setuptools], build: [setuptools] } } + expect: uses_setuptools + - name: has_windows_bat_file + add_files: + build.bat: "" + expect: has_windows_bat_file + - name: has_windows_bat_file + add_files: + any.bat: "" + expect: has_windows_bat_file + # FIXME: should_be_noarch without python + - name: should_be_noarch_python + add: { requirements: { build: [python], run: [python] } } + expect: should_be_noarch_python + - name: should_be_noarch_python_good + add: + { + build: { noarch: python }, + requirements: { build: [python], run: [python] }, + } + - name: should_be_noarch_python_good_skip + add: + build: { noarch: python, skip: false } + requirements: { host: [python], run: [python] } + - name: should_not_be_noarch_compiler + add: + build: { noarch: python } + requirements: { build: [compiler_gcc] } + expect: should_not_be_noarch_compiler + - name: should_not_be_noarch_skip + add: + build: + noarch: python + skip: True # [osx] + expect: should_not_be_noarch_skip + - name: should_not_be_noarch_source + expect: should_not_be_noarch_source + remove: source + add: + source: + - url: https://somewhere # [linux] + sha256: 123 # [linux] + - url: https://elsewhere # [osx] + sha256: 123 # [osx] + - name: selector_in_source_with_compiler + remove: + - source + - build/noarch + add: + source: + - url: https://somewhere # [linux] + sha256: 123 # [linux] + - url: https://elsewhere # [osx] + sha256: 123 # [osx] + requirements: { build: ['{{compiler("c")}}'] } + - name: noarch_java_with_python_wrapper + add: + build: { noarch: generic } + requirements: { run: [openjdk, python] } + - name: setup_py_only_install_build_sh + add_files: + build.sh: | + $PYTHON setup.py install + - name: setup_py_only_install_script + add: + build: + script: $PYTHON setup.py install + - name: setup_py_correct_build_sh + add: { requirements: { build: [setuptools] } } + add_files: + build.sh: | + $PYTHON setup install --single-version-externally-managed --report=a.txt + - name: setup_py_correct_build_sh_wrapped + add: { requirements: { build: [setuptools] } } + add_files: + build.sh: | + $PYTHON setup install \\ + --single-version-externally-managed --report=a.txt + - name: setup_py_correct_build_sh_no_setuptools + add_files: + build.sh: | + $PYTHON setup install --single-version-externally-managed --report=a.txt + - name: setup_py_correct_script + add: { requirements: { build: [setuptools] } } + add_files: + build.sh: | + $PYTHON setup install --single-version-externally-managed --report=a.txt + - name: setup_py_incorrect_script + expect: setup_py_install_args + add: + requirements: { build: [setuptools] } + build: + script: $PYTHON setup.py install + - name: setup_py_incorrect_build_sh + expect: setup_py_install_args + add: { requirements: { build: [setuptools] } } + add_files: + build.sh: | + $PYTHON setup.py install + - name: version_constraints_whitespace_ok + add: { requirements: { run: ["one >1", "two >=1", "three >1,<2"] } } + - name: version_constraints_whitespace_missing + expect: version_constraints_missing_whitespace + add: { requirements: { run: ["one>1"] } } + - name: extra_identifiers_ok + add: { extra: { identifiers: ["doi:123"] } } + - name: extra_identifiers_not_list + expect: extra_identifiers_not_list + add: { extra: { identifiers: "doi:123" } } + - name: extra_identifiers_not_list + expect: extra_identifiers_not_list + add: { extra: { identifiers: { doi: 123 } } } + - name: extra_identifiers_not_string + expect: extra_identifiers_not_string + add: { extra: { identifiers: [{ doi: 123 }] } } + - name: extra_identifiers_missing_colon + expect: extra_identifiers_missing_colon + add: { extra: { identifiers: ["123"] } } + - name: numpy_ok + add: { requirements: { build: [numpy], run: [numpy] } } + - name: numpy_deprecated + expect: deprecated_numpy_spec + add: { requirements: { build: ["numpy x.x"], run: ["numpy x.x"] } } + - name: compiler_ok + remove: build/noarch + add: { requirements: { build: ['{{compiler("c")}}'] } } + - name: compiler_old_1 + expect: should_use_compilers + add: { requirements: { build: ["gcc # [linux]"] } } + - name: compiler_old_2 + expect: should_use_compilers + add: { requirements: { build: ["libgcc # [linux]"] } } + - name: compiler_old_3 + expect: should_use_compilers + add: { requirements: { build: ["llvm # [osx]"] } } + - name: compiler_in_host + expect: compilers_must_be_in_build + remove: build/noarch + add: { requirements: { host: ['{{compiler("c")}}'] } } + - name: compiler_in_run + expect: compilers_must_be_in_build + remove: build/noarch + add: { requirements: { run: ['{{compiler("c")}}'] } } + - name: use_fn + expect: should_not_use_fn + add: { source: { fn: abc } } + - name: folder_and_package_name_mismatch + expect: folder_and_package_name_must_match + add: { package: { name: somethingelse } } + - name: gpl_without_license_file + expect: gpl_requires_license_distributed + add: { about: { license: GPLv3 } } + - name: build_number_needs_reset + expect: build_number_needs_reset + add: { build: { number: 1 } } + - name: build_number_needs_bump + expect: build_number_needs_bump + add: { package: { version: 0.0.1 } } + - name: version_starts_with_v + expect: version_starts_with_v + add: { package: { version: "v0.1" } } + - name: in_other_channels + expect: in_other_channels + repodata: { conda-forge: { one: [{ version: 0.1 }], two: [version: 0.1] } } + - name: long_summary + expect: long_summary + add: + about: + summary: | + 123456789012345678901234567890123456789012345678901234567890 + 123456789012345678901234567890123456789012345678901234567890 + - name: blacklisted + expect: recipe_is_blacklisted + add_root_files: + blacklist: | + recipes/one + recipes/two + config: + blacklists: + - blacklist + - name: cran_packages_to_conda_forge + expect: cran_packages_to_conda_forge + add: { requirements: { run: [R, three] } } + - name: cran_packages_to_conda_forge_bioconda_dep + add: { requirements: { run: [R, four] } } + - name: cython_in_run_no_compiler + expect: [cython_must_be_in_host, cython_needs_compiler] + add: { requirements: { run: [cython] } } + - name: cython_in_run + expect: cython_must_be_in_host + add: + requirements: { run: [cython], build: ['{{compiler("c")}}'] } + build: { noarch: False } + - name: cython_in_host_no_compiler + expect: cython_needs_compiler + add: { requirements: { host: [cython] } } + - name: cython_in_host + add: + requirements: { host: [cython], build: ['{{compiler("c")}}'] } + build: { noarch: False } + - name: missing_run_exports + remove: build/run_exports + expect: missing_run_exports From 676f95bbb6e8f257718d80a09c5178051c7a545c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 9 Aug 2023 13:22:02 +0200 Subject: [PATCH 04/11] add dummy run export --- test/lint_cases.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/lint_cases.yaml b/test/lint_cases.yaml index 62dd2f9d61..6589b8388f 100644 --- a/test/lint_cases.yaml +++ b/test/lint_cases.yaml @@ -13,7 +13,11 @@ setup: noarch: True number: 0 run_exports: - - { { pin_subpackage('one') } } + # add a dummy run export + # (jinja templating as in reality is not + # supported by our test system to the same + # extend as in conda build) + - dummy test: commands: - do nothing @@ -36,7 +40,11 @@ setup: noarch: True number: 0 run_exports: - - { { pin_subpackage('one') } } + # add a dummy run export + # (jinja templating as in reality is not + # supported by our test system to the same + # extend as in conda build) + - dummy test: commands: - do nothing From 0f7f3d184256ff6d7e44a50fa3e698d8bcbda31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 9 Aug 2023 13:23:17 +0200 Subject: [PATCH 05/11] dbg --- test/lint_cases.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/lint_cases.yaml b/test/lint_cases.yaml index 6589b8388f..8525a36edd 100644 --- a/test/lint_cases.yaml +++ b/test/lint_cases.yaml @@ -13,11 +13,11 @@ setup: noarch: True number: 0 run_exports: - # add a dummy run export - # (jinja templating as in reality is not + # add a run export + # (note that jinja templating as in reality is not # supported by our test system to the same # extend as in conda build) - - dummy + - "{{ pin_subpackage('one') }}" test: commands: - do nothing @@ -40,11 +40,11 @@ setup: noarch: True number: 0 run_exports: - # add a dummy run export - # (jinja templating as in reality is not + # add a run export + # (note that jinja templating as in reality is not # supported by our test system to the same # extend as in conda build) - - dummy + - "{{ pin_subpackage('two') }}" test: commands: - do nothing From 6ab989f14bae65ce8bf91b6af8bd274ddcaf094c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 16 Aug 2023 17:52:59 +0200 Subject: [PATCH 06/11] fix lint implementation for case of missing build section --- bioconda_utils/lint/check_build_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index e9154cb0fe..f6407ffd8b 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -177,6 +177,6 @@ class missing_run_exports(LintCheck): has to be merged before the one updating or creating the depending recipe is created. """ def check_recipe(self, recipe): - build = recipe.meta["build"] + build = recipe.meta.get("build", dict()) if "run_exports" not in build: self.message() From fb14f7e4ff8cb0abf78c1cb576dc6d94377461ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 16 Aug 2023 20:03:35 +0200 Subject: [PATCH 07/11] add missing lint --- test/lint_cases.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lint_cases.yaml b/test/lint_cases.yaml index 8525a36edd..52c307bde4 100644 --- a/test/lint_cases.yaml +++ b/test/lint_cases.yaml @@ -78,7 +78,8 @@ tests: expect: missing_meta_yaml - name: empty_build_section remove: build - expect: [should_be_noarch_generic, missing_build_number] + expect: + [should_be_noarch_generic, missing_build_number, missing_run_exports] - name: missing_home remove: about/home expect: missing_home From 3338bb98e615ec79147f7aa5f0ca88d4ba468101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Thu, 17 Aug 2023 17:04:07 +0200 Subject: [PATCH 08/11] lint description --- bioconda_utils/cli.py | 5 ++++- bioconda_utils/lint/__init__.py | 2 +- bioconda_utils/lint/check_build_help.py | 22 +++++++++++++--------- test/lint_cases.yaml | 1 + 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 917d463fc9..b572df25ba 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -353,7 +353,10 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, messages = linter.get_messages() if messages: - print("The following problems have been found:\n") + print( + "The following problems have been found (visit https://bioconda.github.io/contributor/linting.html " + "for details on the particular lints you get below.):\n" + ) print(linter.get_report()) if not result: diff --git a/bioconda_utils/lint/__init__.py b/bioconda_utils/lint/__init__.py index c16b755575..82be133ea3 100644 --- a/bioconda_utils/lint/__init__.py +++ b/bioconda_utils/lint/__init__.py @@ -487,7 +487,7 @@ def __init__(self, config: Dict, recipe_folder: str, try: self.checks_ordered = reversed(list(nx.topological_sort(dag))) except nx.NetworkXUnfeasible: - raise RunTimeError("Cycle in LintCheck requirements!") + raise RuntimeError("Cycle in LintCheck requirements!") self.reload_checks() def reload_checks(self): diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index f6407ffd8b..1ac51929ff 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -139,8 +139,9 @@ def check_deps(self, deps): class missing_run_exports(LintCheck): - """Every recipe should have a run_export statement that ensures - that the package is automatically pinned to a compatible version if + """Recipe should have a run_export statement that ensures correct pinning in downstream packages + + This ensures that the package is automatically pinned to a compatible version if it is used as a dependency in another recipe. This is a conservative strategy to avoid breakaged. We came to the conclusion that it is better to require this little overhead instead @@ -149,21 +150,24 @@ class missing_run_exports(LintCheck): libraries) but also for e.g. Python packages, as those might also introduce breaking changes in their APIs or command line interfaces. - Add run_exports to the recipe like this:: build: run_exports: - - {{ pin_subpackage('myrecipe') }} + - {{ pin_subpackage('myrecipe', max_pin="x") }} + + with ``myrecipe`` being the name of the recipe (you can also use the name variable). + This will by default pin the package to ``>=1.2.0,<2.0.0`` where ``1.2.0`` is the + version of the package at build time of the one depending on it and ``<2.0.0`` constrains + it to be less than the next major (i.e. potentially not backward compatible) version. - with ``myrecipe`` being the name of the recipe. This will by default pin the - package to ``>=x.x.x,=1.2.0,<1.3.0`` ). + + Also check out the possible arguments of `pin_subpackage` here: https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#export-runtime-requirements Since this strategy can lead to potentially more conflicts in dependency pinnings between tools, diff --git a/test/lint_cases.yaml b/test/lint_cases.yaml index 52c307bde4..a9d882a509 100644 --- a/test/lint_cases.yaml +++ b/test/lint_cases.yaml @@ -102,6 +102,7 @@ tests: - missing_license - missing_build_number - missing_summary + - missing_run_exports - missing_tests - no_tests - gpl_requires_license_distributed From 9754513514ec9d951b2220ad5d2706a2bb0f249f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 23 Aug 2023 17:03:51 +0200 Subject: [PATCH 09/11] Update bioconda_utils/lint/check_build_help.py Co-authored-by: Christian Brueffer --- bioconda_utils/lint/check_build_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index 1ac51929ff..a03efdf524 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -143,7 +143,7 @@ class missing_run_exports(LintCheck): This ensures that the package is automatically pinned to a compatible version if it is used as a dependency in another recipe. - This is a conservative strategy to avoid breakaged. We came to the + This is a conservative strategy to avoid breakage. We came to the conclusion that it is better to require this little overhead instead of trying to fix things when they break later on. This holds for compiled packages (in particular those with shared From f44e8ec63863d3d4c6859df7dd181db83a451903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 23 Aug 2023 17:11:10 +0200 Subject: [PATCH 10/11] Update bioconda_utils/lint/check_build_help.py --- bioconda_utils/lint/check_build_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index a03efdf524..3a8fe1ce3b 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -150,7 +150,7 @@ class missing_run_exports(LintCheck): libraries) but also for e.g. Python packages, as those might also introduce breaking changes in their APIs or command line interfaces. - Add run_exports to the recipe like this:: + We distinguish between three cases. If the software follows semantic versioning (or it has at least a normal version string (like 1.2.3) and the actual strategy of the devs is unknown), add run_exports to the recipe like this:: build: run_exports: From f986cab803e95f206885deb3b09ac442d109b2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 23 Aug 2023 17:11:17 +0200 Subject: [PATCH 11/11] Update bioconda_utils/lint/check_build_help.py --- bioconda_utils/lint/check_build_help.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index 3a8fe1ce3b..a646fd1f36 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -160,6 +160,10 @@ class missing_run_exports(LintCheck): This will by default pin the package to ``>=1.2.0,<2.0.0`` where ``1.2.0`` is the version of the package at build time of the one depending on it and ``<2.0.0`` constrains it to be less than the next major (i.e. potentially not backward compatible) version. + + If the software has a normal versioning (like 1.2.3) but does reportedly not follow semantic versioning, please choose the ``max_pin`` argument such that it captures the potential next version that will introduce a breaking change. E.g. if you expect breaking changes to occur with the next minor release, choose ``max_pin="x.x"``, if they even can occur with the next patch release, choose ``max_pin="x.x.x"``. + + If the software does have a non-standard versioning (e.g. calendar versioning like 20220602), we cannot really protect well against breakages. However, we can at least pin to the current version as a minimum and skip the max_pin constraint. This works by setting ``max_pin=None``. In the recipe depending on this one, one just needs to specify the package name and no version at all.