From 79a9c1848b39992603973018f45db4eda10af806 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 13 Mar 2024 14:10:40 +0100 Subject: [PATCH] Feature/config install from pkg auto (#15748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * wip * wip * wip * wip * wip * package_id config_mode * test * wip * trying auto * wip * fix conan_server timestamp truncation * fix test * wip * wip * lockfiles * Update conan/cli/commands/config.py Co-authored-by: Francisco Ramírez * Update conans/client/graph/graph_binaries.py Co-authored-by: Francisco Ramírez * Update conans/test/functional/command/config_install_test.py Co-authored-by: Francisco Ramírez * Fix `conan profile show --format=json` for profiles with scoped confs (#15747) Fix conan profile show --format=json for profiles with scopd confs * review * fix issues and improve CMakeToolchain.preprocessor (#15756) * fix issues and improve CMakeToolchain.preprocessor * fix test * list --profile argument (#15697) * list --profile argument * filtering * fix PkgConfigDeps in build context (#15763) fix PkgConfigDeps * [CMakeToolchain]Adding visibility flag to CONAN_C_FLAGS (#15762) Adding visibility flags to C flags too * PkgConfigDeps refactor (#15771) * allow self.name/self.version for build_folder_vars (#15705) * wip * wip * wip * wip --------- Co-authored-by: Francisco Ramírez Co-authored-by: Rubén Rincón Blanco --- conan/api/subapi/config.py | 67 ++++++++++ conan/api/subapi/lockfile.py | 10 +- conan/cli/commands/config.py | 30 ++++- conan/cli/commands/lock.py | 11 +- conan/internal/cache/home_paths.py | 4 + conans/client/conf/config_installer.py | 14 +- conans/client/graph/compute_pid.py | 6 +- conans/client/graph/graph.py | 1 + conans/client/graph/graph_binaries.py | 41 +++++- conans/client/graph/graph_builder.py | 6 + conans/model/conf.py | 1 + conans/model/graph_lock.py | 18 ++- conans/model/info.py | 13 +- conans/model/pkg_type.py | 1 + .../functional/command/config_install_test.py | 121 ++++++++++++++++++ .../lockfile/test_user_overrides.py | 10 ++ .../package_id/test_config_package_id.py | 39 ++++++ 17 files changed, 369 insertions(+), 24 deletions(-) create mode 100644 conans/test/integration/package_id/test_config_package_id.py diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index f489036d1f1..d7f490abf03 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -1,3 +1,4 @@ +import json import os import platform import textwrap @@ -5,12 +6,18 @@ from jinja2 import Environment, FileSystemLoader from conan import conan_version +from conan.api.output import ConanOutput from conans.client.conf import default_settings_yml from conan.internal.api import detect_api from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp +from conans.client.graph.graph import CONTEXT_HOST, RECIPE_VIRTUAL, Node +from conans.client.graph.graph_builder import DepsGraphBuilder +from conans.client.graph.profile_node_definer import consumer_definer from conans.errors import ConanException from conans.model.conf import ConfDefinition, BUILT_IN_CONFS +from conans.model.pkg_type import PackageType +from conans.model.recipe_ref import RecipeReference from conans.model.settings import Settings from conans.util.files import load, save @@ -32,6 +39,66 @@ def install(self, path_or_url, verify_ssl, config_type=None, args=None, configuration_install(app, path_or_url, verify_ssl, config_type=config_type, args=args, source_folder=source_folder, target_folder=target_folder) + def install_pkg(self, ref, lockfile=None, force=False): + ConanOutput().warning("The 'conan config install-pkg' is experimental", + warn_tag="experimental") + conan_api = self.conan_api + remotes = conan_api.remotes.list() # ready to use remotes arguments + # Ready to use profiles as inputs, but NOT using profiles yet, empty ones + profile_host = profile_build = conan_api.profiles.get_profile([]) + + app = ConanApp(self.conan_api) + + # Computation of a very simple graph that requires "ref" + conanfile = app.loader.load_virtual(requires=[RecipeReference.loads(ref)]) + consumer_definer(conanfile, profile_build, profile_host) + root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL) + root_node.is_conf = True + update = ["*"] + builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, app.cache, remotes, + update, update, self.conan_api.config.global_conf) + deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) + + # Basic checks of the package: correct package_type and no-dependencies + deps_graph.report_graph_error() + pkg = deps_graph.root.dependencies[0].dst + ConanOutput().info(f"Configuration from package: {pkg}") + if pkg.conanfile.package_type is not PackageType.CONF: + raise ConanException(f'{pkg.conanfile} is not of package_type="configuration"') + if pkg.dependencies: + raise ConanException(f"Configuration package {pkg.ref} cannot have dependencies") + + # The computation of the "package_id" and the download of the package is done as usual + # By default we allow all remotes, and build_mode=None, always updating + conan_api.graph.analyze_binaries(deps_graph, None, remotes, update=update, lockfile=lockfile) + conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) + + # We check if this specific version is already installed + config_pref = pkg.pref.repr_notime() + config_versions = [] + config_version_file = HomePaths(conan_api.home_folder).config_version_path + if os.path.exists(config_version_file): + config_versions = json.loads(load(config_version_file)) + config_versions = config_versions["config_version"] + if config_pref in config_versions: + if force: + ConanOutput().info(f"Package '{pkg}' already configured, " + "but re-installation forced") + else: + ConanOutput().info(f"Package '{pkg}' already configured, " + "skipping configuration install") + return pkg.pref # Already installed, we can skip repeating the install + + from conans.client.conf.config_installer import configuration_install + configuration_install(app, uri=pkg.conanfile.package_folder, verify_ssl=False, + config_type="dir", ignore=["conaninfo.txt", "conanmanifest.txt"]) + # We save the current package full reference in the file for future + # And for ``package_id`` computation + config_versions = {ref.split("/", 1)[0]: ref for ref in config_versions} + config_versions[pkg.pref.ref.name] = pkg.pref.repr_notime() + save(config_version_file, json.dumps({"config_version": list(config_versions.values())})) + return pkg.pref + def get(self, name, default=None, check_type=None): return self.global_conf.get(name, default=default, check_type=check_type) diff --git a/conan/api/subapi/lockfile.py b/conan/api/subapi/lockfile.py index 7e4bf7de760..e2d51c1f985 100644 --- a/conan/api/subapi/lockfile.py +++ b/conan/api/subapi/lockfile.py @@ -79,19 +79,21 @@ def update_lockfile(lockfile, graph, lock_packages=False, clean=False): return lockfile @staticmethod - def add_lockfile(lockfile=None, requires=None, build_requires=None, python_requires=None): + def add_lockfile(lockfile=None, requires=None, build_requires=None, python_requires=None, + config_requires=None): if lockfile is None: lockfile = Lockfile() # create a new lockfile lockfile.partial = True lockfile.add(requires=requires, build_requires=build_requires, - python_requires=python_requires) + python_requires=python_requires, config_requires=config_requires) return lockfile @staticmethod - def remove_lockfile(lockfile, requires=None, build_requires=None, python_requires=None): + def remove_lockfile(lockfile, requires=None, build_requires=None, python_requires=None, + config_requires=None): lockfile.remove(requires=requires, build_requires=build_requires, - python_requires=python_requires) + python_requires=python_requires, config_requires=config_requires) return lockfile @staticmethod diff --git a/conan/cli/commands/config.py b/conan/cli/commands/config.py index 045c31643e6..6fd695f7577 100644 --- a/conan/cli/commands/config.py +++ b/conan/cli/commands/config.py @@ -1,5 +1,5 @@ from conan.api.output import cli_out_write -from conan.cli.command import conan_command, conan_subcommand +from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.formatters import default_json_formatter from conans.util.config_parser import get_bool_from_text @@ -37,13 +37,39 @@ def config_install(conan_api, parser, subparser, *args): 'specified origin') subparser.add_argument("-tf", "--target-folder", help='Install to that path in the conan cache') + args = parser.parse_args(*args) - verify_ssl = args.verify_ssl if isinstance(args.verify_ssl, bool) else get_bool_from_text(args.verify_ssl) + verify_ssl = args.verify_ssl if isinstance(args.verify_ssl, bool) \ + else get_bool_from_text(args.verify_ssl) conan_api.config.install(args.item, verify_ssl, args.type, args.args, source_folder=args.source_folder, target_folder=args.target_folder) +@conan_subcommand() +def config_install_pkg(conan_api, parser, subparser, *args): + """ + (Experimental) Install the configuration (remotes, profiles, conf), from a Conan package + """ + subparser.add_argument("item", help="Conan require") + subparser.add_argument("-l", "--lockfile", action=OnceArgument, + help="Path to a lockfile. Use --lockfile=\"\" to avoid automatic use of " + "existing 'conan.lock' file") + subparser.add_argument("--lockfile-partial", action="store_true", + help="Do not raise an error if some dependency is not found in lockfile") + subparser.add_argument("--lockfile-out", action=OnceArgument, + help="Filename of the updated lockfile") + subparser.add_argument("-f", "--force", action='store_true', + help="Force the re-installation of configuration") + args = parser.parse_args(*args) + + lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, + partial=args.lockfile_partial) + config_pref = conan_api.config.install_pkg(args.item, lockfile=lockfile, force=args.force) + lockfile = conan_api.lockfile.add_lockfile(lockfile, config_requires=[config_pref.ref]) + conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out) + + def list_text_formatter(confs): for k, v in confs.items(): cli_out_write(f"{k}: {v}") diff --git a/conan/cli/commands/lock.py b/conan/cli/commands/lock.py index 31630863a43..984661a98ae 100644 --- a/conan/cli/commands/lock.py +++ b/conan/cli/commands/lock.py @@ -98,6 +98,8 @@ def lock_add(conan_api, parser, subparser, *args): help='Add build-requires to lockfile') subparser.add_argument('--python-requires', action="append", help='Add python-requires to lockfile') + subparser.add_argument('--config-requires', action="append", + help='Add config-requires to lockfile') subparser.add_argument("--lockfile-out", action=OnceArgument, default=LOCKFILE, help="Filename of the created lockfile") subparser.add_argument("--lockfile", action=OnceArgument, help="Filename of the input lockfile") @@ -117,11 +119,13 @@ def _parse_requires(reqs): requires = _parse_requires(args.requires) build_requires = _parse_requires(args.build_requires) python_requires = _parse_requires(args.python_requires) + config_requires = _parse_requires(args.config_requires) lockfile = conan_api.lockfile.add_lockfile(lockfile, requires=requires, python_requires=python_requires, - build_requires=build_requires) + build_requires=build_requires, + config_requires=config_requires) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out) @@ -136,6 +140,8 @@ def lock_remove(conan_api, parser, subparser, *args): help='Remove build-requires from lockfile') subparser.add_argument('--python-requires', action="append", help='Remove python-requires from lockfile') + subparser.add_argument('--config-requires', action="append", + help='Remove config-requires from lockfile') subparser.add_argument("--lockfile-out", action=OnceArgument, default=LOCKFILE, help="Filename of the created lockfile") subparser.add_argument("--lockfile", action=OnceArgument, help="Filename of the input lockfile") @@ -145,5 +151,6 @@ def lock_remove(conan_api, parser, subparser, *args): lockfile = conan_api.lockfile.remove_lockfile(lockfile, requires=args.requires, python_requires=args.python_requires, - build_requires=args.build_requires) + build_requires=args.build_requires, + config_requires=args.config_requires) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out) diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index b192e64d76e..9b907c7ac90 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -74,3 +74,7 @@ def settings_path(self): @property def settings_path_user(self): return os.path.join(self._home, "settings_user.yml") + + @property + def config_version_path(self): + return os.path.join(self._home, "config_version.json") diff --git a/conans/client/conf/config_installer.py b/conans/client/conf/config_installer.py index a83cbea4c07..fc4284fd9d1 100644 --- a/conans/client/conf/config_installer.py +++ b/conans/client/conf/config_installer.py @@ -8,12 +8,12 @@ from conan.api.output import ConanOutput from conans.client.downloaders.file_downloader import FileDownloader from conans.errors import ConanException -from conans.util.files import mkdir, rmdir, remove, unzip, chdir +from conans.util.files import mkdir, rmdir, remove, unzip, chdir, load from conans.util.runners import detect_runner class _ConanIgnoreMatcher: - def __init__(self, conanignore_path): + def __init__(self, conanignore_path, ignore=None): conanignore_path = os.path.abspath(conanignore_path) self._ignored_entries = {".conanignore"} if os.path.exists(conanignore_path): @@ -22,6 +22,8 @@ def __init__(self, conanignore_path): line_content = line.strip() if line_content != "": self._ignored_entries.add(line_content) + if ignore: + self._ignored_entries.update(ignore) def matches(self, path): for ignore_entry in self._ignored_entries: @@ -112,13 +114,13 @@ def _process_file(directory, filename, config, cache, folder): _filecopy(directory, filename, target_folder) -def _process_folder(config, folder, cache): +def _process_folder(config, folder, cache, ignore=None): if not os.path.isdir(folder): raise ConanException("No such directory: '%s'" % str(folder)) if config.source_folder: folder = os.path.join(folder, config.source_folder) conanignore_path = os.path.join(folder, '.conanignore') - conanignore = _ConanIgnoreMatcher(conanignore_path) + conanignore = _ConanIgnoreMatcher(conanignore_path, ignore) for root, dirs, files in os.walk(folder): # .git is always ignored by default, even if not present in .conanignore dirs[:] = [d for d in dirs if d != ".git"] @@ -178,14 +180,14 @@ def _is_compressed_file(filename): def configuration_install(app, uri, verify_ssl, config_type=None, - args=None, source_folder=None, target_folder=None): + args=None, source_folder=None, target_folder=None, ignore=None): cache, requester = app.cache, app.requester config = _ConfigOrigin(uri, config_type, verify_ssl, args, source_folder, target_folder) try: if config.type == "git": _process_git_repo(config, cache) elif config.type == "dir": - _process_folder(config, config.uri, cache) + _process_folder(config, config.uri, cache, ignore) elif config.type == "file": if _is_compressed_file(config.uri): with tmp_config_install_folder(cache) as tmp_folder: diff --git a/conans/client/graph/compute_pid.py b/conans/client/graph/compute_pid.py index a81497bb002..d86a146f6e4 100644 --- a/conans/client/graph/compute_pid.py +++ b/conans/client/graph/compute_pid.py @@ -6,12 +6,13 @@ from conans.client.conanfile.implementations import auto_header_only_package_id -def compute_package_id(node, new_config): +def compute_package_id(node, new_config, config_version): """ Compute the binary package ID of this node """ conanfile = node.conanfile + # TODO: Extract unknown_mode = new_config.get("core.package_id:default_unknown_mode", default="semver_mode") non_embed_mode = new_config.get("core.package_id:default_non_embed_mode", default="minor_mode") # recipe_revision_mode already takes into account the package_id @@ -50,7 +51,8 @@ def compute_package_id(node, new_config): reqs_info=reqs_info, build_requires_info=build_requires_info, python_requires=python_requires, - conf=conanfile.conf.copy_conaninfo_conf()) + conf=conanfile.conf.copy_conaninfo_conf(), + config_version=config_version.copy() if config_version else None) conanfile.original_info = conanfile.info.clone() if hasattr(conanfile, "validate_build"): diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index 2481c2fadb3..63f79a41a34 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -66,6 +66,7 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False): self.cant_build = False # It will set to a str with a reason if the validate_build() fails self.should_build = False # If the --build or policy wants to build this binary self.build_allowed = False + self.is_conf = False def __lt__(self, other): """ diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 5e1274d3f8f..79c53e4c76f 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -1,3 +1,7 @@ +import json +import os +from collections import OrderedDict + from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conans.client.graph.build_mode import BuildMode @@ -10,7 +14,11 @@ BINARY_PLATFORM) from conans.client.graph.proxy import should_update_reference from conans.errors import NoRemoteAvailable, NotFoundException, \ - PackageNotFoundException, conanfile_exception_formatter, ConanConnectionError + PackageNotFoundException, conanfile_exception_formatter, ConanConnectionError, ConanException +from conans.model.info import RequirementInfo, RequirementsInfo +from conans.model.package_ref import PkgReference +from conans.model.recipe_ref import RecipeReference +from conans.util.files import load class GraphBinariesAnalyzer(object): @@ -321,8 +329,30 @@ def _evaluate_in_cache(self, cache_latest_prev, node, remotes, update): node.pref_timestamp = cache_latest_prev.timestamp assert node.prev, "PREV for %s is None" % str(node.pref) - def _evaluate_package_id(self, node): - compute_package_id(node, self._global_conf) + def _config_version(self): + config_mode = self._global_conf.get("core.package_id:config_mode", default=None) + if config_mode is None: + return + config_version_file = HomePaths(self._cache.cache_folder).config_version_path + try: + config_refs = json.loads(load(config_version_file))["config_version"] + result = OrderedDict() + for r in config_refs: + try: + config_ref = PkgReference.loads(r) + req_info = RequirementInfo(config_ref.ref, config_ref.package_id, config_mode) + except ConanException: + config_ref = RecipeReference.loads(r) + req_info = RequirementInfo(config_ref, None, config_mode) + result[config_ref] = req_info + except Exception as e: + raise ConanException(f"core.package_id:config_mode defined, but error while loading " + f"'{os.path.basename(config_version_file)}'" + f" file in cache: {self._cache.cache_folder}: {e}") + return RequirementsInfo(result) + + def _evaluate_package_id(self, node, config_version): + compute_package_id(node, self._global_conf, config_version=config_version) # TODO: layout() execution don't need to be evaluated at GraphBuilder time. # it could even be delayed until installation time, but if we got enough info here for @@ -359,9 +389,10 @@ def _evaluate_single(n): self._evaluate_node(n, mode, remotes, update) levels = deps_graph.by_levels() + config_version = self._config_version() for level in levels[:-1]: # all levels but the last one, which is the single consumer for node in level: - self._evaluate_package_id(node) + self._evaluate_package_id(node, config_version) # group by pref to paralelize, so evaluation is done only 1 per pref nodes = {} for node in level: @@ -382,7 +413,7 @@ def _evaluate_single(n): if node.path is not None: if node.path.endswith(".py"): # For .py we keep evaluating the package_id, validate(), etc - compute_package_id(node, self._global_conf) + compute_package_id(node, self._global_conf, config_version=config_version) # To support the ``[layout]`` in conanfile.txt if hasattr(node.conanfile, "layout"): with conanfile_exception_formatter(node.conanfile, "layout"): diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index 1320172fe00..e190fd1e61d 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -14,6 +14,7 @@ from conans.errors import ConanException from conans.model.conan_file import ConanFile from conans.model.options import Options +from conans.model.pkg_type import PackageType from conans.model.recipe_ref import RecipeReference, ref_matches from conans.model.requires import Requirement @@ -318,6 +319,7 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr raise GraphMissingError(node, require, str(e)) layout, dep_conanfile, recipe_status, remote = resolved + new_ref = layout.reference dep_conanfile.folders.set_base_recipe_metadata(layout.metadata()) # None for platform_xxx # If the node is virtual or a test package, the require is also "root" @@ -355,6 +357,10 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr if recipe_status != RECIPE_PLATFORM: self._prepare_node(new_node, profile_host, profile_build, down_options) + if dep_conanfile.package_type is PackageType.CONF and node.recipe != RECIPE_VIRTUAL: + raise ConanException(f"Configuration package {dep_conanfile} cannot be used as " + f"requirement, but {node.ref} is requiring it") + require.process_package_type(node, new_node) graph.add_node(new_node) graph.add_edge(node, new_node, require) diff --git a/conans/model/conf.py b/conans/model/conf.py index 520c73b8f5f..d35ec8fa752 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -39,6 +39,7 @@ "core.package_id:default_embed_mode": "By default, 'full_mode'", "core.package_id:default_python_mode": "By default, 'minor_mode'", "core.package_id:default_build_mode": "By default, 'None'", + "core.package_id:config_mode": "How the 'config_version' affects binaries. By default 'None'", # General HTTP(python-requests) configuration "core.net.http:max_retries": "Maximum number of connection retries (requests library)", "core.net.http:timeout": "Number of seconds without response to timeout (requests library)", diff --git a/conans/model/graph_lock.py b/conans/model/graph_lock.py index d69979f3b55..2ac43ac58c3 100644 --- a/conans/model/graph_lock.py +++ b/conans/model/graph_lock.py @@ -106,6 +106,7 @@ def __init__(self, deps_graph=None, lock_packages=False): self._requires = _LockRequires() self._python_requires = _LockRequires() self._build_requires = _LockRequires() + self._conf_requires = _LockRequires() self._alias = {} self._overrides = Overrides() self.partial = False @@ -138,6 +139,7 @@ def update_lock(self, deps_graph, lock_packages=False): self._requires.sort() self._build_requires.sort() self._python_requires.sort() + self._conf_requires.sort() @staticmethod def load(path): @@ -168,10 +170,11 @@ def merge(self, other): self._requires.merge(other._requires) self._build_requires.merge(other._build_requires) self._python_requires.merge(other._python_requires) + self._conf_requires.merge(other._conf_requires) self._alias.update(other._alias) self._overrides.update(other._overrides) - def add(self, requires=None, build_requires=None, python_requires=None): + def add(self, requires=None, build_requires=None, python_requires=None, config_requires=None): """ adding new things manually will trigger the sort() of the locked list, so lockfiles alwasys keep the ordered lists. This means that for some especial edge cases it might be necessary to allow removing from a lockfile, for example to test an older version @@ -189,8 +192,12 @@ def add(self, requires=None, build_requires=None, python_requires=None): for r in python_requires: self._python_requires.add(r) self._python_requires.sort() + if config_requires: + for r in config_requires: + self._conf_requires.add(r) + self._conf_requires.sort() - def remove(self, requires=None, build_requires=None, python_requires=None): + def remove(self, requires=None, build_requires=None, python_requires=None, config_requires=None): def _remove(reqs, self_reqs, name): if reqs: removed = [] @@ -202,6 +209,7 @@ def _remove(reqs, self_reqs, name): _remove(requires, self._requires, "require") _remove(build_requires, self._build_requires, "build_require") _remove(python_requires, self._python_requires, "python_require") + _remove(config_requires, self._conf_requires, "config_requires") @staticmethod def deserialize(data): @@ -223,6 +231,8 @@ def deserialize(data): for k, v in data["alias"].items()} if "overrides" in data: graph_lock._overrides = Overrides.deserialize(data["overrides"]) + if "config_requires" in data: + graph_lock._conf_requires = _LockRequires.deserialize(data["config_requires"]) return graph_lock def serialize(self): @@ -240,11 +250,15 @@ def serialize(self): result["alias"] = {repr(k): repr(v) for k, v in self._alias.items()} if self._overrides: result["overrides"] = self._overrides.serialize() + if self._conf_requires: + result["config_requires"] = self._conf_requires.serialize() return result def resolve_locked(self, node, require, resolve_prereleases): if require.build or node.context == CONTEXT_BUILD: locked_refs = self._build_requires.refs() + elif node.is_conf: + locked_refs = self._conf_requires.refs() else: locked_refs = self._requires.refs() self._resolve_overrides(require) diff --git a/conans/model/info.py b/conans/model/info.py index 552364dea6d..95e7862f5f4 100644 --- a/conans/model/info.py +++ b/conans/model/info.py @@ -309,7 +309,7 @@ def load_binary_info(text): class ConanInfo: def __init__(self, settings=None, options=None, reqs_info=None, build_requires_info=None, - python_requires=None, conf=None): + python_requires=None, conf=None, config_version=None): self.invalid = None self.settings = settings self.settings_target = None # needs to be explicitly defined by recipe package_id() @@ -318,6 +318,7 @@ def __init__(self, settings=None, options=None, reqs_info=None, build_requires_i self.build_requires = build_requires_info self.python_requires = python_requires self.conf = conf + self.config_version = config_version def clone(self): """ Useful for build_id implementation and for compatibility() @@ -331,6 +332,7 @@ def clone(self): result.python_requires = self.python_requires.copy() result.conf = self.conf.copy() result.settings_target = self.settings_target.copy() if self.settings_target else None + result.config_version = self.config_version.copy() if self.config_version else None return result def serialize(self): @@ -357,6 +359,9 @@ def serialize(self): conf_dumps = self.conf.serialize() if conf_dumps: result["conf"] = conf_dumps + config_version_dumps = self.config_version.serialize() if self.config_version else None + if config_version_dumps: + result["config_version"] = config_version_dumps return result def dumps(self): @@ -398,6 +403,10 @@ def dumps(self): # TODO: Think about the serialization of Conf, not 100% sure if dumps() is the best result.append("[conf]") result.append(self.conf.dumps()) + config_version_dumps = self.config_version.dumps() if self.config_version else None + if config_version_dumps: + result.append("[config_version]") + result.append(config_version_dumps) result.append("") # Append endline so file ends with LF return '\n'.join(result) @@ -427,6 +436,8 @@ def clear(self): self.conf.clear() self.build_requires.clear() self.python_requires.clear() + if self.config_version is not None: + self.config_version.clear() def validate(self): # If the options are not fully defined, this is also an invalid case diff --git a/conans/model/pkg_type.py b/conans/model/pkg_type.py index 0134e3aa46f..5d51d0488a3 100644 --- a/conans/model/pkg_type.py +++ b/conans/model/pkg_type.py @@ -11,6 +11,7 @@ class PackageType(Enum): BUILD_SCRIPTS = "build-scripts" APP = "application" PYTHON = "python-require" + CONF = "configuration" UNKNOWN = "unknown" def __str__(self): diff --git a/conans/test/functional/command/config_install_test.py b/conans/test/functional/command/config_install_test.py index b8ef676018b..0d1e8ad2e04 100644 --- a/conans/test/functional/command/config_install_test.py +++ b/conans/test/functional/command/config_install_test.py @@ -543,6 +543,7 @@ class ConfigInstallSchedTest(unittest.TestCase): def setUp(self): self.folder = temp_folder(path_with_spaces=False) + # FIXME: This is broken, this config no longer exist, but it doesn't raise, fix it save_files(self.folder, {"global.conf": "core:config_install_interval=5m"}) self.client = TestClient() self.client.save({"conanfile.txt": ""}) @@ -616,3 +617,123 @@ def test_config_install_reestructuring_source(self): client.save({"profiles/debug/address-sanitizer": ""}) client.run("config install .") assert os.path.isdir(debug_cache_folder) + + +class TestConfigInstallPkg: + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + class Conf(ConanFile): + name = "myconf" + version = "0.1" + package_type = "configuration" + def package(self): + copy(self, "*.conf", src=self.build_folder, dst=self.package_folder) + """) + + @pytest.fixture() + def client(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": self.conanfile, + "global.conf": "user.myteam:myconf=myvalue"}) + c.run("export-pkg .") + c.run("upload * -r=default -c") + c.run("remove * -c") + return c + + def test_config_install_from_pkg(self, client): + # Now install it + c = client + c.run("config install-pkg myconf/[*]") + assert "myconf/0.1: Downloaded package revision" in c.out + assert "Copying file global.conf" in c.out + c.run("config show *") + assert "user.myteam:myconf: myvalue" in c.out + + # Just to make sure it doesn't crash in the update + c.run("config install-pkg myconf/[*]") + # Conan will not re-download fromthe server the same revision + assert "myconf/0.1: Downloaded package revision" not in c.out + # It doesn't re-install either + assert "Copying file global.conf" not in c.out + c.run("config show *") + assert "user.myteam:myconf: myvalue" in c.out + + # We can force the re-installation + c.run("config install-pkg myconf/[*] --force") + assert "Copying file global.conf" in c.out + c.run("config show *") + assert "user.myteam:myconf: myvalue" in c.out + + def test_update_flow(self, client): + # Now try the update flow + c = client + c2 = TestClient(servers=c.servers, inputs=["admin", "password"]) + c2.save({"conanfile.py": self.conanfile, + "global.conf": "user.myteam:myconf=othervalue"}) + c2.run("export-pkg .") + c2.run("upload * -r=default -c") + + c.run("config install-pkg myconf/[*]") + assert "myconf/0.1: Downloaded package revision" in c.out + c.run("config show *") + assert "user.myteam:myconf: othervalue" in c.out + + def test_cant_use_as_dependency(self): + c = TestClient() + conanfile = GenConanfile("myconf", "0.1").with_package_type("configuration") + c.save({"myconf/conanfile.py": conanfile, + "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_requires("myconf/0.1")}) + c.run("create myconf") + c.run("install pkg", assert_error=True) + assert "ERROR: Configuration package myconf/0.1 cannot be used as requirement, " \ + "but pkg/0.1 is requiring it" in c.out + + def test_cant_use_without_type(self): + c = TestClient() + conanfile = GenConanfile("myconf", "0.1") + c.save({"myconf/conanfile.py": conanfile}) + c.run("create myconf") + c.run("config install-pkg myconf/[*]", assert_error=True) + assert 'ERROR: myconf/0.1 is not of package_type="configuration"' in c.out + + def test_lockfile(self, client): + """ it should be able to install the config using a lockfile + """ + c = client + c.run("config install-pkg myconf/[*] --lockfile-out=config.lock") + + c2 = TestClient(servers=c.servers, inputs=["admin", "password"]) + # Make sure we bump the version, otherwise only a package revision will be created + c2.save({"conanfile.py": self.conanfile.replace("0.1", "0.2"), + "global.conf": "user.myteam:myconf=othervalue"}) + c2.run("export-pkg .") + c2.run("upload * -r=default -c") + + c.run("config install-pkg myconf/[*] --lockfile=config.lock") + c.run("config show *") + assert "user.myteam:myconf: myvalue" in c.out + + def test_create_also(self): + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + class Conf(ConanFile): + name = "myconf" + version = "0.1" + package_type = "configuration" + exports_sources = "*.conf" + def package(self): + copy(self, "*.conf", src=self.build_folder, dst=self.package_folder) + """) + + c = TestClient(default_server_user=True) + c.save({"conanfile.py": conanfile, + "global.conf": "user.myteam:myconf=myvalue"}) + c.run("create .") + c.run("upload * -r=default -c") + c.run("remove * -c") + + c.run("config install-pkg myconf/[*]") + c.run("config show *") + assert "user.myteam:myconf: myvalue" in c.out diff --git a/conans/test/integration/lockfile/test_user_overrides.py b/conans/test/integration/lockfile/test_user_overrides.py index 53648a5c6c2..b861b1233d8 100644 --- a/conans/test/integration/lockfile/test_user_overrides.py +++ b/conans/test/integration/lockfile/test_user_overrides.py @@ -109,6 +109,16 @@ def test_user_python_overrides(): assert "pytool/1.1" in new_lock +def test_config_overrides(): + """ Test that it is possible to lock also config-requires + """ + c = TestClient() + c.run("lock add --config-requires=config/1.0") + assert json.loads(c.load("conan.lock"))["config_requires"] == ["config/1.0"] + c.run("lock remove --config-requires=config/1.0") + assert json.loads(c.load("conan.lock"))["config_requires"] == [] + + def test_add_revisions(): """ Is it possible to add revisions explicitly too """ diff --git a/conans/test/integration/package_id/test_config_package_id.py b/conans/test/integration/package_id/test_config_package_id.py new file mode 100644 index 00000000000..ec077e65254 --- /dev/null +++ b/conans/test/integration/package_id/test_config_package_id.py @@ -0,0 +1,39 @@ +import json + +import pytest + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.tools import TestClient +from conans.util.files import save + + +@pytest.mark.parametrize("config_version, mode, result", [ + ("myconfig/1.2.3#rev1:pid1#prev1", "minor_mode", "myconfig/1.2.Z"), + ("myconfig/1.2.3#rev1:pid1#prev1", "patch_mode", "myconfig/1.2.3"), + ("myconfig/1.2.3#rev1:pid1#prev1", "full_mode", "myconfig/1.2.3#rev1:pid1"), + ("myconfig/1.2.3", "minor_mode", "myconfig/1.2.Z")]) +def test_config_package_id(config_version, mode, result): + c = TestClient() + config_version = json.dumps({"config_version": [config_version]}) + save(c.cache.config_version_path, config_version) + save(c.cache.global_conf_path, f"core.package_id:config_mode={mode}") + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .") + c.run("list pkg/0.1:* --format=json") + info = json.loads(c.stdout) + rrev = info["Local Cache"]["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"] + package_id = {"myconfig/1.2.Z": "c78b4d8224154390356fe04fe598d67aec930199", + "myconfig/1.2.3": "60005f5b11bef3ddd686b13f5c6bf576a9b882b8", + "myconfig/1.2.3#rev1:pid1": "b1525975eb5420cef45b4ddd1544f87c29c773a5"} + package_id = package_id.get(result) + pkg = rrev["packages"][package_id] + assert pkg["info"] == {"config_version": [result]} + + +def test_error_config_package_id(): + c = TestClient() + save(c.cache.global_conf_path, f"core.package_id:config_mode=minor_mode") + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .", assert_error=True) + assert "ERROR: core.package_id:config_mode defined, " \ + "but error while loading 'config_version.json'" in c.out