Feature/config install from pkg auto (#15748)

* wip

* wip

* wip

* wip

* wip

* wip

* package_id config_mode

* test

* wip

* trying auto

* wip

* fix conan_server timestamp truncation

* fix test

* wip

* wip

* lockfiles

* Update conan/cli/commands/config.py

Co-authored-by: Francisco Ramírez <[email protected]>

* Update conans/client/graph/graph_binaries.py

Co-authored-by: Francisco Ramírez <[email protected]>

* Update conans/test/functional/command/config_install_test.py

Co-authored-by: Francisco Ramírez <[email protected]>

* Fix `conan profile show --format=json` for profiles with scoped confs (#15747)

Fix conan profile show --format=json for profiles with scopd confs

* review

* fix issues and improve CMakeToolchain.preprocessor (#15756)

* fix issues and improve CMakeToolchain.preprocessor

* fix test

* list --profile argument (#15697)

* list --profile argument

* filtering

* fix PkgConfigDeps in build context (#15763)

fix PkgConfigDeps

* [CMakeToolchain]Adding visibility flag to CONAN_C_FLAGS (#15762)

Adding visibility flags to C flags too

* PkgConfigDeps refactor (#15771)

* allow self.name/self.version for build_folder_vars (#15705)

* wip

* wip

* wip

* wip

---------

Co-authored-by: Francisco Ramírez <[email protected]>
Co-authored-by: Rubén Rincón Blanco <[email protected]>

conan/api/subapi/config.py

@@ -1,16 +1,23 @@
+import json
 import os
 import platform
 import textwrap
 import yaml
 from jinja2 import Environment, FileSystemLoader
 
 from conan import conan_version
+from conan.api.output import ConanOutput
 from conans.client.conf import default_settings_yml
 from conan.internal.api import detect_api
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.conan_app import ConanApp
+from conans.client.graph.graph import CONTEXT_HOST, RECIPE_VIRTUAL, Node
+from conans.client.graph.graph_builder import DepsGraphBuilder
+from conans.client.graph.profile_node_definer import consumer_definer
 from conans.errors import ConanException
 from conans.model.conf import ConfDefinition, BUILT_IN_CONFS
+from conans.model.pkg_type import PackageType
+from conans.model.recipe_ref import RecipeReference
 from conans.model.settings import Settings
 from conans.util.files import load, save
 
@@ -32,6 +39,66 @@ def install(self, path_or_url, verify_ssl, config_type=None, args=None,
         configuration_install(app, path_or_url, verify_ssl, config_type=config_type, args=args,
                               source_folder=source_folder, target_folder=target_folder)
 
+    def install_pkg(self, ref, lockfile=None, force=False):
+        ConanOutput().warning("The 'conan config install-pkg' is experimental",
+                              warn_tag="experimental")
+        conan_api = self.conan_api
+        remotes = conan_api.remotes.list()  # ready to use remotes arguments
+        # Ready to use profiles as inputs, but NOT using profiles yet, empty ones
+        profile_host = profile_build = conan_api.profiles.get_profile([])
+
+        app = ConanApp(self.conan_api)
+
+        # Computation of a very simple graph that requires "ref"
+        conanfile = app.loader.load_virtual(requires=[RecipeReference.loads(ref)])
+        consumer_definer(conanfile, profile_build, profile_host)
+        root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL)
+        root_node.is_conf = True
+        update = ["*"]
+        builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, app.cache, remotes,
+                                   update, update, self.conan_api.config.global_conf)
+        deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile)
+
+        # Basic checks of the package: correct package_type and no-dependencies
+        deps_graph.report_graph_error()
+        pkg = deps_graph.root.dependencies[0].dst
+        ConanOutput().info(f"Configuration from package: {pkg}")
+        if pkg.conanfile.package_type is not PackageType.CONF:
+            raise ConanException(f'{pkg.conanfile} is not of package_type="configuration"')
+        if pkg.dependencies:
+            raise ConanException(f"Configuration package {pkg.ref} cannot have dependencies")
+
+        # The computation of the "package_id" and the download of the package is done as usual
+        # By default we allow all remotes, and build_mode=None, always updating
+        conan_api.graph.analyze_binaries(deps_graph, None, remotes, update=update, lockfile=lockfile)
+        conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes)
+
+        # We check if this specific version is already installed
+        config_pref = pkg.pref.repr_notime()
+        config_versions = []
+        config_version_file = HomePaths(conan_api.home_folder).config_version_path
+        if os.path.exists(config_version_file):
+            config_versions = json.loads(load(config_version_file))
+            config_versions = config_versions["config_version"]
+            if config_pref in config_versions:
+                if force:
+                    ConanOutput().info(f"Package '{pkg}' already configured, "
+                                       "but re-installation forced")
+                else:
+                    ConanOutput().info(f"Package '{pkg}' already configured, "
+                                       "skipping configuration install")
+                    return pkg.pref  # Already installed, we can skip repeating the install
+
+        from conans.client.conf.config_installer import configuration_install
+        configuration_install(app, uri=pkg.conanfile.package_folder, verify_ssl=False,
+                              config_type="dir", ignore=["conaninfo.txt", "conanmanifest.txt"])
+        # We save the current package full reference in the file for future
+        # And for ``package_id`` computation
+        config_versions = {ref.split("/", 1)[0]: ref for ref in config_versions}
+        config_versions[pkg.pref.ref.name] = pkg.pref.repr_notime()
+        save(config_version_file, json.dumps({"config_version": list(config_versions.values())}))
+        return pkg.pref
+
     def get(self, name, default=None, check_type=None):
         return self.global_conf.get(name, default=default, check_type=check_type)
 

conan/api/subapi/lockfile.py

@@ -79,19 +79,21 @@ def update_lockfile(lockfile, graph, lock_packages=False, clean=False):
         return lockfile
 
     @staticmethod
-    def add_lockfile(lockfile=None, requires=None, build_requires=None, python_requires=None):
+    def add_lockfile(lockfile=None, requires=None, build_requires=None, python_requires=None,
+                     config_requires=None):
         if lockfile is None:
             lockfile = Lockfile()  # create a new lockfile
             lockfile.partial = True
 
         lockfile.add(requires=requires, build_requires=build_requires,
-                     python_requires=python_requires)
+                     python_requires=python_requires, config_requires=config_requires)
         return lockfile
 
     @staticmethod
-    def remove_lockfile(lockfile, requires=None, build_requires=None, python_requires=None):
+    def remove_lockfile(lockfile, requires=None, build_requires=None, python_requires=None,
+                        config_requires=None):
         lockfile.remove(requires=requires, build_requires=build_requires,
-                        python_requires=python_requires)
+                        python_requires=python_requires, config_requires=config_requires)
         return lockfile
 
     @staticmethod

conan/cli/commands/config.py

@@ -1,5 +1,5 @@
 from conan.api.output import cli_out_write
-from conan.cli.command import conan_command, conan_subcommand
+from conan.cli.command import conan_command, conan_subcommand, OnceArgument
 from conan.cli.formatters import default_json_formatter
 from conans.util.config_parser import get_bool_from_text
 
@@ -37,13 +37,39 @@ def config_install(conan_api, parser, subparser, *args):
                                 'specified origin')
     subparser.add_argument("-tf", "--target-folder",
                            help='Install to that path in the conan cache')
+
     args = parser.parse_args(*args)
-    verify_ssl = args.verify_ssl if isinstance(args.verify_ssl, bool) else get_bool_from_text(args.verify_ssl)
+    verify_ssl = args.verify_ssl if isinstance(args.verify_ssl, bool) \
+        else get_bool_from_text(args.verify_ssl)
     conan_api.config.install(args.item, verify_ssl, args.type, args.args,
                              source_folder=args.source_folder,
                              target_folder=args.target_folder)
 
 
+@conan_subcommand()
+def config_install_pkg(conan_api, parser, subparser, *args):
+    """
+    (Experimental) Install the configuration (remotes, profiles, conf), from a Conan package
+    """
+    subparser.add_argument("item", help="Conan require")
+    subparser.add_argument("-l", "--lockfile", action=OnceArgument,
+                           help="Path to a lockfile. Use --lockfile=\"\" to avoid automatic use of "
+                                "existing 'conan.lock' file")
+    subparser.add_argument("--lockfile-partial", action="store_true",
+                           help="Do not raise an error if some dependency is not found in lockfile")
+    subparser.add_argument("--lockfile-out", action=OnceArgument,
+                           help="Filename of the updated lockfile")
+    subparser.add_argument("-f", "--force", action='store_true',
+                           help="Force the re-installation of configuration")
+    args = parser.parse_args(*args)
+
+    lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
+                                               partial=args.lockfile_partial)
+    config_pref = conan_api.config.install_pkg(args.item, lockfile=lockfile, force=args.force)
+    lockfile = conan_api.lockfile.add_lockfile(lockfile, config_requires=[config_pref.ref])
+    conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out)
+
+
 def list_text_formatter(confs):
     for k, v in confs.items():
         cli_out_write(f"{k}: {v}")

conan/cli/commands/lock.py

@@ -98,6 +98,8 @@ def lock_add(conan_api, parser, subparser, *args):
                            help='Add build-requires to lockfile')
     subparser.add_argument('--python-requires', action="append",
                            help='Add python-requires to lockfile')
+    subparser.add_argument('--config-requires', action="append",
+                           help='Add config-requires to lockfile')
     subparser.add_argument("--lockfile-out", action=OnceArgument, default=LOCKFILE,
                            help="Filename of the created lockfile")
     subparser.add_argument("--lockfile", action=OnceArgument, help="Filename of the input lockfile")
@@ -117,11 +119,13 @@ def _parse_requires(reqs):
     requires = _parse_requires(args.requires)
     build_requires = _parse_requires(args.build_requires)
     python_requires = _parse_requires(args.python_requires)
+    config_requires = _parse_requires(args.config_requires)
 
     lockfile = conan_api.lockfile.add_lockfile(lockfile,
                                                requires=requires,
                                                python_requires=python_requires,
-                                               build_requires=build_requires)
+                                               build_requires=build_requires,
+                                               config_requires=config_requires)
     conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out)
 
 
@@ -136,6 +140,8 @@ def lock_remove(conan_api, parser, subparser, *args):
                            help='Remove build-requires from lockfile')
     subparser.add_argument('--python-requires', action="append",
                            help='Remove python-requires from lockfile')
+    subparser.add_argument('--config-requires', action="append",
+                           help='Remove config-requires from lockfile')
     subparser.add_argument("--lockfile-out", action=OnceArgument, default=LOCKFILE,
                            help="Filename of the created lockfile")
     subparser.add_argument("--lockfile", action=OnceArgument, help="Filename of the input lockfile")
@@ -145,5 +151,6 @@ def lock_remove(conan_api, parser, subparser, *args):
     lockfile = conan_api.lockfile.remove_lockfile(lockfile,
                                                   requires=args.requires,
                                                   python_requires=args.python_requires,
-                                                  build_requires=args.build_requires)
+                                                  build_requires=args.build_requires,
+                                                  config_requires=args.config_requires)
     conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out)

conan/internal/cache/home_paths.py

@@ -74,3 +74,7 @@ def settings_path(self):
     @property
     def settings_path_user(self):
         return os.path.join(self._home, "settings_user.yml")
+
+    @property
+    def config_version_path(self):
+        return os.path.join(self._home, "config_version.json")

conans/client/conf/config_installer.py

@@ -8,12 +8,12 @@
 from conan.api.output import ConanOutput
 from conans.client.downloaders.file_downloader import FileDownloader
 from conans.errors import ConanException
-from conans.util.files import mkdir, rmdir, remove, unzip, chdir
+from conans.util.files import mkdir, rmdir, remove, unzip, chdir, load
 from conans.util.runners import detect_runner
 
 
 class _ConanIgnoreMatcher:
-    def __init__(self, conanignore_path):
+    def __init__(self, conanignore_path, ignore=None):
         conanignore_path = os.path.abspath(conanignore_path)
         self._ignored_entries = {".conanignore"}
         if os.path.exists(conanignore_path):
@@ -22,6 +22,8 @@ def __init__(self, conanignore_path):
                     line_content = line.strip()
                     if line_content != "":
                         self._ignored_entries.add(line_content)
+        if ignore:
+            self._ignored_entries.update(ignore)
 
     def matches(self, path):
         for ignore_entry in self._ignored_entries:
@@ -112,13 +114,13 @@ def _process_file(directory, filename, config, cache, folder):
         _filecopy(directory, filename, target_folder)
 
 
-def _process_folder(config, folder, cache):
+def _process_folder(config, folder, cache, ignore=None):
     if not os.path.isdir(folder):
         raise ConanException("No such directory: '%s'" % str(folder))
     if config.source_folder:
         folder = os.path.join(folder, config.source_folder)
     conanignore_path = os.path.join(folder, '.conanignore')
-    conanignore = _ConanIgnoreMatcher(conanignore_path)
+    conanignore = _ConanIgnoreMatcher(conanignore_path, ignore)
     for root, dirs, files in os.walk(folder):
         # .git is always ignored by default, even if not present in .conanignore
         dirs[:] = [d for d in dirs if d != ".git"]
@@ -178,14 +180,14 @@ def _is_compressed_file(filename):
 
 
 def configuration_install(app, uri, verify_ssl, config_type=None,
-                          args=None, source_folder=None, target_folder=None):
+                          args=None, source_folder=None, target_folder=None, ignore=None):
     cache, requester = app.cache, app.requester
     config = _ConfigOrigin(uri, config_type, verify_ssl, args, source_folder, target_folder)
     try:
         if config.type == "git":
             _process_git_repo(config, cache)
         elif config.type == "dir":
-            _process_folder(config, config.uri, cache)
+            _process_folder(config, config.uri, cache, ignore)
         elif config.type == "file":
             if _is_compressed_file(config.uri):
                 with tmp_config_install_folder(cache) as tmp_folder:

conans/client/graph/compute_pid.py

@@ -6,12 +6,13 @@
 from conans.client.conanfile.implementations import auto_header_only_package_id
 
 
-def compute_package_id(node, new_config):
+def compute_package_id(node, new_config, config_version):
     """
     Compute the binary package ID of this node
     """
     conanfile = node.conanfile
 
+    # TODO: Extract
     unknown_mode = new_config.get("core.package_id:default_unknown_mode", default="semver_mode")
     non_embed_mode = new_config.get("core.package_id:default_non_embed_mode", default="minor_mode")
     # recipe_revision_mode already takes into account the package_id
@@ -50,7 +51,8 @@ def compute_package_id(node, new_config):
                                reqs_info=reqs_info,
                                build_requires_info=build_requires_info,
                                python_requires=python_requires,
-                               conf=conanfile.conf.copy_conaninfo_conf())
+                               conf=conanfile.conf.copy_conaninfo_conf(),
+                               config_version=config_version.copy() if config_version else None)
     conanfile.original_info = conanfile.info.clone()
 
     if hasattr(conanfile, "validate_build"):

conans/client/graph/graph.py

@@ -66,6 +66,7 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False):
         self.cant_build = False  # It will set to a str with a reason if the validate_build() fails
         self.should_build = False  # If the --build or policy wants to build this binary
         self.build_allowed = False
+        self.is_conf = False
 
     def __lt__(self, other):
         """

conans/client/graph/graph_binaries.py

@@ -1,3 +1,7 @@
+import json
+import os
+from collections import OrderedDict
+
 from conan.api.output import ConanOutput
 from conan.internal.cache.home_paths import HomePaths
 from conans.client.graph.build_mode import BuildMode
@@ -10,7 +14,11 @@
                                        BINARY_PLATFORM)
 from conans.client.graph.proxy import should_update_reference
 from conans.errors import NoRemoteAvailable, NotFoundException, \
-    PackageNotFoundException, conanfile_exception_formatter, ConanConnectionError
+    PackageNotFoundException, conanfile_exception_formatter, ConanConnectionError, ConanException
+from conans.model.info import RequirementInfo, RequirementsInfo
+from conans.model.package_ref import PkgReference
+from conans.model.recipe_ref import RecipeReference
+from conans.util.files import load
 
 
 class GraphBinariesAnalyzer(object):
@@ -321,8 +329,30 @@ def _evaluate_in_cache(self, cache_latest_prev, node, remotes, update):
             node.pref_timestamp = cache_latest_prev.timestamp
             assert node.prev, "PREV for %s is None" % str(node.pref)
 
-    def _evaluate_package_id(self, node):
-        compute_package_id(node, self._global_conf)
+    def _config_version(self):
+        config_mode = self._global_conf.get("core.package_id:config_mode", default=None)
+        if config_mode is None:
+            return
+        config_version_file = HomePaths(self._cache.cache_folder).config_version_path
+        try:
+            config_refs = json.loads(load(config_version_file))["config_version"]
+            result = OrderedDict()
+            for r in config_refs:
+                try:
+                    config_ref = PkgReference.loads(r)
+                    req_info = RequirementInfo(config_ref.ref, config_ref.package_id, config_mode)
+                except ConanException:
+                    config_ref = RecipeReference.loads(r)
+                    req_info = RequirementInfo(config_ref, None, config_mode)
+                result[config_ref] = req_info
+        except Exception as e:
+            raise ConanException(f"core.package_id:config_mode defined, but error while loading "
+                                 f"'{os.path.basename(config_version_file)}'"
+                                 f" file in cache: {self._cache.cache_folder}: {e}")
+        return RequirementsInfo(result)
+
+    def _evaluate_package_id(self, node, config_version):
+        compute_package_id(node, self._global_conf, config_version=config_version)
 
         # TODO: layout() execution don't need to be evaluated at GraphBuilder time.
         # it could even be delayed until installation time, but if we got enough info here for
@@ -359,9 +389,10 @@ def _evaluate_single(n):
             self._evaluate_node(n, mode, remotes, update)
 
         levels = deps_graph.by_levels()
+        config_version = self._config_version()
         for level in levels[:-1]:  # all levels but the last one, which is the single consumer
             for node in level:
-                self._evaluate_package_id(node)
+                self._evaluate_package_id(node, config_version)
             # group by pref to paralelize, so evaluation is done only 1 per pref
             nodes = {}
             for node in level:
@@ -382,7 +413,7 @@ def _evaluate_single(n):
         if node.path is not None:
             if node.path.endswith(".py"):
                 # For .py we keep evaluating the package_id, validate(), etc
-                compute_package_id(node, self._global_conf)
+                compute_package_id(node, self._global_conf, config_version=config_version)
             # To support the ``[layout]`` in conanfile.txt
             if hasattr(node.conanfile, "layout"):
                 with conanfile_exception_formatter(node.conanfile, "layout"):