New "conf" configuration mechanism (#8266)

* working on config

* working

* working

* marking tests as visual

* add command line

* removed command line support

* updates for composition and inclusion working

* trying delimiters

* removed unrelated refactor

* proposing config not in profile

* working

* working

* working

* user vs global config

* working

* fix Py2 error

* more general paackage pattern match

* adding MSBuildToolchain compile_options conf example

* added some package_id capabilities

* added assignment of conf for virtual and txt

* more tests

* review

* changed file name conan.cfg

conan/tools/cmake/cmake.py

@@ -3,6 +3,7 @@
 
 from conan.tools.cmake.base import CMakeToolchainBase
 from conan.tools.cmake.utils import get_generator, is_multi_configuration
+from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg
 from conans.client import tools
 from conans.client.build import join_arguments
 from conans.client.tools.files import chdir
@@ -20,7 +21,7 @@ def _validate_recipe(conanfile):
                              " or 'cmake_find_package_multi' generators")
 
 
-def _cmake_cmd_line_args(conanfile, generator, parallel, msbuild_verbosity):
+def _cmake_cmd_line_args(conanfile, generator, parallel):
     args = []
     compiler_version = conanfile.settings.get_safe("compiler.version")
     if generator and parallel:
@@ -30,9 +31,10 @@ def _cmake_cmd_line_args(conanfile, generator, parallel, msbuild_verbosity):
             # Parallel for building projects in the solution
             args.append("/m:%i" % cpu_count(output=conanfile.output))
 
-    if generator and msbuild_verbosity:
-        if "Visual Studio" in generator and compiler_version and Version(compiler_version) >= "10":
-            args.append("/verbosity:%s" % msbuild_verbosity)
+    if generator and "Visual Studio" in generator:
+        verbosity = msbuild_verbosity_cmd_line_arg(conanfile)
+        if verbosity:
+            args.append(verbosity)
 
     return args
 
@@ -44,9 +46,7 @@ class CMake(object):
     are passed to the command line, plus the ``--config Release`` for builds in multi-config
     """
 
-    def __init__(self, conanfile, generator=None, build_folder=None, parallel=True,
-                 msbuild_verbosity="minimal"):
-
+    def __init__(self, conanfile, generator=None, build_folder=None, parallel=True):
         _validate_recipe(conanfile)
 
         # assert generator is None, "'generator' is handled by the toolchain"
@@ -56,7 +56,6 @@ def __init__(self, conanfile, generator=None, build_folder=None, parallel=True,
         # Store a reference to useful data
         self._conanfile = conanfile
         self._parallel = parallel
-        self._msbuild_verbosity = msbuild_verbosity
 
         self._build_folder = build_folder
         self._cmake_program = "cmake"  # Path to CMake should be handled by environment
@@ -115,8 +114,7 @@ def _build(self, build_type=None, target=None):
         if target is not None:
             args = ["--target", target]
 
-        cmd_line_args = _cmake_cmd_line_args(self._conanfile, self._generator, self._parallel,
-                                             self._msbuild_verbosity)
+        cmd_line_args = _cmake_cmd_line_args(self._conanfile, self._generator, self._parallel)
         if cmd_line_args:
             args += ['--'] + cmd_line_args
 

conan/tools/gnu/__init__.py

@@ -1,2 +1 @@
-# noinspection PyUnresolvedReferences
 from .make import MakeToolchain

conan/tools/meson/__init__.py

@@ -1,3 +1,2 @@
-# noinspection PyUnresolvedReferences
 from conan.tools.meson.toolchain import MesonToolchain
 from conan.tools.meson.meson import Meson

conan/tools/microsoft/__init__.py

@@ -1,4 +1,3 @@
-# noinspection PyUnresolvedReferences
 from .toolchain import MSBuildToolchain
 from .msbuild import MSBuild
 from .msbuilddeps import MSBuildDeps

conan/tools/microsoft/msbuild.py

@@ -1,13 +1,22 @@
 from conan.tools.microsoft.visual import vcvars_arch, vcvars_command
 from conans.client.tools import intel_compilervars_command
+from conans.errors import ConanException
+
+
+def msbuild_verbosity_cmd_line_arg(conanfile):
+    verbosity = conanfile.conf["tools.microsoft"].msbuild_verbosity
+    if verbosity:
+        if verbosity not in ("Quiet", "Minimal", "Normal", "Detailed", "Diagnostic"):
+            raise ConanException("Unknown msbuild verbosity: {}".format(verbosity))
+        return '/verbosity:{}'.format(verbosity)
 
 
 class MSBuild(object):
     def __init__(self, conanfile):
         self._conanfile = conanfile
         self.compiler = conanfile.settings.get_safe("compiler")
-        self.version = conanfile.settings.get_safe("compiler.base.version") or \
-                       conanfile.settings.get_safe("compiler.version")
+        self.version = (conanfile.settings.get_safe("compiler.base.version") or
+                        conanfile.settings.get_safe("compiler.version"))
         self.vcvars_arch = vcvars_arch(conanfile)
         self.build_type = conanfile.settings.get_safe("build_type")
         msvc_arch = {'x86': 'x86',
@@ -29,9 +38,13 @@ def command(self, sln):
             cvars = vcvars_command(self.version, architecture=self.vcvars_arch,
                                    platform_type=None, winsdk_version=None,
                                    vcvars_ver=None)
-        cmd = ('%s && msbuild "%s" /p:Configuration=%s /p:Platform=%s '
+        cmd = ('%s && msbuild "%s" /p:Configuration=%s /p:Platform=%s'
                % (cvars, sln, self.build_type, self.platform))
 
+        verbosity = msbuild_verbosity_cmd_line_arg(self._conanfile)
+        if verbosity:
+            cmd += " {}".format(verbosity)
+
         return cmd
 
     def build(self, sln):

conan/tools/microsoft/toolchain.py

@@ -15,6 +15,7 @@ class MSBuildToolchain(object):
     def __init__(self, conanfile):
         self._conanfile = conanfile
         self.preprocessor_definitions = {}
+        self.compile_options = {}
         self.configuration = conanfile.settings.build_type
 
     def _name_condition(self, settings):
@@ -70,7 +71,7 @@ def format_macro(k, value):
                      {};%(PreprocessorDefinitions)
                   </PreprocessorDefinitions>
                   <RuntimeLibrary>{}</RuntimeLibrary>
-                  <LanguageStandard>{}</LanguageStandard>
+                  <LanguageStandard>{}</LanguageStandard>{}
                 </ClCompile>
               </ItemDefinitionGroup>
               <PropertyGroup Label="Configuration">
@@ -83,7 +84,14 @@ def format_macro(k, value):
         # It is useless to set PlatformToolset in the config file, because the conditional checks it
         cppstd = "stdcpp%s" % cppstd if cppstd else ""
         toolset = toolset or ""
-        config_props = content.format(preprocessor_definitions, runtime_library, cppstd, toolset)
+        compile_options = self._conanfile.conf["tools.microsoft.msbuildtoolchain"].compile_options
+        if compile_options is not None:
+            compile_options = eval(compile_options)
+            self.compile_options.update(compile_options)
+        compile_options = "".join("\n      <{k}>{v}</{k}>".format(k=k, v=v)
+                                  for k, v in self.compile_options.items())
+        config_props = content.format(preprocessor_definitions, runtime_library, cppstd,
+                                      compile_options, toolset)
         config_filepath = os.path.abspath(config_filename)
         self._conanfile.output.info("MSBuildToolchain created %s" % config_filename)
         save(config_filepath, config_props)

conans/client/cache/cache.py

@@ -15,6 +15,7 @@
 from conans.client.profile_loader import read_profile
 from conans.client.store.localdb import LocalDB
 from conans.errors import ConanException
+from conans.model.conf import ConfDefinition
 from conans.model.profile import Profile
 from conans.model.ref import ConanFileReference
 from conans.model.settings import Settings
@@ -77,6 +78,7 @@ def __init__(self, cache_folder, output):
         # Caching
         self._no_lock = None
         self._config = None
+        self._new_config = None
         self.editable_packages = EditablePackages(self.cache_folder)
         # paths
         self._store_folder = self.config.storage_path or os.path.join(self.cache_folder, "data")
@@ -155,6 +157,22 @@ def config(self):
             self._config = ConanClientConfigParser(self.conan_conf_path)
         return self._config
 
+    @property
+    def new_config_path(self):
+        return os.path.join(self.cache_folder, "conan.cfg")
+
+    @property
+    def new_config(self):
+        """ this is the new conan.cfgto replace the old conan.conf that contains
+        configuration defined with the new syntax as in profiles, this config will be composed
+        to the profile ones and passed to the conanfiles.conf, which can be passed to collaborators
+        """
+        if self._new_config is None:
+            self._new_config = ConfDefinition()
+            if os.path.exists(self.new_config_path):
+                self._new_config.loads(load(self.new_config_path))
+        return self._new_config
+
     @property
     def localdb(self):
         localdb_filename = os.path.join(self.cache_folder, LOCALDB)

conans/client/conan_api.py

@@ -1468,6 +1468,13 @@ def get_graph_info(profile_host, profile_build, cwd, install_folder, cache, outp
         root_ref = ConanFileReference(name, version, user, channel, validate=False)
         graph_info = GraphInfo(profile_host=phost, profile_build=pbuild, root_ref=root_ref)
         # Preprocess settings and convert to real settings
+
+    # Apply the new_config to the profiles the global one, so recipes get it too
+    # TODO: This means lockfiles contain whole copy of the config here?
+    # FIXME: Apply to locked graph-info as well
+    graph_info.profile_host.conf.rebase_conf_definition(cache.new_config)
+    if graph_info.profile_build is not None:
+        graph_info.profile_build.conf.rebase_conf_definition(cache.new_config)
     return graph_info
 
 

conans/client/conf/required_version.py

@@ -1,3 +1,5 @@
+import six
+
 from conans.client.cache.cache import ClientCache
 from semver import satisfies
 from conans import __version__ as client_version
@@ -28,3 +30,8 @@ def check_required_conan_version(cache_folder, out):
     if required_range:
         validate_conan_version(required_range)
 
+    required_range_new = cache.new_config["core"].required_conan_version
+    if required_range_new:
+        if six.PY2 and not isinstance(required_range_new, str):
+            required_range_new = required_range_new.encode()
+        validate_conan_version(required_range_new)

conans/client/loader.py

@@ -175,25 +175,25 @@ def _initialize_conanfile(conanfile, profile):
         # Mixing the global settings with the specified for that name if exist
         tmp_settings = profile.processed_settings.copy()
         package_settings_values = profile.package_settings_values
+        if conanfile._conan_user is not None:
+            ref_str = "%s/%s@%s/%s" % (conanfile.name, conanfile.version,
+                                       conanfile._conan_user, conanfile._conan_channel)
+        else:
+            ref_str = "%s/%s" % (conanfile.name, conanfile.version)
         if package_settings_values:
             # First, try to get a match directly by name (without needing *)
             # TODO: Conan 2.0: We probably want to remove this, and leave a pure fnmatch
             pkg_settings = package_settings_values.get(conanfile.name)
             if pkg_settings is None:  # If there is not exact match by package name, do fnmatch
-                if conanfile._conan_user is not None:
-                    ref = "%s/%s@%s/%s" % (conanfile.name, conanfile.version,
-                                           conanfile._conan_user, conanfile._conan_channel)
-                else:
-                    ref = "%s/%s" % (conanfile.name, conanfile.version)
-
                 for pattern, settings in package_settings_values.items():
-                    if fnmatch.fnmatchcase(ref, pattern):
+                    if fnmatch.fnmatchcase(ref_str, pattern):
                         pkg_settings = settings
                         break
             if pkg_settings:
                 tmp_settings.update_values(pkg_settings)
 
         conanfile.initialize(tmp_settings, profile.env_values)
+        conanfile.conf = profile.conf.get_conanfile_conf(ref_str)
 
     def load_consumer(self, conanfile_path, profile_host, name=None, version=None, user=None,
                       channel=None, lock_python_requires=None):
@@ -262,6 +262,7 @@ def load_conanfile_txt(self, conan_txt_path, profile_host, ref=None):
     def _parse_conan_txt(self, contents, path, display_name, profile):
         conanfile = ConanFile(self._output, self._runner, display_name)
         conanfile.initialize(Settings(), profile.env_values)
+        conanfile.conf = profile.conf.get_conanfile_conf(None)
         # It is necessary to copy the settings, because the above is only a constraint of
         # conanfile settings, and a txt doesn't define settings. Necessary for generators,
         # as cmake_multi, that check build_type.
@@ -301,6 +302,7 @@ def load_virtual(self, references, profile_host, scope_options=True,
         conanfile = ConanFile(self._output, self._runner, display_name="virtual")
         conanfile.initialize(profile_host.processed_settings.copy(),
                              profile_host.env_values)
+        conanfile.conf = profile_host.conf.get_conanfile_conf(None)
         conanfile.settings = profile_host.processed_settings.copy_values()
 
         for reference in references:

conans/client/profile_loader.py

@@ -2,6 +2,7 @@
 from collections import OrderedDict, defaultdict
 
 from conans.errors import ConanException, ConanV2Exception
+from conans.model.conf import ConfDefinition
 from conans.model.env_info import EnvValues, unquote
 from conans.model.options import OptionsValues
 from conans.model.profile import Profile
@@ -148,7 +149,7 @@ def _load_profile(text, profile_path, default_folder):
 
         # Current profile before update with parents (but parent variables already applied)
         doc = ConfigParser(profile_parser.profile_text,
-                           allowed_fields=["build_requires", "settings", "env", "options"])
+                           allowed_fields=["build_requires", "settings", "env", "options", "conf"])
 
         # Merge the inherited profile with the readed from current profile
         _apply_inner_profile(doc, inherited_profile)
@@ -218,6 +219,11 @@ def get_package_name_value(item):
     current_env_values.update(base_profile.env_values)
     base_profile.env_values = current_env_values
 
+    if doc.conf:
+        new_prof = ConfDefinition()
+        new_prof.loads(doc.conf, profile=True)
+        base_profile.conf.update_conf_definition(new_prof)
+
 
 def profile_from_args(profiles, settings, options, env, cwd, cache):
     """ Return a Profile object, as the result of merging a potentially existing Profile