From 098835f0eb28912a9cfb010f666fb7c8485afeb8 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 31 May 2024 15:48:57 +0900 Subject: [PATCH] add a logger to the pip extension --- python/private/bzlmod/pip.bzl | 44 ++++++++++++++++++- .../private/parse_requirements_add_dists.bzl | 7 ++- python/private/whl_target_platforms.bzl | 14 +++++- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 6afeb8adc0..5f71fc62da 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -100,7 +100,30 @@ You cannot use both the additive_build_content and additive_build_content_file a whl_mods = whl_mods, ) +def _new_logger(verbosity_level = None): + verbosity = { + "DEBUG": 2, + "INFO": 1, + "TRACE": 3, + }.get(verbosity_level, 0) + + # buildifier: disable=print + def _log(enabled_on_verbosity, level, *args): + if verbosity < enabled_on_verbosity: + return + print("{}: ".format(level.upper()), *args) + + return struct( + trace = lambda *args: _log(3, "TRACE", *args), + debug = lambda *args: _log(2, "DEBUG", *args), + info = lambda *args: _log(1, "INFO", *args), + # buildifier: disable=print + warn = lambda *args: print("WARNING: ", *args), + fail = lambda *args: fail(*args), + ) + def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache): + logger = _new_logger(pip_attr.verbosity) python_interpreter_target = pip_attr.python_interpreter_target # if we do not have the python_interpreter set in the attributes @@ -198,6 +221,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s requirements_by_platform, index_urls, python_version = major_minor, + logger = logger, ) repository_platform = host_platform(module_ctx.os) @@ -261,11 +285,22 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) if requirement.whls or requirement.sdists: + logger.debug("Selecting a compatible dist for {} from dists:\n{}".format( + repository_platform, + json.encode( + struct( + whls = requirement.whls, + sdists = requirement.sdists, + ), + ), + )) distribution = select_whl( whls = requirement.whls, want_platform = repository_platform, ) or (requirement.sdists[0] if requirement.sdists else None) + logger.debug("Selected: {}".format(distribution)) + if distribution: whl_library_args["requirement"] = requirement.srcs.requirement whl_library_args["urls"] = [distribution.url] @@ -282,7 +317,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # This is no-op because pip is not used to download the wheel. whl_library_args.pop("download_only", None) else: - print("WARNING: falling back to pip for installing the right file for {}".format(requirement.requirement_line)) # buildifier: disable=print + logger.warn("falling back to pip for installing the right file for {}".format(requirement.requirement_line)) # We sort so that the lock-file remains the same no matter the order of how the # args are manipulated in the code going before. @@ -548,6 +583,13 @@ The Python version the dependencies are targetting, in Major.Minor format If an interpreter isn't explicitly provided (using `python_interpreter` or `python_interpreter_target`), then the version specified here must have a corresponding `python.toolchain()` configured. +""", + ), + "verbosity": attr.string( + default = "", + values = ["TRACE", "DEBUG", "INFO"], + doc = """ +The verbosity with which we should print diagnostic messages when 'quiet = False'. """, ), "whl_modifications": attr.label_keyed_string_dict( diff --git a/python/private/parse_requirements_add_dists.bzl b/python/private/parse_requirements_add_dists.bzl index e8c89c344a..c6f25dd194 100644 --- a/python/private/parse_requirements_add_dists.bzl +++ b/python/private/parse_requirements_add_dists.bzl @@ -22,7 +22,7 @@ am keeping it together with parse_requirements.bzl. load(":whl_target_platforms.bzl", "select_whls") -def parse_requirements_add_dists(requirements_by_platform, index_urls, python_version): +def parse_requirements_add_dists(requirements_by_platform, index_urls, python_version, logger = None): """Populate dists based on the information from the PyPI index. This function will modify the given requirements_by_platform data structure. @@ -31,6 +31,7 @@ def parse_requirements_add_dists(requirements_by_platform, index_urls, python_ve requirements_by_platform: The result of parse_requirements function. index_urls: The result of simpleapi_download. python_version: The version of the python interpreter. + logger: A logger for printing diagnostic info. """ for whl_name, requirements in requirements_by_platform.items(): for requirement in requirements: @@ -55,7 +56,8 @@ def parse_requirements_add_dists(requirements_by_platform, index_urls, python_ve sdist = maybe_sdist continue - print("WARNING: Could not find a whl or an sdist with sha256={}".format(sha256)) # buildifier: disable=print + if logger: + logger.warn("Could not find a whl or an sdist with sha256={}".format(sha256)) # Filter out the wheels that are incompatible with the target_platforms. whls = select_whls( @@ -69,6 +71,7 @@ def parse_requirements_add_dists(requirements_by_platform, index_urls, python_ve ], want_platforms = requirement.target_platforms, want_version = python_version, + logger = logger, ) requirement.whls.extend(whls) diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl index 863512b8fe..52ad8ea0f2 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/whl_target_platforms.bzl @@ -103,7 +103,7 @@ def _whl_priority(value): # Windows does not have multiple wheels for the same target platform return (False, False, 0, 0) -def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = []): +def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = [], logger = None): """Select a subset of wheels suitable for target platforms from a list. Args: @@ -111,6 +111,7 @@ def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = want_version(str): An optional parameter to filter whls by version. Defaults to '3.0'. want_abis(list[str]): A list of ABIs that are supported. want_platforms(str): The platforms + logger: A logger for printing diagnostic messages. Returns: None or a struct with `url`, `sha256` and `filename` attributes for the @@ -127,6 +128,9 @@ def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = for whl in whls: parsed = parse_whl_name(whl.filename) + if logger: + logger.trace("Deciding whether to use '{}'".format(whl.filename)) + supported_implementations = {} whl_version_min = 0 for tag in parsed.python_tag.split("."): @@ -142,13 +146,19 @@ def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = whl_version_min = version if not ("cp" in supported_implementations or "py" in supported_implementations): + if logger: + logger.trace("Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations)) continue if want_abis and parsed.abi_tag not in want_abis: # Filter out incompatible ABIs + if logger: + logger.trace("Discarding the whl because the whl abi did not match") continue if version_limit != -1 and whl_version_min > version_limit: + if logger: + logger.trace("Discarding the whl because the whl supported python version is too high") continue compatible = False @@ -161,6 +171,8 @@ def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = break if not compatible: + if logger: + logger.trace("Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms)) continue for implementation in supported_implementations: