From 3a3b0f5feb8b5d37dd98042e2ca26f9346fc382d Mon Sep 17 00:00:00 2001 From: "Reid D. McKenzie" Date: Wed, 12 Apr 2023 13:20:01 -0600 Subject: [PATCH] feat(pip_repository): Enable PyPi dep cycles This patch adjusts the pip_repository interface to accept a new parameter: `composite_libs`, being a list of PyPi package names which form a cycle and must be installed together. The intuition behind this design is that a dependency cycle {a <-> b} is implemented simply as emplacing both a and b at once. Hence a dependency graph {c -> a, c -> b} has the same effect. If we modify the installation of a and b to remove their mutual dependency, and generate a c which dominates a and b, we can then modify the `requirement()` and `whl_requirement()` helper functions to recognize the requirements a and b and provide a reference to c instead. --- .bazelrc | 4 +- docs/pip_repository.md | 8 +- examples/pip_parse_vendored/BUILD.bazel | 18 ++- examples/pip_parse_vendored/WORKSPACE | 6 + examples/pip_parse_vendored/requirements.bzl | 126 ++++++++++++++++-- examples/pip_parse_vendored/requirements.in | 2 + examples/pip_parse_vendored/requirements.txt | 124 +++++++++++++++++ examples/pip_parse_vendored/test.py | 11 ++ python/pip_install/pip_repository.bzl | 98 ++++++++++---- .../pip_repository_build.bazel.tmpl | 10 ++ .../pip_repository_build_bzlmod.bazel.tmpl | 6 + .../pip_install/pip_repository_lib.bzl.tmpl | 21 +++ .../pip_repository_requirements.bzl.tmpl | 45 +++++-- .../tools/dependency_resolver/__init__.py | 1 - python/pip_install/tools/lib/__init__.py | 1 - python/pip_install/tools/lib/annotation.py | 1 - .../pip_install/tools/lib/annotations_test.py | 1 - .../tools/wheel_installer/wheel_installer.py | 29 ++-- 18 files changed, 448 insertions(+), 64 deletions(-) create mode 100644 examples/pip_parse_vendored/test.py create mode 100644 python/pip_install/pip_repository_build.bazel.tmpl create mode 100644 python/pip_install/pip_repository_build_bzlmod.bazel.tmpl create mode 100644 python/pip_install/pip_repository_lib.bzl.tmpl diff --git a/.bazelrc b/.bazelrc index e7e4af7bbd..5741a64f39 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/get_url,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/py_proto_library,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/get_url,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/py_proto_library,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points test --test_output=errors diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 29cb3d9c32..d3fdb5976c 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -7,8 +7,8 @@ ## pip_repository 
-pip_repository(name, annotations, download_only, enable_implicit_namespace_pkgs, environment,
-               extra_pip_args, incompatible_generate_aliases, isolated, pip_data_exclude,
+pip_repository(name, annotations, composite_libs, download_only, enable_implicit_namespace_pkgs,
+               environment, extra_pip_args, incompatible_generate_aliases, isolated, pip_data_exclude,
                python_interpreter, python_interpreter_target, quiet, repo_mapping, repo_prefix,
                requirements_darwin, requirements_linux, requirements_lock, requirements_windows,
                timeout)
@@ -61,6 +61,7 @@ py_binary(
 | :------------- | :------------- | :------------- | :------------- | :------------- |
 | name |  A unique name for this repository.   | Name | required |  |
 | annotations |  Optional annotations to apply to packages   | Dictionary: String -> String | optional | {} |
+| composite_libs |  Groups of requirements which represent dependency cycles and must be treated as composites.   | Dictionary: String -> List of strings | optional | {} |
 | download_only |  Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform.   | Boolean | optional | False |
 | enable_implicit_namespace_pkgs |  If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style.   | Boolean | optional | False |
 | environment |  Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed.   | Dictionary: String -> String | optional | {} |
@@ -112,7 +113,7 @@ A rule for bzlmod pip_repository creation. Intended for private use only.
 
 whl_library(name, annotation, download_only, enable_implicit_namespace_pkgs, environment,
             extra_pip_args, isolated, pip_data_exclude, python_interpreter, python_interpreter_target,
-            quiet, repo, repo_mapping, repo_prefix, requirement, timeout)
+            quiet, repo, repo_mapping, repo_prefix, requirement, skip_deps, timeout)
 

 
 
@@ -139,6 +140,7 @@ Instantiated from pip_repository and inherits config options from there.
 | repo_mapping |  A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target).   | Dictionary: String -> String | required |  |
 | repo_prefix |  Prefix for the generated packages will be of the form @<prefix><sanitized-package-name>//...   | String | optional | "" |
 | requirement |  Python requirement string describing the package to make available   | String | required |  |
+| skip_deps |  List of requirements to skip due to clustering   | List of strings | optional | [] |
 | timeout |  Timeout (in seconds) on the rule's execution duration.   | Integer | optional | 600 |
 
 
diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel
index 56630e513d..c868c641ff 100644
--- a/examples/pip_parse_vendored/BUILD.bazel
+++ b/examples/pip_parse_vendored/BUILD.bazel
@@ -1,6 +1,8 @@
 load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
+load("@rules_python//python:defs.bzl", "py_test")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("//:requirements.bzl", "requirement")
 
 # This rule adds a convenient way to update the requirements.txt
 # lockfile based on the requirements.in.
@@ -20,8 +22,10 @@ genrule(
         # Replace the bazel 6.0.0 specific comment with something that bazel 5.4.0 would produce.
         # This enables this example to be run as a test under bazel 5.4.0.
         """sed -e 's#@//#//#'""",
-        """sed 's#"@python39_.*//:bin/python3"#interpreter#' >$@""",
-    ]),
+        """sed 's#"@python39_.*//:bin/python3"#interpreter#'""",
+        # Stick some buildifiler disables in as needed
+        """sed 's/^def /# buildifier: disable=function-docstring\\\ndef /g'""",
+    ]) + " >$@",
 )
 
 write_file(
@@ -50,3 +54,13 @@ diff_test(
     file1 = "requirements.bzl",
     file2 = ":make_platform_agnostic",
 )
+
+py_test(
+    name = "test",
+    srcs = ["test.py"],
+    deps = [
+        requirement("oletools"),
+        requirement("pcodedmp"),
+        requirement("requests"),
+    ],
+)
diff --git a/examples/pip_parse_vendored/WORKSPACE b/examples/pip_parse_vendored/WORKSPACE
index 157f70aeb6..6a0b214677 100644
--- a/examples/pip_parse_vendored/WORKSPACE
+++ b/examples/pip_parse_vendored/WORKSPACE
@@ -21,6 +21,12 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 # It also wouldn't be needed by users of this ruleset.
 pip_parse(
     name = "pip",
+    composite_libs = {
+        "vbap": [
+            "oletools",
+            "pcodedmp",
+        ],
+    },
     python_interpreter_target = interpreter,
     requirements_lock = "//:requirements.txt",
 )
diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl
index 015df9340a..160bef80de 100644
--- a/examples/pip_parse_vendored/requirements.bzl
+++ b/examples/pip_parse_vendored/requirements.bzl
@@ -7,47 +7,149 @@ from //:requirements.txt
 load("@python39//:defs.bzl", "interpreter")
 load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
 
-all_requirements = ["@pip_certifi//:pkg", "@pip_charset_normalizer//:pkg", "@pip_idna//:pkg", "@pip_requests//:pkg", "@pip_urllib3//:pkg"]
+all_requirements = [
+    "@pip_certifi//:pkg",
+    "@pip_cffi//:pkg",
+    "@pip_charset_normalizer//:pkg",
+    "@pip_colorclass//:pkg",
+    "@pip_cryptography//:pkg",
+    "@pip_easygui//:pkg",
+    "@pip_idna//:pkg",
+    "@pip_msoffcrypto_tool//:pkg",
+    "@pip_olefile//:pkg",
+    "@pip_oletools//:pkg",
+    "@pip_pcodedmp//:pkg",
+    "@pip_pycparser//:pkg",
+    "@pip_pyparsing//:pkg",
+    "@pip_requests//:pkg",
+    "@pip_urllib3//:pkg",
+]
 
-all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "@pip_idna//:whl", "@pip_requests//:whl", "@pip_urllib3//:whl"]
+all_whl_requirements = [
+    "@pip_certifi//:whl",
+    "@pip_cffi//:whl",
+    "@pip_charset_normalizer//:whl",
+    "@pip_colorclass//:whl",
+    "@pip_cryptography//:whl",
+    "@pip_easygui//:whl",
+    "@pip_idna//:whl",
+    "@pip_msoffcrypto_tool//:whl",
+    "@pip_olefile//:whl",
+    "@pip_oletools//:whl",
+    "@pip_pcodedmp//:whl",
+    "@pip_pycparser//:whl",
+    "@pip_pyparsing//:whl",
+    "@pip_requests//:whl",
+    "@pip_urllib3//:whl",
+]
 
-_packages = [("pip_certifi", "certifi==2022.12.7     --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3     --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"), ("pip_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
-_annotations = {}
+_packages = [
+    ("pip_certifi", "certifi==2022.12.7     --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3     --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"),
+    ("pip_cffi", "cffi==1.15.1     --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5     --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef     --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104     --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426     --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405     --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375     --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a     --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e     --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc     --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf     --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185     --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497     --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3     --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35     --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c     --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83     --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21     --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca     --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984     --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac     --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd     --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee     --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a     --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2     --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192     --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7     --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585     --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f     --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e     --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27     --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b     --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e     --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e     --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d     --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c     --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415     --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82     --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02     --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314     --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325     --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c     --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3     --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914     --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045     --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d     --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9     --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5     --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2     --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c     --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3     --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2     --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8     --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d     --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d     --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9     --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162     --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76     --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4     --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e     --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9     --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6     --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b     --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01     --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"),
+    ("pip_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"),
+    ("pip_colorclass", "colorclass==2.2.2     --hash=sha256:6d4fe287766166a98ca7bc6f6312daf04a0481b1eda43e7173484051c0ab4366     --hash=sha256:6f10c273a0ef7a1150b1120b6095cbdd68e5cf36dfd5d0fc957a2500bbf99a55"),
+    ("pip_cryptography", "cryptography==40.0.2     --hash=sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440     --hash=sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288     --hash=sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b     --hash=sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958     --hash=sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b     --hash=sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d     --hash=sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a     --hash=sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404     --hash=sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b     --hash=sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e     --hash=sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2     --hash=sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c     --hash=sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b     --hash=sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9     --hash=sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b     --hash=sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636     --hash=sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99     --hash=sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e     --hash=sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9"),
+    ("pip_easygui", "easygui==0.98.3     --hash=sha256:33498710c68b5376b459cd3fc48d1d1f33822139eb3ed01defbc0528326da3ba     --hash=sha256:d653ff79ee1f42f63b5a090f2f98ce02335d86ad8963b3ce2661805cafe99a04"),
+    ("pip_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"),
+    ("pip_msoffcrypto_tool", "msoffcrypto-tool==5.0.1     --hash=sha256:2b489c8a2b13bec07b94c8f5ce9054111dec3223ff8bedfd486cae3c299be54b     --hash=sha256:9efd0ef5cc3e086e2d175e7a5d7b2b8cb59836c896b8a486d362bbca166db645"),
+    ("pip_olefile", "olefile==0.46     --hash=sha256:133b031eaf8fd2c9399b78b8bc5b8fcbe4c31e85295749bb17a87cba8f3c3964"),
+    ("pip_pycparser", "pycparser==2.21     --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9     --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"),
+    ("pip_pyparsing", "pyparsing==2.4.7     --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1     --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"),
+    ("pip_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"),
+    ("pip_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"),
+]
 
+_cluster_mappings = {
+    "oletools": "vbap",
+    "pcodedmp": "vbap",
+}
+
+requirement_clusters = {
+    "vbap": [("oletools", "oletools==0.60.1     --hash=sha256:67a796da4c4b8e2feb9a6b2495bef8798a3323a75512de4e5669d9dc9d1fae31     --hash=sha256:edef92374e688989a39269eb9a11142fb20a023629c23538c849c14d1d1144ea"), ("pcodedmp", "pcodedmp==1.2.6     --hash=sha256:025f8c809a126f45a082ffa820893e6a8d990d9d7ddb68694b5a9f0a6dbcd955     --hash=sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278")],
+}
+
+_config = {
+    "download_only": False,
+    "enable_implicit_namespace_pkgs": False,
+    "environment": {},
+    "extra_pip_args": [],
+    "isolated": True,
+    "pip_data_exclude": [],
+    "python_interpreter": "python3",
+    "python_interpreter_target": interpreter,
+    "quiet": True,
+    "repo": "pip",
+    "repo_prefix": "pip_",
+    "timeout": 600,
+}
+
+_annotations = {
+}
+
+# buildifier: disable=function-docstring
 def _clean_name(name):
     return name.replace("-", "_").replace(".", "_").lower()
 
-def requirement(name):
-    return "@pip_" + _clean_name(name) + "//:pkg"
+# buildifier: disable=function-docstring
+def requirement(name, use_clusters = True):
+    cname = _clean_name(name)
+    if cname in _cluster_mappings and use_clusters:
+        return "@pip//:_cluster_{}_dep_{}".format(_cluster_mappings[cname], cname)
+    else:
+        return "@pip_" + cname + "//:pkg"
 
-def whl_requirement(name):
-    return "@pip_" + _clean_name(name) + "//:whl"
+# buildifier: disable=function-docstring
+def whl_requirement(name, use_clusters = True):
+    cname = _clean_name(name)
+    if cname in _cluster_mappings and use_clusters:
+        return "@pip//:whl_" + _cluster_mappings[cname]
+    return "@pip_" + cname + "//:whl"
 
+# buildifier: disable=function-docstring
 def data_requirement(name):
     return "@pip_" + _clean_name(name) + "//:data"
 
+# buildifier: disable=function-docstring
 def dist_info_requirement(name):
     return "@pip_" + _clean_name(name) + "//:dist_info"
 
+# buildifier: disable=function-docstring
 def entry_point(pkg, script = None):
     if not script:
         script = pkg
     return "@pip_" + _clean_name(pkg) + "//:rules_python_wheel_entry_point_" + script
 
+# buildifier: disable=function-docstring
 def _get_annotation(requirement):
     # This expects to parse `setuptools==58.2.0     --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11`
     # down to `setuptools`.
     name = requirement.split(" ")[0].split("=")[0].split("[")[0]
     return _annotations.get(name)
 
+# buildifier: disable=function-docstring
 def install_deps(**whl_library_kwargs):
     whl_config = dict(_config)
     whl_config.update(whl_library_kwargs)
-    for name, requirement in _packages:
+
+    # Install normal requirements
+    for name, spec in _packages:
         whl_library(
             name = name,
-            requirement = requirement,
-            annotation = _get_annotation(requirement),
+            requirement = spec,
+            annotation = _get_annotation(spec),
             **whl_config
         )
+
+    # And deal with requirement_clusters
+    for components in requirement_clusters.values():
+        # Generate the component libraries
+        cnames = [c[0] for c in components]
+        for rname, spec in components:
+            name = "pip_" + rname
+            whl_library(
+                name = name,
+                requirement = spec,
+                annotation = _get_annotation(spec),
+                skip_deps = cnames,
+                **whl_config
+            )
diff --git a/examples/pip_parse_vendored/requirements.in b/examples/pip_parse_vendored/requirements.in
index f2293605cf..07f253a9e2 100644
--- a/examples/pip_parse_vendored/requirements.in
+++ b/examples/pip_parse_vendored/requirements.in
@@ -1 +1,3 @@
 requests
+pcodedmp
+oletools
diff --git a/examples/pip_parse_vendored/requirements.txt b/examples/pip_parse_vendored/requirements.txt
index ff1a3633a2..f75ce18d28 100644
--- a/examples/pip_parse_vendored/requirements.txt
+++ b/examples/pip_parse_vendored/requirements.txt
@@ -8,14 +8,138 @@ certifi==2022.12.7 \
     --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
     --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
     # via requests
+cffi==1.15.1 \
+    --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \
+    --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \
+    --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \
+    --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \
+    --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \
+    --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \
+    --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \
+    --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \
+    --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \
+    --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \
+    --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \
+    --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \
+    --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \
+    --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \
+    --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \
+    --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \
+    --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \
+    --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \
+    --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \
+    --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \
+    --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \
+    --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \
+    --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \
+    --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \
+    --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \
+    --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \
+    --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \
+    --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \
+    --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \
+    --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \
+    --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \
+    --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \
+    --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \
+    --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \
+    --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \
+    --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \
+    --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \
+    --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \
+    --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \
+    --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \
+    --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \
+    --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \
+    --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \
+    --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \
+    --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \
+    --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \
+    --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \
+    --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \
+    --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \
+    --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \
+    --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \
+    --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \
+    --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \
+    --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \
+    --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \
+    --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \
+    --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \
+    --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \
+    --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \
+    --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \
+    --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \
+    --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \
+    --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \
+    --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0
+    # via cryptography
 charset-normalizer==2.1.1 \
     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \
     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f
     # via requests
+colorclass==2.2.2 \
+    --hash=sha256:6d4fe287766166a98ca7bc6f6312daf04a0481b1eda43e7173484051c0ab4366 \
+    --hash=sha256:6f10c273a0ef7a1150b1120b6095cbdd68e5cf36dfd5d0fc957a2500bbf99a55
+    # via oletools
+cryptography==40.0.2 \
+    --hash=sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440 \
+    --hash=sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288 \
+    --hash=sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b \
+    --hash=sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958 \
+    --hash=sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b \
+    --hash=sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d \
+    --hash=sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a \
+    --hash=sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404 \
+    --hash=sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b \
+    --hash=sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e \
+    --hash=sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2 \
+    --hash=sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c \
+    --hash=sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b \
+    --hash=sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9 \
+    --hash=sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b \
+    --hash=sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636 \
+    --hash=sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99 \
+    --hash=sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e \
+    --hash=sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9
+    # via msoffcrypto-tool
+easygui==0.98.3 \
+    --hash=sha256:33498710c68b5376b459cd3fc48d1d1f33822139eb3ed01defbc0528326da3ba \
+    --hash=sha256:d653ff79ee1f42f63b5a090f2f98ce02335d86ad8963b3ce2661805cafe99a04
+    # via oletools
 idna==3.4 \
     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \
     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2
     # via requests
+msoffcrypto-tool==5.0.1 \
+    --hash=sha256:2b489c8a2b13bec07b94c8f5ce9054111dec3223ff8bedfd486cae3c299be54b \
+    --hash=sha256:9efd0ef5cc3e086e2d175e7a5d7b2b8cb59836c896b8a486d362bbca166db645
+    # via oletools
+olefile==0.46 \
+    --hash=sha256:133b031eaf8fd2c9399b78b8bc5b8fcbe4c31e85295749bb17a87cba8f3c3964
+    # via
+    #   msoffcrypto-tool
+    #   oletools
+oletools==0.60.1 \
+    --hash=sha256:67a796da4c4b8e2feb9a6b2495bef8798a3323a75512de4e5669d9dc9d1fae31 \
+    --hash=sha256:edef92374e688989a39269eb9a11142fb20a023629c23538c849c14d1d1144ea
+    # via
+    #   -r requirements.in
+    #   pcodedmp
+pcodedmp==1.2.6 \
+    --hash=sha256:025f8c809a126f45a082ffa820893e6a8d990d9d7ddb68694b5a9f0a6dbcd955 \
+    --hash=sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278
+    # via
+    #   -r requirements.in
+    #   oletools
+pycparser==2.21 \
+    --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
+    --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
+    # via cffi
+pyparsing==2.4.7 \
+    --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
+    --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
+    # via oletools
 requests==2.28.1 \
     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \
     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349
diff --git a/examples/pip_parse_vendored/test.py b/examples/pip_parse_vendored/test.py
new file mode 100644
index 0000000000..6457c292e6
--- /dev/null
+++ b/examples/pip_parse_vendored/test.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+import oletools
+import pcodedmp
+import requests
+
+print(oletools.__file__)
+print(pcodedmp.__file__)
+print(requests.__file__)
+
+exit(0)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 5462f1b14d..e960a11bab 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -226,13 +226,6 @@ def _create_repository_execution_environment(rctx):
 
     return env
 
-_BUILD_FILE_CONTENTS = """\
-package(default_visibility = ["//visibility:public"])
-
-# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
-exports_files(["requirements.bzl"])
-"""
-
 def locked_requirements_label(ctx, attr):
     """Get the preferred label for a locked requirements file based on platform.
 
@@ -360,12 +353,12 @@ def _pip_repository_bzlmod_impl(rctx):
 
     repo_name = rctx.attr.name.split("~")[-1]
 
-    build_contents = _BUILD_FILE_CONTENTS
+    footer = ""
 
     if rctx.attr.incompatible_generate_aliases:
         _pkg_aliases(rctx, repo_name, bzl_packages)
     else:
-        build_contents += _bzlmod_pkg_aliases(repo_name, bzl_packages)
+        footer = _bzlmod_pkg_aliases(repo_name, bzl_packages)
 
     # NOTE: we are using the canonical name with the double '@' in order to
     # always uniquely identify a repository, as the labels are being passed as
@@ -376,8 +369,10 @@ def _pip_repository_bzlmod_impl(rctx):
     else:
         macro_tmpl = "@@{name}//:{{}}_{{}}".format(name = rctx.attr.name)
 
-    rctx.file("BUILD.bazel", build_contents)
-    rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
+    rctx.template("BUILD.bazel", rctx.attr._build_template, substitutions = {
+        "%%FOOTER%%": footer,
+    })
+    rctx.template("requirements.bzl", rctx.attr._requirements_template, substitutions = {
         "%%ALL_REQUIREMENTS%%": _format_repr_list([
             "@{}//{}".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p)
             for p in bzl_packages
@@ -415,7 +410,10 @@ wheels are fetched/built only for the targets specified by 'build/run/test'.
         allow_single_file = True,
         doc = "Override the requirements_lock attribute when the host platform is Windows",
     ),
-    "_template": attr.label(
+    "_build_template": attr.label(
+        default = ":pip_repository_build_bzlmod.bazel.tmpl",
+    ),
+    "_requirements_template": attr.label(
         default = ":pip_repository_requirements_bzlmod.bzl.tmpl",
     ),
 }
@@ -431,14 +429,44 @@ def _pip_repository_impl(rctx):
     content = rctx.read(requirements_txt)
     parsed_requirements_txt = parse_requirements(content)
 
-    packages = [(_clean_pkg_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements]
+    # Apply name normalizations to the composite libs def once
+    composite_libs = {
+        _clean_pkg_name(name): [_clean_pkg_name(it) for it in components]
+        for name, components in rctx.attr.composite_libs.items()
+    }
 
-    bzl_packages = sorted([name for name, _ in packages])
+    # Ditto for requirements defs
+    requirements = {
+        _clean_pkg_name(name): requirement
+        for name, requirement in parsed_requirements_txt.requirements
+    }
+
+    # Map normalized package names to a composite
+    composite_mapping = {
+        name: composite_name
+        for composite_name, names in composite_libs.items()
+        for name in names
+    }
 
-    imports = [
-        'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")',
+    # Normal packages are defined by a single requirement.
+    # We will deal with composites shortly.
+    normal_packages = [
+        (name, requirement)
+        for name, requirement in requirements.items()
+        if name not in composite_mapping
     ]
 
+    # Composite packages are a cluster which can only be depended on together
+    composite_packages = {
+        _clean_pkg_name(composite_name): [
+            (rname, requirements[rname])
+            for rname in composite_components
+        ]
+        for composite_name, composite_components in rctx.attr.composite_libs.items()
+    }
+
+    bzl_packages = sorted([name for name, _ in requirements.items()])
+
     annotations = {}
     for pkg, annotation in rctx.attr.annotations.items():
         filename = "{}.annotation.json".format(_clean_pkg_name(pkg))
@@ -472,8 +500,14 @@ def _pip_repository_impl(rctx):
     if rctx.attr.incompatible_generate_aliases:
         _pkg_aliases(rctx, rctx.attr.name, bzl_packages)
 
-    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
-    rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
+    rctx.template("lib.bzl", rctx.attr._lib_template, substitutions = {
+        "%%NAME%%": rctx.attr.name,
+    })
+    rctx.template("BUILD.bazel", rctx.attr._build_template, substitutions = {
+        "%%FOOTER%%": "",
+        "%%NAME%%": rctx.attr.name,
+    })
+    rctx.template("requirements.bzl", rctx.attr._requirements_template, substitutions = {
         "%%ALL_REQUIREMENTS%%": _format_repr_list([
             "@{}//{}".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p)
             for p in bzl_packages
@@ -483,21 +517,20 @@ def _pip_repository_impl(rctx):
             for p in bzl_packages
         ]),
         "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)),
+        "%%CLUSTERS%%": _format_dict(_repr_dict(composite_packages)),
+        "%%CLUSTER_MAPPINGS%%": _format_dict(_repr_dict(composite_mapping)),
         "%%CONFIG%%": _format_dict(_repr_dict(config)),
         "%%EXTRA_PIP_ARGS%%": json.encode(options),
-        "%%IMPORTS%%": "\n".join(sorted(imports)),
         "%%NAME%%": rctx.attr.name,
         "%%PACKAGES%%": _format_repr_list(
             [
                 ("{}_{}".format(rctx.attr.name, p), r)
-                for p, r in packages
+                for p, r in normal_packages
             ],
         ),
         "%%REQUIREMENTS_LOCK%%": str(requirements_txt),
     })
 
-    return
-
 common_env = [
     "RULES_PYTHON_PIP_ISOLATED",
 ]
@@ -587,6 +620,9 @@ pip_repository_attrs = {
     "annotations": attr.string_dict(
         doc = "Optional annotations to apply to packages",
     ),
+    "composite_libs": attr.string_list_dict(
+        doc = "Groups of requirements which represent dependency cycles and must be treated as composites.",
+    ),
     "incompatible_generate_aliases": attr.bool(
         default = False,
         doc = "Allow generating aliases '@pip//' -> '@pip_//:pkg'.",
@@ -611,7 +647,13 @@ wheels are fetched/built only for the targets specified by 'build/run/test'.
         allow_single_file = True,
         doc = "Override the requirements_lock attribute when the host platform is Windows",
     ),
-    "_template": attr.label(
+    "_build_template": attr.label(
+        default = ":pip_repository_build.bazel.tmpl",
+    ),
+    "_lib_template": attr.label(
+        default = ":pip_repository_lib.bzl.tmpl",
+    ),
+    "_requirements_template": attr.label(
         default = ":pip_repository_requirements.bzl.tmpl",
     ),
 }
@@ -682,6 +724,8 @@ def _whl_library_impl(rctx):
             "--annotation",
             rctx.path(rctx.attr.annotation),
         ])
+    for d in rctx.attr.skip_deps:
+        args.extend(["--skip", d])
 
     args = _parse_optional_attrs(rctx, args)
 
@@ -714,6 +758,10 @@ whl_library_attrs = {
         mandatory = True,
         doc = "Python requirement string describing the package to make available",
     ),
+    "skip_deps": attr.string_list(
+        doc = "List of requirements to skip due to clustering",
+        default = [],
+    ),
 }
 
 whl_library_attrs.update(**common_attrs)
@@ -764,7 +812,7 @@ def package_annotation(
 # pip_repository implementation
 
 def _format_list(items):
-    return "[{}]".format(", ".join(items))
+    return "[\n{}]".format("".join(["    {},\n".format(it) for it in items]))
 
 def _format_repr_list(strings):
     return _format_list(
@@ -775,4 +823,4 @@ def _repr_dict(items):
     return {k: repr(v) for k, v in items.items()}
 
 def _format_dict(items):
-    return "{{{}}}".format(", ".join(sorted(['"{}": {}'.format(k, v) for k, v in items.items()])))
+    return "{{\n{}}}".format("".join(sorted(['    "{}": {},\n'.format(k, v) for k, v in items.items()])))
diff --git a/python/pip_install/pip_repository_build.bazel.tmpl b/python/pip_install/pip_repository_build.bazel.tmpl
new file mode 100644
index 0000000000..2ee7c1e012
--- /dev/null
+++ b/python/pip_install/pip_repository_build.bazel.tmpl
@@ -0,0 +1,10 @@
+load(":lib.bzl", "install_clusters")
+
+package(default_visibility = ["//visibility:public"])
+
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+
+install_clusters()
+
+%%FOOTER%%
diff --git a/python/pip_install/pip_repository_build_bzlmod.bazel.tmpl b/python/pip_install/pip_repository_build_bzlmod.bazel.tmpl
new file mode 100644
index 0000000000..029bca2051
--- /dev/null
+++ b/python/pip_install/pip_repository_build_bzlmod.bazel.tmpl
@@ -0,0 +1,6 @@
+package(default_visibility = ["//visibility:public"])
+
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+
+%%FOOTER%%
diff --git a/python/pip_install/pip_repository_lib.bzl.tmpl b/python/pip_install/pip_repository_lib.bzl.tmpl
new file mode 100644
index 0000000000..46145ec282
--- /dev/null
+++ b/python/pip_install/pip_repository_lib.bzl.tmpl
@@ -0,0 +1,21 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("//:requirements.bzl", "requirement", "whl_requirement", "requirement_clusters")
+
+def install_clusters():
+    for cluster_name, components in requirement_clusters.items():
+        py_library(
+            name = cluster_name,
+            deps = [requirement(c, use_clusters=False) for c, _ in components]
+        )
+
+        for r in components:
+            requirement_name = r[0]
+            native.alias(
+                name = "_cluster_{}_dep_{}".format(cluster_name, requirement_name),
+                actual = cluster_name,
+            )
+
+        native.filegroup(
+            name = "whl_" + cluster_name,
+            data = [whl_requirement(c, use_clusters=False) for c, _ in components]
+        )
diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl
index bf6a053622..42c86d0447 100644
--- a/python/pip_install/pip_repository_requirements.bzl.tmpl
+++ b/python/pip_install/pip_repository_requirements.bzl.tmpl
@@ -4,24 +4,37 @@
 from %%REQUIREMENTS_LOCK%%
 """
 
-%%IMPORTS%%
+load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
 
 all_requirements = %%ALL_REQUIREMENTS%%
 
 all_whl_requirements = %%ALL_WHL_REQUIREMENTS%%
 
 _packages = %%PACKAGES%%
+
+_cluster_mappings = %%CLUSTER_MAPPINGS%%
+
+requirement_clusters = %%CLUSTERS%%
+
 _config = %%CONFIG%%
+
 _annotations = %%ANNOTATIONS%%
 
 def _clean_name(name):
     return name.replace("-", "_").replace(".", "_").lower()
 
-def requirement(name):
-    return "@%%NAME%%_" + _clean_name(name) + "//:pkg"
+def requirement(name, use_clusters = True):
+    cname = _clean_name(name)
+    if cname in _cluster_mappings and use_clusters:
+        return "@%%NAME%%//:_cluster_{}_dep_{}".format(_cluster_mappings[cname], cname)
+    else:
+        return "@%%NAME%%_" + cname + "//:pkg"
 
-def whl_requirement(name):
-    return "@%%NAME%%_" + _clean_name(name) + "//:whl"
+def whl_requirement(name, use_clusters = True):
+    cname = _clean_name(name)
+    if cname in _cluster_mappings and use_clusters:
+        return "@%%NAME%%//:whl_" + _cluster_mappings[cname]
+    return "@%%NAME%%_" + cname + "//:whl"
 
 def data_requirement(name):
     return "@%%NAME%%_" + _clean_name(name) + "//:data"
@@ -43,10 +56,26 @@ def _get_annotation(requirement):
 def install_deps(**whl_library_kwargs):
     whl_config = dict(_config)
     whl_config.update(whl_library_kwargs)
-    for name, requirement in _packages:
+
+    # Install normal requirements
+    for name, spec in _packages:
         whl_library(
             name = name,
-            requirement = requirement,
-            annotation = _get_annotation(requirement),
+            requirement = spec,
+            annotation = _get_annotation(spec),
             **whl_config
         )
+
+    # And deal with requirement_clusters
+    for components in requirement_clusters.values():
+        # Generate the component libraries
+        cnames = [c[0] for c in components]
+        for rname, spec in components:
+            name = "%%NAME%%_" + rname
+            whl_library(
+                name = name,
+                requirement = spec,
+                annotation = _get_annotation(spec),
+                skip_deps = cnames,
+                **whl_config
+            )
diff --git a/python/pip_install/tools/dependency_resolver/__init__.py b/python/pip_install/tools/dependency_resolver/__init__.py
index bbdfb4c588..41010956cf 100644
--- a/python/pip_install/tools/dependency_resolver/__init__.py
+++ b/python/pip_install/tools/dependency_resolver/__init__.py
@@ -11,4 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
diff --git a/python/pip_install/tools/lib/__init__.py b/python/pip_install/tools/lib/__init__.py
index bbdfb4c588..41010956cf 100644
--- a/python/pip_install/tools/lib/__init__.py
+++ b/python/pip_install/tools/lib/__init__.py
@@ -11,4 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
diff --git a/python/pip_install/tools/lib/annotation.py b/python/pip_install/tools/lib/annotation.py
index c98008005e..eafbdf7490 100644
--- a/python/pip_install/tools/lib/annotation.py
+++ b/python/pip_install/tools/lib/annotation.py
@@ -23,7 +23,6 @@ class Annotation(OrderedDict):
     """A python representation of `@rules_python//python:pip.bzl%package_annotation`"""
 
     def __init__(self, content: Dict[str, Any]) -> None:
-
         missing = []
         ordered_content = OrderedDict()
         for field in (
diff --git a/python/pip_install/tools/lib/annotations_test.py b/python/pip_install/tools/lib/annotations_test.py
index f7c360fbc9..8c7ba6485d 100644
--- a/python/pip_install/tools/lib/annotations_test.py
+++ b/python/pip_install/tools/lib/annotations_test.py
@@ -24,7 +24,6 @@
 
 
 class AnnotationsTestCase(unittest.TestCase):
-
     maxDiff = None
 
     def test_annotations_constructor(self) -> None:
diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py
index 77aa3a406c..44fda351de 100644
--- a/python/pip_install/tools/wheel_installer/wheel_installer.py
+++ b/python/pip_install/tools/wheel_installer/wheel_installer.py
@@ -195,6 +195,7 @@ def _generate_copy_commands(src, dest, is_executable=False) -> str:
 
 def _generate_build_file_contents(
     name: str,
+    repo_prefix: str,
     dependencies: List[str],
     whl_file_deps: List[str],
     data_exclude: List[str],
@@ -241,6 +242,7 @@ def _generate_build_file_contents(
                 """\
         load("@rules_python//python:defs.bzl", "py_library", "py_binary")
         load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+        load("@{repo_prefix}//:requirements.bzl", "requirement", "whl_requirement")
 
         package(default_visibility = ["//visibility:public"])
 
@@ -272,6 +274,7 @@ def _generate_build_file_contents(
         )
         """.format(
                     name=name,
+                    repo_prefix=repo_prefix.rstrip("_"),
                     dependencies=",".join(sorted(dependencies)),
                     data_exclude=json.dumps(sorted(data_exclude)),
                     whl_file_label=bazel.WHEEL_FILE_LABEL,
@@ -297,6 +300,7 @@ def _extract_wheel(
     repo_prefix: str,
     installation_dir: Path = Path("."),
     annotation: Optional[annotation.Annotation] = None,
+    skip_deps: List[str] = [],
 ) -> None:
     """Extracts wheel into given directory and creates py_library and filegroup targets.
 
@@ -318,15 +322,21 @@ def _extract_wheel(
     extras_requested = extras[whl.name] if whl.name in extras else set()
     # Packages may create dependency cycles when specifying optional-dependencies / 'extras'.
     # Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32.
-    self_edge_dep = set([whl.name])
-    whl_deps = sorted(whl.dependencies(extras_requested) - self_edge_dep)
+    to_skip = {bazel.sanitise_name(it, "") for it in [whl.name] + skip_deps}
+    deps = {bazel.sanitise_name(it, "") for it in whl.dependencies(extras_requested)}
+    whl_deps = sorted(deps - to_skip)
+    print(
+        "While building %s\n\tDeps: %r\n\tSkipping: %r\n\tEffective: %r"
+        % (
+            whl.name,
+            deps,
+            to_skip,
+            whl_deps,
+        )
+    )
 
-    sanitised_dependencies = [
-        bazel.sanitised_repo_library_label(d, repo_prefix=repo_prefix) for d in whl_deps
-    ]
-    sanitised_wheel_file_dependencies = [
-        bazel.sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps
-    ]
+    sanitised_dependencies = ["requirement(%r)" % d for d in whl_deps]
+    sanitised_wheel_file_dependencies = ["whl_requirement(%r)" % d for d in whl_deps]
 
     entry_points = []
     for name, (module, attribute) in sorted(whl.entry_points().items()):
@@ -370,6 +380,7 @@ def _extract_wheel(
 
         contents = _generate_build_file_contents(
             name=bazel.PY_LIBRARY_LABEL,
+            repo_prefix=repo_prefix,
             dependencies=sanitised_dependencies,
             whl_file_deps=sanitised_wheel_file_dependencies,
             data_exclude=data_exclude,
@@ -396,6 +407,7 @@ def main() -> None:
         type=annotation.annotation_from_str_path,
         help="A json encoded file containing annotations for rendered packages.",
     )
+    parser.add_argument("--skip", action="append", dest="skip_deps", default=[])
     arguments.parse_common_args(parser)
     args = parser.parse_args()
     deserialized_args = dict(vars(args))
@@ -443,6 +455,7 @@ def main() -> None:
         enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
         repo_prefix=args.repo_prefix,
         annotation=args.annotation,
+        skip_deps=args.skip_deps,
     )