diff --git a/examples/pip_install/.bazelrc b/examples/pip_install/.bazelrc new file mode 100644 index 0000000000..b98fc09774 --- /dev/null +++ b/examples/pip_install/.bazelrc @@ -0,0 +1 @@ +test --test_output=errors diff --git a/examples/pip_install/BUILD b/examples/pip_install/BUILD index 0907052516..c8fbc0bbee 100644 --- a/examples/pip_install/BUILD +++ b/examples/pip_install/BUILD @@ -1,5 +1,6 @@ load("@pip//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_test") +load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements") # Toolchain setup, this is optional. # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE). @@ -40,3 +41,8 @@ py_test( srcs = ["test.py"], deps = [":main"], ) + +# Check that our compiled requirements are up-to-date +compile_pip_requirements( + name = "requirements", +) diff --git a/examples/pip_install/requirements.in b/examples/pip_install/requirements.in new file mode 100644 index 0000000000..cbc5542582 --- /dev/null +++ b/examples/pip_install/requirements.in @@ -0,0 +1 @@ +boto3==1.14.51 diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index cbc5542582..6e8b2320bd 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -1 +1,43 @@ -boto3==1.14.51 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bazel run //:requirements.update +# +boto3==1.14.51 \ + --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \ + --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 + # via -r ./requirements.in +botocore==1.17.63 \ + --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ + --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 + # via + # boto3 + # s3transfer +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 + # via botocore +jmespath==0.10.0 \ + --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ + --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f + # via + # boto3 + # botocore +python-dateutil==2.8.1 \ + --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ + --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a + # via botocore +s3transfer==0.3.3 \ + --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \ + --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db + # via boto3 +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced + # via python-dateutil +urllib3==1.25.11 \ + --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ + --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e + # via botocore diff --git a/python/BUILD b/python/BUILD index d4b6e3e018..88ee4dc34f 100644 --- a/python/BUILD +++ b/python/BUILD @@ -24,6 +24,8 @@ In an ideal renaming, we'd move the packaging rules to a different package so that @rules_python//python is only concerned with the core rules. """ +load("//python/pip_install:requirements.bzl", "compile_pip_requirements") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 @@ -145,3 +147,8 @@ exports_files([ "pip.bzl", "whl.bzl", ]) + +compile_pip_requirements( + name = "requirements", + extra_args = ["--allow-unsafe"], +) diff --git a/python/pip_install/BUILD b/python/pip_install/BUILD index 22f48f75fa..afcbcd4f17 100644 --- a/python/pip_install/BUILD +++ b/python/pip_install/BUILD @@ -1,7 +1,10 @@ +exports_files(["pip_compile.py"]) + filegroup( name = "distribution", srcs = glob(["*.bzl"]) + [ "BUILD", + "pip_compile.py", "//python/pip_install/extract_wheels:distribution", "//python/pip_install/parse_requirements_to_bzl:distribution", ], diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py new file mode 100644 index 0000000000..1d440a5572 --- /dev/null +++ b/python/pip_install/pip_compile.py @@ -0,0 +1,89 @@ +"Set defaults for the pip-compile command to run it under Bazel" + +import os +import sys +from shutil import copyfile + +from piptools.scripts.compile import cli + +if len(sys.argv) < 4: + print( + "Expected at least two arguments: requirements_in requirements_out", + file=sys.stderr, + ) + sys.exit(1) + +requirements_in = sys.argv.pop(1) +requirements_txt = sys.argv.pop(1) +update_target_name = sys.argv.pop(1) + +# Before loading click, set the locale for its parser. +# If it leaks through to the system setting, it may fail: +# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII +# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for +# mitigation steps. +os.environ["LC_ALL"] = "C.UTF-8" +os.environ["LANG"] = "C.UTF-8" + +UPDATE = True +# Detect if we are running under `bazel test` +if "TEST_TMPDIR" in os.environ: + UPDATE = False + # pip-compile wants the cache files to be writeable, but if we point + # to the real user cache, Bazel sandboxing makes the file read-only + # and we fail. + # In theory this makes the test more hermetic as well. + sys.argv.append("--cache-dir") + sys.argv.append(os.environ["TEST_TMPDIR"]) + # Make a copy for pip-compile to read and mutate + requirements_out = os.path.join( + os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" + ) + copyfile(requirements_txt, requirements_out) + +elif "BUILD_WORKING_DIRECTORY" in os.environ: + os.chdir(os.environ['BUILD_WORKING_DIRECTORY']) +else: + print( + "Expected to find BUILD_WORKING_DIRECTORY in environment", + file=sys.stderr, + ) + sys.exit(1) + +update_target_pkg = "/".join(requirements_in.split('/')[:-1]) +# $(rootpath) in the workspace root gives ./requirements.in +if update_target_pkg == ".": + update_target_pkg = "" +update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name) + +os.environ["CUSTOM_COMPILE_COMMAND"] = update_command + +sys.argv.append("--generate-hashes") +sys.argv.append("--output-file") +sys.argv.append(requirements_txt if UPDATE else requirements_out) +sys.argv.append(requirements_in) + +if UPDATE: + print("Updating " + requirements_txt) + cli() +else: + # cli will exit(0) on success + try: + print("Checking " + requirements_txt) + cli() + print("cli() should exit", file=sys.stderr) + sys.exit(1) + except SystemExit: + golden = open(requirements_txt).readlines() + out = open(requirements_out).readlines() + if golden != out: + import difflib + + print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr) + print( + "Lock file out of date. Run '" + + update_command + + "' to update.", + file=sys.stderr, + ) + sys.exit(1) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 57d15f3480..302ff0ef3b 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -4,11 +4,21 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ + ( + "pypi__click", + "https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl", + "dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc", + ), ( "pypi__pip", "https://files.pythonhosted.org/packages/fe/ef/60d7ba03b5c442309ef42e7d69959f73aacccd0d86008362a681c4698e83/pip-21.0.1-py3-none-any.whl", "37fd50e056e2aed635dec96594606f0286640489b0db0ce7607f7e51890372d5", ), + ( + "pypi__pip_tools", + "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl", + "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985", + ), ( "pypi__pkginfo", "https://files.pythonhosted.org/packages/4f/3c/535287349af1b117e082f8e77feca52fbe2fdf61ef1e6da6bcc2a72a3a79/pkginfo-1.6.1-py2.py3-none-any.whl", diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl new file mode 100644 index 0000000000..b6a6be46e0 --- /dev/null +++ b/python/pip_install/requirements.bzl @@ -0,0 +1,94 @@ +"Rules to verify and update pip-compile locked requirements.txt" + +load("//python:defs.bzl", "py_binary", "py_test") +load("//python/pip_install:repositories.bzl", "requirement") + +def compile_pip_requirements( + name, + extra_args = [], + visibility = ["//visibility:private"], + requirements_in = "requirements.in", + requirements_txt = "requirements.txt", + **kwargs): + """ + Macro creating targets for running pip-compile + + Produce a filegroup by default, named "[name]" which can be included in the data + of some other compile_pip_requirements rule that references these requirements + (e.g. with `-r ../other/requirements.txt`) + + Produce two targets for checking pip-compile: + + - validate with `bazel test _test` + - update with `bazel run .update` + + Args: + name: base name for generated targets, typically "requirements" + extra_args: passed to pip-compile + visibility: passed to both the _test and .update rules + requirements_in: file expressing desired dependencies + requirements_txt: result of "compiling" the requirements.in file + **kwargs: other bazel attributes passed to the "_test" rule + """ + requirements_in = kwargs.pop("requirements_in", name + ".in") + requirements_txt = kwargs.pop("requirements_locked", name + ".txt") + + # "Default" target produced by this macro + # Allow a compile_pip_requirements rule to include another one in the data + # for a requirements file that does `-r ../other/requirements.txt` + native.filegroup( + name = name, + srcs = kwargs.pop("data", []) + [requirements_txt], + visibility = visibility, + ) + + data = [name, requirements_in, requirements_txt] + + # Use the Label constructor so this is expanded in the context of the file + # where it appears, which is to say, in @rules_python + pip_compile = Label("//python/pip_install:pip_compile.py") + + loc = "$(rootpath %s)" + + args = [ + loc % requirements_in, + loc % requirements_txt, + name + ".update", + ] + extra_args + + deps = [ + requirement("click"), + requirement("pip"), + requirement("pip_tools"), + requirement("setuptools"), + ] + + attrs = { + "args": args, + "data": data, + "deps": deps, + "main": pip_compile, + "srcs": [pip_compile], + "visibility": visibility, + } + + # cheap way to detect the bazel version + _bazel_version_4_or_greater = "propeller_optimize" in dir(native) + + # Bazel 4.0 added the "env" attribute to py_test/py_binary + if _bazel_version_4_or_greater: + attrs["env"] = kwargs.pop("env", {}) + + py_binary( + name = name + ".update", + **attrs + ) + + timeout = kwargs.pop("timeout", "short") + + py_test( + name = name + "_test", + timeout = timeout, + # kwargs could contain test-specific attributes like size or timeout + **dict(attrs, **kwargs) + ) diff --git a/python/requirements.in b/python/requirements.in new file mode 100644 index 0000000000..cae7f2925c --- /dev/null +++ b/python/requirements.in @@ -0,0 +1,10 @@ +pip==9.0.3 +setuptools==44.0.0 +wheel==0.30.0a0 + +# For tests +mock==2.0.0 +# This is a transitive dependency of mock, which only appears on some python versions +# Right now our repo doesn't pin our Python interpreter, so our locked requirements +# may differ between local dev and CI, for example. +funcsigs==1.0.2 diff --git a/python/requirements.txt b/python/requirements.txt index 164fa2650c..9bba20d1b3 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,6 +1,36 @@ -pip==9.0.3 -setuptools==44.0.0 -wheel==0.30.0a0 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bazel run //python:requirements.update +# +funcsigs==1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ + --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 + # via -r python/requirements.in +mock==2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ + --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba + # via -r python/requirements.in +pbr==5.5.1 \ + --hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \ + --hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00 + # via mock +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced + # via mock +wheel==0.30.0a0 \ + --hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \ + --hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0 + # via -r python/requirements.in -# For tests -mock==2.0.0 +# The following packages are considered to be unsafe in a requirements file: +pip==9.0.3 \ + --hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \ + --hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5 + # via -r python/requirements.in +setuptools==44.0.0 \ + --hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \ + --hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008 + # via -r python/requirements.in