diff --git a/examples/pip_install/.bazelrc b/examples/pip_install/.bazelrc new file mode 100644 index 0000000000..b98fc09774 --- /dev/null +++ b/examples/pip_install/.bazelrc @@ -0,0 +1 @@ +test --test_output=errors diff --git a/examples/pip_install/BUILD b/examples/pip_install/BUILD index 0907052516..3a5fe336b5 100644 --- a/examples/pip_install/BUILD +++ b/examples/pip_install/BUILD @@ -1,4 +1,5 @@ load("@pip//:requirements.bzl", "requirement") +load("@rules_python//python/pip_install:pip_requirements.bzl", "pip_requirements") load("@rules_python//python:defs.bzl", "py_binary", "py_test") # Toolchain setup, this is optional. @@ -40,3 +41,6 @@ py_test( srcs = ["test.py"], deps = [":main"], ) + +# Check that our compiled requirements are up-to-date +pip_requirements() diff --git a/examples/pip_install/WORKSPACE b/examples/pip_install/WORKSPACE index b97f0c5412..c24704d329 100644 --- a/examples/pip_install/WORKSPACE +++ b/examples/pip_install/WORKSPACE @@ -8,6 +8,17 @@ http_archive( sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", ) +# Should not be here, workaround for +# https://github.com/bazelbuild/rules_python/issues/372 +http_archive( + name = "bazel_skylib", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + ], + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", +) + load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() diff --git a/examples/pip_install/requirements.in b/examples/pip_install/requirements.in new file mode 100644 index 0000000000..cbc5542582 --- /dev/null +++ b/examples/pip_install/requirements.in @@ -0,0 +1 @@ +boto3==1.14.51 diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index cbc5542582..4446e36d71 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -1 +1,39 @@ -boto3==1.14.51 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bazel run //:requirements.update +# +boto3==1.14.51 \ + --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \ + --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 \ + # via -r ./requirements.in +botocore==1.17.63 \ + --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ + --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 \ + # via boto3, s3transfer +docutils==0.15.2 \ + --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \ + --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \ + --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \ + # via botocore +jmespath==0.10.0 \ + --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ + --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \ + # via boto3, botocore +python-dateutil==2.8.1 \ + --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ + --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \ + # via botocore +s3transfer==0.3.3 \ + --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \ + --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \ + # via boto3 +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ + # via python-dateutil +urllib3==1.25.11 \ + --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ + --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \ + # via botocore diff --git a/python/BUILD b/python/BUILD index 124ddd08d7..5b7f60a846 100644 --- a/python/BUILD +++ b/python/BUILD @@ -23,6 +23,7 @@ In an ideal renaming, we'd move the packaging rules to a different package so that @rules_python//python is only concerned with the core rules. """ +load("//python/pip_install:pip_requirements.bzl", "pip_requirements") package(default_visibility = ["//visibility:public"]) @@ -130,3 +131,5 @@ exports_files([ "pip.bzl", "whl.bzl", ]) + +pip_requirements(extra_args = ["--allow-unsafe"]) diff --git a/python/pip_install/BUILD b/python/pip_install/BUILD index b37170eb53..bfd2d2026a 100644 --- a/python/pip_install/BUILD +++ b/python/pip_install/BUILD @@ -1,9 +1,12 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +exports_files(["pip_compile.py"]) + filegroup( name = "distribution", srcs = glob(["*.bzl"]) + [ "BUILD", + "pip_compile.py", "//python/pip_install/extract_wheels:distribution", ], visibility = ["//:__pkg__"], diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py new file mode 100644 index 0000000000..61b5d93df9 --- /dev/null +++ b/python/pip_install/pip_compile.py @@ -0,0 +1,81 @@ +"Set defaults for the pip-compile command to run it under Bazel" + +import os +import sys +from shutil import copyfile + +from piptools.scripts.compile import cli + +if len(sys.argv) < 4: + print( + "Expected at least two arguments: requirements_in requirements_out", + file=sys.stderr, + ) + sys.exit(1) + +requirements_in = sys.argv.pop(1) +requirements_txt = sys.argv.pop(1) +update_target_name = sys.argv.pop(1) + +UPDATE = True +# Detect if we are running under `bazel test` +if "TEST_TMPDIR" in os.environ: + UPDATE = False + # pip-compile wants the cache files to be writeable, but if we point + # to the real user cache, Bazel sandboxing makes the file read-only + # and we fail. + # In theory this makes the test more hermetic as well. + sys.argv.append("--cache-dir") + sys.argv.append(os.environ["TEST_TMPDIR"]) + # Make a copy for pip-compile to read and mutate + requirements_out = os.path.join( + os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" + ) + copyfile(requirements_txt, requirements_out) + +elif "BUILD_WORKING_DIRECTORY" in os.environ: + os.chdir(os.environ['BUILD_WORKING_DIRECTORY']) +else: + print( + "Expected to find BUILD_WORKING_DIRECTORY in environment", + file=sys.stderr, + ) + sys.exit(1) + +update_target_pkg = "/".join(requirements_in.split('/')[:-1]) +# $(rootpath) in the workspace root gives ./requirements.in +if update_target_pkg == ".": + update_target_pkg = "" +update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name) + +os.environ["CUSTOM_COMPILE_COMMAND"] = update_command + +sys.argv.append("--generate-hashes") +sys.argv.append("--output-file") +sys.argv.append(requirements_txt if UPDATE else requirements_out) +sys.argv.append(requirements_in) + +if UPDATE: + print("Updating " + requirements_txt) + cli() +else: + # cli will exit(0) on success + try: + print("Checking " + requirements_txt) + cli() + print("cl() should exit", file=sys.stderr) + sys.exit(1) + except SystemExit: + golden = open(requirements_txt).readlines() + out = open(requirements_out).readlines() + if golden != out: + import difflib + + print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr) + print( + "Lock file out of date. Run '" + + update_command + + "' to update.", + file=sys.stderr, + ) + sys.exit(1) diff --git a/python/pip_install/pip_requirements.bzl b/python/pip_install/pip_requirements.bzl new file mode 100644 index 0000000000..16851332ce --- /dev/null +++ b/python/pip_install/pip_requirements.bzl @@ -0,0 +1,70 @@ +"Rules to verify and update pip-compile locked requirements.txt" + +load("//python:defs.bzl", "py_binary", "py_test") +load("//python/pip_install:repositories.bzl", "requirement") + +def pip_requirements( + name = "requirements", + extra_args = [], + **kwargs): + """ + Produce two targets for checking pip-compile: + + - validate with `bazel test _test` + - update with `bazel run .update` + + By default requirements in file is expected to be .in and output + requirements txt lock file is expected to be .txt. These may be customized + with `requirements_in` and `requirements_locked` params. + + Args: + name: string + extra_args: passed to pip-compile + **kwargs: other bazel attributes + """ + requirements_in = kwargs.pop("requirements_in", name + ".in") + requirements_txt = kwargs.pop("requirements_locked", name + ".txt") + + data = kwargs.pop("data", []) + [requirements_in, requirements_txt] + + loc = "$(rootpath %s)" + + # Use the Label constructor so this is expanded in the context of the file + # where it appears, which is to say, in @rules_python + pip_compile = Label("//python/pip_install:pip_compile.py") + + args = [ + loc % requirements_in, + loc % requirements_txt, + name + ".update", + ] + extra_args + + py_binary( + name = name + ".update", + srcs = [pip_compile], + main = pip_compile, + args = args, + visibility = ["//visibility:public"], + deps = [ + requirement("pip"), + requirement("pip_tools"), + ], + data = data, + ) + + timeout = kwargs.pop("timeout", "short") + + py_test( + name = name + "_test", + srcs = [pip_compile], + main = pip_compile, + args = args, + visibility = ["//visibility:public"], + deps = [ + requirement("pip"), + requirement("pip_tools"), + ], + data = data, + timeout = timeout, + **kwargs + ) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index df6367484f..b70b0cda90 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,8 +6,13 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl", - "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7", + "https://files.pythonhosted.org/packages/43/84/23ed6a1796480a6f1a2d38f2802901d078266bda38388954d01d3f2e821d/pip-20.1.1-py2.py3-none-any.whl", + "b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4", + ), + ( + "pypi__pip_tools", + "https://files.pythonhosted.org/packages/a1/d5/c0f282060c483e6fd4635ed8f4f88aff02c5490d0fc588edec52b0cb9d7b/pip_tools-5.3.1-py2.py3-none-any.whl", + "73787e23269bf8a9230f376c351297b9037ed0d32ab0f9bef4a187d976acc054", ), ( "pypi__pkginfo", diff --git a/python/requirements.in b/python/requirements.in new file mode 100644 index 0000000000..164fa2650c --- /dev/null +++ b/python/requirements.in @@ -0,0 +1,6 @@ +pip==9.0.3 +setuptools==44.0.0 +wheel==0.30.0a0 + +# For tests +mock==2.0.0 diff --git a/python/requirements.txt b/python/requirements.txt index 164fa2650c..bcacfb020f 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,6 +1,32 @@ -pip==9.0.3 -setuptools==44.0.0 -wheel==0.30.0a0 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bazel run //python:requirements.update +# +mock==2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ + --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba \ + # via -r python/requirements.in +pbr==5.5.1 \ + --hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \ + --hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00 \ + # via mock +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ + # via mock +wheel==0.30.0a0 \ + --hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \ + --hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0 \ + # via -r python/requirements.in -# For tests -mock==2.0.0 +# The following packages are considered to be unsafe in a requirements file: +pip==9.0.3 \ + --hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \ + --hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5 \ + # via -r python/requirements.in +setuptools==44.0.0 \ + --hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \ + --hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008 \ + # via -r python/requirements.in