From 51b1ba8ac3bd1b97f9145af696ecbd2590e7204f Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 4 Nov 2020 09:36:43 -0800 Subject: [PATCH 01/10] rust_doc: strip directory prefix from archive names The Zip archives emitted by `rust_doc` rules used to contain a long `/bazel-bin/k8-fastbuild/path/to/target` prefix before each archive entry name. This made it hard to work with the generated archive. This patch adds a transformation to This could be written in Starlark using the `DirectoryExapnder` API, but that was only introduced in Bazel 3.4.0. Since we have a minimum Bazel version of 0.17.1, we instead write the helper as a small Python script. Fixes #471. Test Plan: Unit tests included. As an end-to-end test, run ``` cd examples/ && bazel build //hello_world:hello_world_doc && unzip -l bazel-bin/hello_world/hello_world_doc.zip ``` to list the contents of the built `hello_world_doc.zip` archive. Before this patch, the result was like: ``` Archive: bazel-bin/hello_world/hello_world_doc.zip Length Date Time Name --------- ---------- ----- ---- 0 2010-01-01 00:00 bazel-out/k8-fastbuild/bin/hello_world/hello_world_doc/.lock 1792 2010-01-01 00:00 bazel-out/k8-fastbuild/bin/hello_world/hello_world_doc/COPYRIGHT.txt 4421 2010-01-01 00:00 bazel-out/k8-fastbuild/bin/hello_world/hello_world_doc/FiraSans-LICENSE.txt ... 2132 2010-01-01 00:00 bazel-out/k8-fastbuild/bin/hello_world/hello_world_doc/storage.js 1180 2010-01-01 00:00 bazel-out/k8-fastbuild/bin/hello_world/hello_world_doc/theme.js 3764 2010-01-01 00:00 bazel-out/k8-fastbuild/bin/hello_world/hello_world_doc/wheel.svg --------- ------- 890555 38 files ``` After this patch: ``` Archive: bazel-bin/hello_world/hello_world_doc.zip Length Date Time Name --------- ---------- ----- ---- 0 2010-01-01 00:00 .lock 1792 2010-01-01 00:00 COPYRIGHT.txt 4421 2010-01-01 00:00 FiraSans-LICENSE.txt ... 2132 2010-01-01 00:00 storage.js 1180 2010-01-01 00:00 theme.js 3764 2010-01-01 00:00 wheel.svg --------- ------- 890555 38 files ``` wchargin-branch: rustdoc-strip-prefix wchargin-source: 29978cf3bdb7de6cee154f8c11d0251574c2b104 --- rust/private/rustdoc.bzl | 15 ++++-- rust/repositories.bzl | 7 +++ util/dir_zipper/BUILD | 31 +++++++++++++ util/dir_zipper/dir_zipper.py | 37 +++++++++++++++ util/dir_zipper/dir_zipper_lib.py | 26 +++++++++++ util/dir_zipper/dir_zipper_lib_test.py | 64 ++++++++++++++++++++++++++ 6 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 util/dir_zipper/BUILD create mode 100644 util/dir_zipper/dir_zipper.py create mode 100644 util/dir_zipper/dir_zipper_lib.py create mode 100644 util/dir_zipper/dir_zipper_lib_test.py diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 3213e20f04..a0514644c5 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -13,6 +13,7 @@ # limitations under the License. # buildifier: disable=module-docstring +load("@bazel_skylib//lib:paths.bzl", "paths") load("@io_bazel_rules_rust//rust:private/rustc.bzl", "CrateInfo", "DepInfo", "add_crate_link_flags", "add_edition_flags") load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain") @@ -119,15 +120,16 @@ def _zip_action(ctx, input_dir, output_zip): output_zip (File): The location of the output archive containing generated documentation """ args = ctx.actions.args() - - # Create but not compress. - args.add("c", output_zip) + args.add("--zipper", ctx.executable._zipper) + args.add("--output", output_zip) + args.add("--root-dir", input_dir.path) args.add_all([input_dir], expand_directories = True) ctx.actions.run( - executable = ctx.executable._zipper, + executable = ctx.executable._dir_zipper, inputs = [input_dir], outputs = [output_zip], arguments = [args], + tools = [ctx.executable._zipper], ) rust_doc = rule( @@ -159,6 +161,11 @@ rust_doc = rule( doc = "File to add in ``, after content.", allow_single_file = [".html", ".md"], ), + "_dir_zipper": attr.label( + default = Label("//util/dir_zipper"), + cfg = "exec", + executable = True, + ), "_zipper": attr.label( default = Label("@bazel_tools//tools/zip:zipper"), cfg = "exec", diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 149ffb35d4..0229f853cb 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -61,6 +61,13 @@ def rust_repositories( type = "zip", ) + maybe( + http_archive, + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", + ) + maybe( http_archive, name = "bazel_skylib", diff --git a/util/dir_zipper/BUILD b/util/dir_zipper/BUILD new file mode 100644 index 0000000000..0f934eca5a --- /dev/null +++ b/util/dir_zipper/BUILD @@ -0,0 +1,31 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +py_binary( + name = "dir_zipper", + srcs = ["dir_zipper.py"], + python_version = "PY3", + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [":dir_zipper_lib"], +) + +py_library( + name = "dir_zipper_lib", + srcs = ["dir_zipper_lib.py"], + srcs_version = "PY3", + visibility = ["//visibility:private"], +) + +py_test( + name = "dir_zipper_lib_test", + srcs = ["dir_zipper_lib_test.py"], + data = [ + "@bazel_tools//tools/zip:zipper", + ], + python_version = "PY3", + srcs_version = "PY3", + visibility = ["//visibility:private"], + deps = [ + ":dir_zipper_lib", + ], +) diff --git a/util/dir_zipper/dir_zipper.py b/util/dir_zipper/dir_zipper.py new file mode 100644 index 0000000000..9abac0370c --- /dev/null +++ b/util/dir_zipper/dir_zipper.py @@ -0,0 +1,37 @@ +import argparse + +from util.dir_zipper import dir_zipper_lib + + +def main(): + parser = argparse.ArgumentParser( + description="Create a zip archive from some files, stripping " + "a common directory prefix from the name of each archive entry." + ) + parser.add_argument( + "--zipper", + help="path to @bazel_tools//tools/zip:zipper", + required=True, + ) + parser.add_argument( + "--output", help="write a zip file to this path", required=True, + ) + parser.add_argument( + "--root-dir", + help="strip this directory from each entry", + required=True, + ) + parser.add_argument( + "files", + help="add these files to the archive", + nargs="*", + metavar="FILE", + ) + args = parser.parse_args() + dir_zipper_lib.create_zip( + args.zipper, args.output, args.root_dir, args.files + ) + + +if __name__ == "__main__": + main() diff --git a/util/dir_zipper/dir_zipper_lib.py b/util/dir_zipper/dir_zipper_lib.py new file mode 100644 index 0000000000..7216ec37ce --- /dev/null +++ b/util/dir_zipper/dir_zipper_lib.py @@ -0,0 +1,26 @@ +import os +import subprocess + + +def create_zip(zipper, output, root_dir, files): + """Create a zip archive, stripping a dir prefix from each archive name. + + Args: + zipper: Path to @bazel_tools//tools/zip:zipper. + output: Path to zip file to create: e.g., "/tmp/out.zip". + root_dir: Directory to strip from each archive name, with no + trailing slash: e.g., "/tmp/myfiles". + files: List of files to include in the archive, all under + `root_dir`: e.g., ["/tmp/myfiles/a", "/tmp/myfiles/b/c"]. + """ + strip_prefix = root_dir + os.path.sep + args = [] + args.append("c") + args.append(output) + for f in files: + if not f.startswith(root_dir): + raise ValueError("non-descendant: %r not under %r" % (f, root_dir)) + rel = f[len(strip_prefix) :] + spec = "%s=%s" % (rel, f) + args.append(spec) + subprocess.run([zipper, *args]).check_returncode() diff --git a/util/dir_zipper/dir_zipper_lib_test.py b/util/dir_zipper/dir_zipper_lib_test.py new file mode 100644 index 0000000000..32357e153f --- /dev/null +++ b/util/dir_zipper/dir_zipper_lib_test.py @@ -0,0 +1,64 @@ +import os +import shutil +import tempfile +import unittest +import zipfile + +from util.dir_zipper import dir_zipper_lib + +_ZIPPER = os.path.join( + "external", "bazel_tools", "tools", "zip", "zipper", "zipper", +) + + +class DirZipperTest(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def test(self): + root_dir = os.path.join(self.tmpdir, "prefix") + os.mkdir(root_dir) + files = [ + os.path.join(root_dir, *path) + for path in [ + (".lock",), + ("main.js",), + ("mylib", "index.html"), + ("src", "mylib", "lib.rs.html"), + ] + ] + for filepath in files: + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w") as outfile: + outfile.write("%s!\n" % os.path.basename(filepath)) + + output = os.path.join(self.tmpdir, "out.zip") + dir_zipper_lib.create_zip( + zipper=_ZIPPER, output=output, root_dir=root_dir, files=files + ) + + self.assertTrue(os.path.exists(output)) + with open(output, "rb") as fp: + with zipfile.ZipFile(fp) as zp: + self.assertEqual( + len(zp.namelist()), + 4, + "expected 4 entries; got: %r" % (zp.namelist(),), + ) + self.assertEqual(zp.read(".lock"), b".lock!\n") + self.assertEqual(zp.read("main.js"), b"main.js!\n") + self.assertEqual( + zp.read(os.path.join("mylib", "index.html")), + b"index.html!\n", + ) + self.assertEqual( + zp.read(os.path.join("src", "mylib", "lib.rs.html")), + b"lib.rs.html!\n", + ) + + +if __name__ == "__main__": + unittest.main() From 0bd884cf738dc57a31b45a306565a9b7b9a248ab Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 4 Nov 2020 10:53:12 -0800 Subject: [PATCH 02/10] [update patch] wchargin-branch: rustdoc-strip-prefix wchargin-source: 69b2e78f1f52c88609aff631c908471c7521c5f7 --- rust/private/rustdoc.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index a0514644c5..76dcbbc875 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -13,7 +13,6 @@ # limitations under the License. # buildifier: disable=module-docstring -load("@bazel_skylib//lib:paths.bzl", "paths") load("@io_bazel_rules_rust//rust:private/rustc.bzl", "CrateInfo", "DepInfo", "add_crate_link_flags", "add_edition_flags") load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain") From 7bc0b9acafbfedaba76e95274e4708af3014ae5a Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 4 Nov 2020 14:51:10 -0800 Subject: [PATCH 03/10] rust_doc: add web server for docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `rust_doc` rule emits its output into a zip archive so that the build graph can be statically known, but this makes it hard to actually view the documentation. This patch adds a `rust_doc_server` macro, a sibling to `rust_doc` that spins up a tiny web server that reads from the zip file. The server can run under `ibazel` and automatically restart when the documentation sources are updated. This is written in Python for a simple cross-platform solution that doesn’t require compiling a Rust web stack, which takes about 20 seconds on my workstation (Xeon W-2135). Fixes #472. Test Plan: From `examples/`, run `bazel run //hello_world:hello_world_doc_server` and note that the server responds properly. Run with `-- --port 0` (extra `--` for `bazel run`) and note that the server properly prints its actual port. wchargin-branch: rustdoc-server wchargin-source: 307dcd532d0f7880b1cceaf0421a51416607b44f --- examples/hello_world/BUILD | 6 +++ rust/BUILD | 1 + rust/doc_server.template.py | 88 +++++++++++++++++++++++++++++++++++++ rust/private/rustdoc.bzl | 53 ++++++++++++++++++++++ rust/rust.bzl | 4 ++ 5 files changed, 152 insertions(+) create mode 100644 rust/doc_server.template.py diff --git a/examples/hello_world/BUILD b/examples/hello_world/BUILD index 527aac408a..58d94309f1 100644 --- a/examples/hello_world/BUILD +++ b/examples/hello_world/BUILD @@ -2,6 +2,7 @@ load( "@io_bazel_rules_rust//rust:rust.bzl", "rust_binary", "rust_doc", + "rust_doc_server", ) package(default_visibility = ["//visibility:public"]) @@ -16,3 +17,8 @@ rust_doc( name = "hello_world_doc", dep = ":hello_world", ) + +rust_doc_server( + name = "hello_world_doc_server", + dep = ":hello_world_doc", +) diff --git a/rust/BUILD b/rust/BUILD index 690786aec9..af2f66e7ea 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -3,6 +3,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") package(default_visibility = ["//visibility:public"]) exports_files([ + "doc_server.template.py", "known_shas.bzl", "repositories.bzl", "rust.bzl", diff --git a/rust/doc_server.template.py b/rust/doc_server.template.py new file mode 100644 index 0000000000..d85a1cb95e --- /dev/null +++ b/rust/doc_server.template.py @@ -0,0 +1,88 @@ +import argparse +import errno +import mimetypes +import os +import sys +from wsgiref import simple_server +import zipfile + + +ZIP_FILE = "{ZIP_FILE}" +CRATE_NAME = "{CRATE_NAME}" + +DEFAULT_PORT = 8000 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--host", + type=str, + default="", + help="start web server on this host (default: %(default)r)", + ) + parser.add_argument( + "--port", + type=int, + default=8000, + help="start web server on this port; pass 0 to automatically " + "select a free port (default: %(default)s)", + ) + args = parser.parse_args() + port = args.port + + webfiles = os.path.join(os.path.dirname(__file__), ZIP_FILE) + data = {} + with open(webfiles, "rb") as fp: + with zipfile.ZipFile(fp) as zp: + for path in zp.namelist(): + data[path] = zp.read(path) + sys.stderr.write("Read %d files from %s\n" % (len(data), ZIP_FILE)) + + default_path = "/%s/index.html" % CRATE_NAME + + def app(environ, start_response): + p = environ.get("PATH_INFO", "/").lstrip("/") + if not p: + start_response("302 Found", [("Location", default_path)]) + yield b"302 Found\n" + return + if p.endswith("/"): + p += "index.html" + blob = data.get(p) + if not blob: + start_response("404 Not Found", []) + yield b"404 Not Found\n" + return + (mime_type, encoding) = mimetypes.guess_type(p) + headers = [] + if mime_type is not None: + headers.append(("Content-Type", mime_type)) + if encoding is not None: + headers.append(("Content-Encoding", encoding)) + start_response("200 OK", headers) + yield blob + + try: + server = simple_server.make_server("", port, app) + except OSError as e: + if e.errno != getattr(errno, "EADDRINUSE", 0): + raise + sys.stderr.write("%s\n" % e) + sys.stderr.write( + "fatal: failed to bind to port %d; try setting a --port argument\n" + % port + ) + sys.exit(1) + # Find which port was actually bound, in case user requested port 0. + real_port = server.socket.getsockname()[1] + msg = "Serving %s docs on port %d\n" % (CRATE_NAME, real_port) + sys.stderr.write(msg) + try: + server.serve_forever() + except KeyboardInterrupt: + print() + + +if __name__ == "__main__": + main() diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 76dcbbc875..2807a476af 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -16,6 +16,13 @@ load("@io_bazel_rules_rust//rust:private/rustc.bzl", "CrateInfo", "DepInfo", "add_crate_link_flags", "add_edition_flags") load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain") +_DocInfo = provider( + doc = "A provider containing information about a Rust documentation target.", + fields = { + "zip_file": "File: the zip file with rustdoc(1) output", + }, +) + _rust_doc_doc = """Generates code documentation. Example: @@ -66,6 +73,7 @@ def _rust_doc_impl(ctx): crate = ctx.attr.dep[CrateInfo] dep_info = ctx.attr.dep[DepInfo] + doc_info = _DocInfo(zip_file = ctx.outputs.rust_doc_zip) toolchain = find_toolchain(ctx) @@ -109,6 +117,7 @@ def _rust_doc_impl(ctx): # This rule does nothing without a single-file output, though the directory should've sufficed. _zip_action(ctx, output_dir, ctx.outputs.rust_doc_zip) + return [crate, doc_info] def _zip_action(ctx, input_dir, output_zip): """Creates an archive of the generated documentation from `rustdoc` @@ -176,3 +185,47 @@ rust_doc = rule( }, toolchains = ["@io_bazel_rules_rust//rust:toolchain"], ) + +def _rust_doc_server_stub_impl(ctx): + dep = ctx.attr.rust_doc_dep + crate_name = dep[CrateInfo].name + zip_file = dep[_DocInfo].zip_file + ctx.actions.expand_template( + template = ctx.file._server_template, + output = ctx.outputs.main, + substitutions = { + "{CRATE_NAME}": crate_name, + "{ZIP_FILE}": zip_file.basename, + }, + ) + +_rust_doc_server_stub = rule( + implementation = _rust_doc_server_stub_impl, + attrs = { + "rust_doc_dep": attr.label( + mandatory = True, + providers = [CrateInfo, _DocInfo], + ), + "main": attr.output(), + "zip_file": attr.output(), + "_server_template": attr.label( + default = Label("//rust:doc_server.template.py"), + allow_single_file = True, + ), + }, +) + +def rust_doc_server(name, dep, **kwargs): + python_stub_name = name + "_python_stub" + python_stub_output = name + ".py" + zip_file = dep + ".zip" + _rust_doc_server_stub( + name = python_stub_name, + rust_doc_dep = dep, + main = python_stub_output, + ) + native.py_binary( + name = name, + srcs = [python_stub_output], + data = [zip_file], + ) diff --git a/rust/rust.bzl b/rust/rust.bzl index 862344c659..61fb371bc4 100644 --- a/rust/rust.bzl +++ b/rust/rust.bzl @@ -24,6 +24,7 @@ load( load( "@io_bazel_rules_rust//rust:private/rustdoc.bzl", _rust_doc = "rust_doc", + _rust_doc_server = "rust_doc_server", ) load( "@io_bazel_rules_rust//rust:private/rustdoc_test.bzl", @@ -53,6 +54,9 @@ rust_benchmark = _rust_benchmark rust_doc = _rust_doc # See @io_bazel_rules_rust//rust:private/rustdoc.bzl for a complete description. +rust_doc_server = _rust_doc_server +# See @io_bazel_rules_rust//rust:private/rustdoc.bzl for a complete description. + rust_doc_test = _rust_doc_test # See @io_bazel_rules_rust//rust:private/rustdoc_test.bzl for a complete description. From d0df8fa5df8d1180098c1f82f1e7544676b7cb0f Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 4 Nov 2020 15:13:39 -0800 Subject: [PATCH 04/10] [update patch] wchargin-branch: rustdoc-server wchargin-source: bdb753c552346c721fdf558a15e95777f8214827 --- rust/private/rustdoc.bzl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 2807a476af..9b5d84489b 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -15,6 +15,7 @@ # buildifier: disable=module-docstring load("@io_bazel_rules_rust//rust:private/rustc.bzl", "CrateInfo", "DepInfo", "add_crate_link_flags", "add_edition_flags") load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain") +load("@rules_python//python:defs.bzl", "py_binary") _DocInfo = provider( doc = "A provider containing information about a Rust documentation target.", @@ -216,6 +217,13 @@ _rust_doc_server_stub = rule( ) def rust_doc_server(name, dep, **kwargs): + """Generates a web server to display code documentation. + + Args: + name: A unique name for this target. + dep: Label for a `rust_doc` rule whose docs to serve. + **kwargs: Any generic binary kwargs, like `tags` or `visibility`. + """ python_stub_name = name + "_python_stub" python_stub_output = name + ".py" zip_file = dep + ".zip" @@ -224,8 +232,10 @@ def rust_doc_server(name, dep, **kwargs): rust_doc_dep = dep, main = python_stub_output, ) - native.py_binary( + py_binary( name = name, srcs = [python_stub_output], data = [zip_file], + srcs_version = "PY3", + python_version = "PY3", ) From 24d6478aceec9542223ad8d4d11798ea5b937c15 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 6 Nov 2020 12:10:31 -0800 Subject: [PATCH 05/10] [update patch] wchargin-branch: rustdoc-strip-prefix wchargin-source: 40227180c7f910d9f76716cf42621e08e5f808a1 --- rust/private/rustdoc.bzl | 6 +- rust/repositories.bzl | 7 --- util/dir_zipper/BUILD | 31 ++--------- util/dir_zipper/dir_zipper.py | 37 ------------- util/dir_zipper/dir_zipper.rs | 77 ++++++++++++++++++++++++++ util/dir_zipper/dir_zipper_lib.py | 26 --------- util/dir_zipper/dir_zipper_lib_test.py | 64 --------------------- 7 files changed, 84 insertions(+), 164 deletions(-) delete mode 100644 util/dir_zipper/dir_zipper.py create mode 100644 util/dir_zipper/dir_zipper.rs delete mode 100644 util/dir_zipper/dir_zipper_lib.py delete mode 100644 util/dir_zipper/dir_zipper_lib_test.py diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 76dcbbc875..fac26629e7 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -119,9 +119,9 @@ def _zip_action(ctx, input_dir, output_zip): output_zip (File): The location of the output archive containing generated documentation """ args = ctx.actions.args() - args.add("--zipper", ctx.executable._zipper) - args.add("--output", output_zip) - args.add("--root-dir", input_dir.path) + args.add(ctx.executable._zipper) + args.add(output_zip) + args.add(input_dir.path) args.add_all([input_dir], expand_directories = True) ctx.actions.run( executable = ctx.executable._dir_zipper, diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 0229f853cb..149ffb35d4 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -61,13 +61,6 @@ def rust_repositories( type = "zip", ) - maybe( - http_archive, - name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", - sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", - ) - maybe( http_archive, name = "bazel_skylib", diff --git a/util/dir_zipper/BUILD b/util/dir_zipper/BUILD index 0f934eca5a..f8d7eb2f64 100644 --- a/util/dir_zipper/BUILD +++ b/util/dir_zipper/BUILD @@ -1,31 +1,8 @@ -load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") +load("//rust:rust.bzl", "rust_binary") -py_binary( +rust_binary( name = "dir_zipper", - srcs = ["dir_zipper.py"], - python_version = "PY3", - srcs_version = "PY3", + srcs = ["dir_zipper.rs"], + edition = "2018", visibility = ["//visibility:public"], - deps = [":dir_zipper_lib"], -) - -py_library( - name = "dir_zipper_lib", - srcs = ["dir_zipper_lib.py"], - srcs_version = "PY3", - visibility = ["//visibility:private"], -) - -py_test( - name = "dir_zipper_lib_test", - srcs = ["dir_zipper_lib_test.py"], - data = [ - "@bazel_tools//tools/zip:zipper", - ], - python_version = "PY3", - srcs_version = "PY3", - visibility = ["//visibility:private"], - deps = [ - ":dir_zipper_lib", - ], ) diff --git a/util/dir_zipper/dir_zipper.py b/util/dir_zipper/dir_zipper.py deleted file mode 100644 index 9abac0370c..0000000000 --- a/util/dir_zipper/dir_zipper.py +++ /dev/null @@ -1,37 +0,0 @@ -import argparse - -from util.dir_zipper import dir_zipper_lib - - -def main(): - parser = argparse.ArgumentParser( - description="Create a zip archive from some files, stripping " - "a common directory prefix from the name of each archive entry." - ) - parser.add_argument( - "--zipper", - help="path to @bazel_tools//tools/zip:zipper", - required=True, - ) - parser.add_argument( - "--output", help="write a zip file to this path", required=True, - ) - parser.add_argument( - "--root-dir", - help="strip this directory from each entry", - required=True, - ) - parser.add_argument( - "files", - help="add these files to the archive", - nargs="*", - metavar="FILE", - ) - args = parser.parse_args() - dir_zipper_lib.create_zip( - args.zipper, args.output, args.root_dir, args.files - ) - - -if __name__ == "__main__": - main() diff --git a/util/dir_zipper/dir_zipper.rs b/util/dir_zipper/dir_zipper.rs new file mode 100644 index 0000000000..a80309e027 --- /dev/null +++ b/util/dir_zipper/dir_zipper.rs @@ -0,0 +1,77 @@ +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Command; + +const USAGE: &str = r#"usage: dir_zipper [...] + +Creates a zip archive, stripping a directory prefix from each file name. + +Args: + zipper: Path to @bazel_tools//tools/zip:zipper. + output: Path to zip file to create: e.g., "/tmp/out.zip". + root_dir: Directory to strip from each archive name, with no trailing + slash: e.g., "/tmp/myfiles". + files: List of files to include in the archive, all under `root_dir`: + e.g., ["/tmp/myfiles/a", "/tmp/myfiles/b/c"]. + +Example: + dir_zipper \ + bazel-rules_rust/external/bazel_tools/tools/zip/zipper/zipper \ + /tmp/out.zip \ + /tmp/myfiles \ + /tmp/myfiles/a /tmp/myfiles/b/c + +This will create /tmp/out.zip with file entries "a" and "b/c". +"#; + +macro_rules! die { + ($($arg:tt)*) => { + { + eprintln!($($arg)*); + std::process::exit(1); + } + }; +} + +fn main() { + let mut args = std::env::args_os().skip(1); + let (zipper, output, root_dir) = match args.next().zip(args.next()).zip(args.next()) { + Some(((zipper, output), root_dir)) => ( + PathBuf::from(zipper), + PathBuf::from(output), + PathBuf::from(root_dir), + ), + _ => { + die!("{}", USAGE); + } + }; + let files = args.map(PathBuf::from).collect::>(); + let mut comm = Command::new(zipper); + comm.arg("c"); // create, but don't compress + comm.arg(output); + for f in files { + let rel = f.strip_prefix(&root_dir).unwrap_or_else(|_e| { + die!( + "fatal: non-descendant: {} not under {}", + f.display(), + root_dir.display() + ); + }); + let mut spec = OsString::new(); + spec.push(rel); + spec.push("="); + spec.push(f); + comm.arg(spec); + } + let exit_status = comm + .spawn() + .unwrap_or_else(|e| die!("fatal: could not spawn zipper: {}", e)) + .wait() + .unwrap_or_else(|e| die!("fatal: could not wait on zipper: {}", e)); + if !exit_status.success() { + match exit_status.code() { + Some(c) => die!("fatal: zipper exited with {}", c), + None => die!("fatal: zipper terminated by signal"), + } + } +} diff --git a/util/dir_zipper/dir_zipper_lib.py b/util/dir_zipper/dir_zipper_lib.py deleted file mode 100644 index 7216ec37ce..0000000000 --- a/util/dir_zipper/dir_zipper_lib.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import subprocess - - -def create_zip(zipper, output, root_dir, files): - """Create a zip archive, stripping a dir prefix from each archive name. - - Args: - zipper: Path to @bazel_tools//tools/zip:zipper. - output: Path to zip file to create: e.g., "/tmp/out.zip". - root_dir: Directory to strip from each archive name, with no - trailing slash: e.g., "/tmp/myfiles". - files: List of files to include in the archive, all under - `root_dir`: e.g., ["/tmp/myfiles/a", "/tmp/myfiles/b/c"]. - """ - strip_prefix = root_dir + os.path.sep - args = [] - args.append("c") - args.append(output) - for f in files: - if not f.startswith(root_dir): - raise ValueError("non-descendant: %r not under %r" % (f, root_dir)) - rel = f[len(strip_prefix) :] - spec = "%s=%s" % (rel, f) - args.append(spec) - subprocess.run([zipper, *args]).check_returncode() diff --git a/util/dir_zipper/dir_zipper_lib_test.py b/util/dir_zipper/dir_zipper_lib_test.py deleted file mode 100644 index 32357e153f..0000000000 --- a/util/dir_zipper/dir_zipper_lib_test.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import shutil -import tempfile -import unittest -import zipfile - -from util.dir_zipper import dir_zipper_lib - -_ZIPPER = os.path.join( - "external", "bazel_tools", "tools", "zip", "zipper", "zipper", -) - - -class DirZipperTest(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmpdir) - - def test(self): - root_dir = os.path.join(self.tmpdir, "prefix") - os.mkdir(root_dir) - files = [ - os.path.join(root_dir, *path) - for path in [ - (".lock",), - ("main.js",), - ("mylib", "index.html"), - ("src", "mylib", "lib.rs.html"), - ] - ] - for filepath in files: - os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, "w") as outfile: - outfile.write("%s!\n" % os.path.basename(filepath)) - - output = os.path.join(self.tmpdir, "out.zip") - dir_zipper_lib.create_zip( - zipper=_ZIPPER, output=output, root_dir=root_dir, files=files - ) - - self.assertTrue(os.path.exists(output)) - with open(output, "rb") as fp: - with zipfile.ZipFile(fp) as zp: - self.assertEqual( - len(zp.namelist()), - 4, - "expected 4 entries; got: %r" % (zp.namelist(),), - ) - self.assertEqual(zp.read(".lock"), b".lock!\n") - self.assertEqual(zp.read("main.js"), b"main.js!\n") - self.assertEqual( - zp.read(os.path.join("mylib", "index.html")), - b"index.html!\n", - ) - self.assertEqual( - zp.read(os.path.join("src", "mylib", "lib.rs.html")), - b"lib.rs.html!\n", - ) - - -if __name__ == "__main__": - unittest.main() From dd649e1526ba04e73a849b53b77187d8a98fee39 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 6 Nov 2020 12:12:18 -0800 Subject: [PATCH 06/10] [update patch] wchargin-branch: rustdoc-server wchargin-source: 1f90836672a1cc8467b6b11e3b045497b8b05903 --- rust/repositories.bzl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 149ffb35d4..0229f853cb 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -61,6 +61,13 @@ def rust_repositories( type = "zip", ) + maybe( + http_archive, + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", + ) + maybe( http_archive, name = "bazel_skylib", From 5ab7c6ee34c58bca28e4420412f8599d4937ecc1 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 6 Nov 2020 12:16:36 -0800 Subject: [PATCH 07/10] [update patch] wchargin-branch: rustdoc-strip-prefix wchargin-source: 8c46290195fc0668f3126589fd07ee863df8749f --- util/dir_zipper/dir_zipper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/dir_zipper/dir_zipper.rs b/util/dir_zipper/dir_zipper.rs index a80309e027..fbc4938e2a 100644 --- a/util/dir_zipper/dir_zipper.rs +++ b/util/dir_zipper/dir_zipper.rs @@ -70,7 +70,7 @@ fn main() { .unwrap_or_else(|e| die!("fatal: could not wait on zipper: {}", e)); if !exit_status.success() { match exit_status.code() { - Some(c) => die!("fatal: zipper exited with {}", c), + Some(c) => std::process::exit(c), None => die!("fatal: zipper terminated by signal"), } } From a2f58a82f6c2befcdbfef7a2f23fb34376b534af Mon Sep 17 00:00:00 2001 From: William Chargin Date: Thu, 19 Nov 2020 15:28:49 -0800 Subject: [PATCH 08/10] [update patch] wchargin-branch: rustdoc-strip-prefix wchargin-source: ea88a0484df0608cb08521fb4cde2d4dc7904553 --- rust/private/rustdoc.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index fac26629e7..54b024505b 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -121,7 +121,7 @@ def _zip_action(ctx, input_dir, output_zip): args = ctx.actions.args() args.add(ctx.executable._zipper) args.add(output_zip) - args.add(input_dir.path) + args.add(ctx.bin_dir.path) args.add_all([input_dir], expand_directories = True) ctx.actions.run( executable = ctx.executable._dir_zipper, From a9efb217d6e69883265f999d405182601d9fb4b8 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Thu, 19 Nov 2020 15:29:35 -0800 Subject: [PATCH 09/10] [update patch] wchargin-branch: rustdoc-server wchargin-source: f1172908d26a8f6621f0ed0a894ec8541db6c345 --- rust/doc_server.template.py | 3 ++- rust/private/rustdoc.bzl | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rust/doc_server.template.py b/rust/doc_server.template.py index d85a1cb95e..faf9e15798 100644 --- a/rust/doc_server.template.py +++ b/rust/doc_server.template.py @@ -8,6 +8,7 @@ ZIP_FILE = "{ZIP_FILE}" +TARGET_PATH = "{TARGET_PATH}" CRATE_NAME = "{CRATE_NAME}" DEFAULT_PORT = 8000 @@ -39,7 +40,7 @@ def main(): data[path] = zp.read(path) sys.stderr.write("Read %d files from %s\n" % (len(data), ZIP_FILE)) - default_path = "/%s/index.html" % CRATE_NAME + default_path = "/%s/%s/index.html" % (TARGET_PATH, CRATE_NAME) def app(environ, start_response): p = environ.get("PATH_INFO", "/").lstrip("/") diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index ae8754566d..fc482725ea 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -191,10 +191,13 @@ def _rust_doc_server_stub_impl(ctx): dep = ctx.attr.rust_doc_dep crate_name = dep[CrateInfo].name zip_file = dep[_DocInfo].zip_file + path_parts = [dep.label.workspace_root, dep.label.package, dep.label.name] + target_path = "/".join([p for p in path_parts if p]) ctx.actions.expand_template( template = ctx.file._server_template, output = ctx.outputs.main, substitutions = { + "{TARGET_PATH}": target_path, "{CRATE_NAME}": crate_name, "{ZIP_FILE}": zip_file.basename, }, From 9fac33a56c320cd5a8dc18e60774cbf876d9c3ea Mon Sep 17 00:00:00 2001 From: William Chargin Date: Thu, 19 Nov 2020 16:09:08 -0800 Subject: [PATCH 10/10] [update patch] wchargin-branch: rustdoc-server wchargin-source: 8817e8fef2309159717320f672701c2fd8999517 --- docs/docs_repositories.bzl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/docs_repositories.bzl b/docs/docs_repositories.bzl index 2393c78810..f756c49045 100644 --- a/docs/docs_repositories.bzl +++ b/docs/docs_repositories.bzl @@ -25,3 +25,10 @@ def repositories(is_top_level = False): sha256 = "5d7191bb0800434a9192d8ac80cba4909e96dbb087c5d51f168fedd7bde7b525", strip_prefix = "stardoc-1ef781ced3b1443dca3ed05dec1989eca1a4e1cd", ) + + maybe( + http_archive, + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", + )