diff --git a/pkg/BUILD b/pkg/BUILD index 92312fa7..a05def91 100644 --- a/pkg/BUILD +++ b/pkg/BUILD @@ -29,6 +29,7 @@ constraint_setting(name = "not_compatible_setting") constraint_value( name = "not_compatible", constraint_setting = ":not_compatible_setting", + visibility = ["//visibility:public"], ) filegroup( diff --git a/pkg/WORKSPACE b/pkg/WORKSPACE index 06d13f15..17523066 100644 --- a/pkg/WORKSPACE +++ b/pkg/WORKSPACE @@ -35,6 +35,14 @@ find_system_rpmbuild( verbose = True, ) +# Needed for making our release notes +load("@rules_pkg//toolchains/git:git_configure.bzl", "experimental_find_system_git") + +experimental_find_system_git( + name = "rules_pkg_git", + verbose = True, +) + http_archive( name = "bazel_stardoc", sha256 = "36b8d6c2260068b9ff82faea2f7add164bf3436eac9ba3ec14809f335346d66a", diff --git a/pkg/distro/BUILD b/pkg/distro/BUILD index 90c2d84f..eb164730 100644 --- a/pkg/distro/BUILD +++ b/pkg/distro/BUILD @@ -15,6 +15,7 @@ load("//:pkg.bzl", "pkg_tar") load("//:version.bzl", "version") load("//releasing:defs.bzl", "print_rel_notes") +load("//releasing:git.bzl", "git_changelog") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@bazel_stardoc//stardoc:stardoc.bzl", "stardoc") load("@rules_python//python:defs.bzl", "py_test") @@ -59,6 +60,7 @@ genrule( print_rel_notes( name = "relnotes", outs = ["relnotes.txt"], + changelog = ":changelog", deps_method = "rules_pkg_dependencies", mirror_host = "mirror.bazel.build", org = "bazelbuild", @@ -66,6 +68,11 @@ print_rel_notes( version = version, ) +git_changelog( + name = "changelog", + out = "changelog.txt", +) + py_test( name = "packaging_test", size = "large", diff --git a/pkg/releasing/BUILD b/pkg/releasing/BUILD index c294d9e3..540e0ed0 100644 --- a/pkg/releasing/BUILD +++ b/pkg/releasing/BUILD @@ -1,4 +1,5 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") +load("git.bzl", "git_changelog") licenses(["notice"]) @@ -6,7 +7,7 @@ package( default_visibility = ["//visibility:public"], ) -# WARNING: 2019-08-05. This is experimental and subject to change. +# WARNING: 2021-06-28. This is experimental and subject to change. # Sample usage: # load("@rules_pkg//releasing:defs.bzl", "print_rel_notes") @@ -55,3 +56,14 @@ py_test( ":release_utils", ], ) + +# This is an internal tool. Use at your own risk. +py_binary( + name = "git_changelog_private", + srcs = [ + "git_changelog_private.py", + ], + srcs_version = "PY3", + # TODO(https://github.com/bazelbuild/bazel/issues/7377): Make this private. + visibility = ["//visibility:public"], +) diff --git a/pkg/releasing/defs.bzl b/pkg/releasing/defs.bzl index 25e72ba1..e665cb65 100644 --- a/pkg/releasing/defs.bzl +++ b/pkg/releasing/defs.bzl @@ -7,6 +7,7 @@ def print_rel_notes( deps_method = "", toolchains_method = "", org = "bazelbuild", + changelog = None, mirror_host = None): tarball_name = ":%s-%s.tar.gz" % (repo, version) cmd = [ @@ -16,23 +17,30 @@ def print_rel_notes( "--version=%s" % version, "--tarball=$(location %s)" % tarball_name, ] + tools = ["//releasing:print_rel_notes"] if setup_file: cmd.append("--setup_file=%s" % setup_file) if deps_method: cmd.append("--deps_method=%s" % deps_method) if toolchains_method: cmd.append("--toolchains_method=%s" % toolchains_method) + if changelog: + cmd.append("--changelog=$(location %s)" % changelog) + # We should depend on a changelog as a tool so that it is always built + # for the host configuration. If the changelog is generated on the fly, + # then we would have to run commands against our revision control + # system. That only makes sense locally on the host, because the + # revision history is never exported to a remote build system. + tools.append(changelog) if mirror_host: cmd.append("--mirror_host=%s" % mirror_host) cmd.append(">$@") native.genrule( - name = "relnotes", + name = name, srcs = [ tarball_name, ], - outs = outs or ["relnotes.txt"], + outs = outs or [name + ".txt"], cmd = " ".join(cmd), - tools = [ - "//releasing:print_rel_notes", - ], + tools = tools, ) diff --git a/pkg/releasing/git.bzl b/pkg/releasing/git.bzl new file mode 100644 index 00000000..135730a1 --- /dev/null +++ b/pkg/releasing/git.bzl @@ -0,0 +1,96 @@ +# Copyright 2021 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A rule to extract the git changelog.""" + +def _git_changelog_impl(ctx): + """Implements to git_changelog rule.""" + + args = ctx.actions.args() + tools = [] + + toolchain = ctx.toolchains["@rules_pkg//toolchains/git:git_toolchain_type"].git + if not toolchain.valid: + fail("The git_toolchain is not properly configured: " + + toolchain.name) + if toolchain.path: + args.add("--git_path", toolchain.path) + else: + executable = toolchain.label.files_to_run.executable + tools.append(executable) + tools.append(toolchain.label.default_runfiles.files.to_list()) + args.add("--git_path", executable.path) + args.add("--git_root", toolchain.client_top) + args.add("--from_ref", ctx.attr.from_ref) + args.add("--to_ref", ctx.attr.to_ref) + args.add("--out", ctx.outputs.out.path) + if ctx.attr.verbose: + args.add("--verbose") + + ctx.actions.run( + mnemonic = "GitChangelog", + executable = ctx.executable._git_changelog, + use_default_shell_env = True, + arguments = [args], + outputs = [ctx.outputs.out], + env = { + "LANG": "en_US.UTF-8", + "LC_CTYPE": "UTF-8", + "PYTHONIOENCODING": "UTF-8", + "PYTHONUTF8": "1", + }, + execution_requirements = { + "local": "1", + }, + tools = tools, + ) + +# Define the rule. +_git_changelog = rule( + doc = "Extracts the git changelog between two refs.", + attrs = { + "from_ref": attr.string( + doc = "lower commit ref. The default is to use the latest tag", + default = "_LATEST_TAG_", + ), + "to_ref": attr.string( + doc = "upper commit ref. The default is HEAD", + default = "HEAD", + ), + "out": attr.output(mandatory = True), + "verbose": attr.bool( + doc = "Be verbose", + default = False, + ), + "_git_changelog": attr.label( + default = Label("//releasing:git_changelog_private"), + cfg = "exec", + executable = True, + allow_files = True, + ), + }, + implementation = _git_changelog_impl, + toolchains = ["@rules_pkg//toolchains/git:git_toolchain_type"], +) + + +def git_changelog(name, **kwargs): + _git_changelog( + name = name, + # This requires bazel 4.x + target_compatible_with = select({ + "//toolchains/git:have_git": [], + "//conditions:default": ["//:not_compatible"], + }), + **kwargs, + ) diff --git a/pkg/releasing/git_changelog_private.py b/pkg/releasing/git_changelog_private.py new file mode 100644 index 00000000..8344cef1 --- /dev/null +++ b/pkg/releasing/git_changelog_private.py @@ -0,0 +1,72 @@ +# Copyright 2021 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities to extract git commit descriptions in useful ways.""" + +import argparse +import os +import subprocess +import sys + + +def guess_previous_release_tag(git_path, pattern=None): + assert git_path + most_recent = None + cmd = [git_path, 'tag'] + if pattern: + cmd.extend(['--list', pattern]) + # We are doing something dumb here for now. Grab the list of tags, and pick + # the last one. + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + most_recent = proc.stdout.read().decode('utf-8') + most_recent = most_recent.strip().replace('\n\n', '\n').split('\n')[-1] + return most_recent + + +def git_changelog(from_ref, to_ref='HEAD', git_path=None): + assert from_ref + assert to_ref + assert git_path + cmd = [git_path, 'log', '%s..%s' % (from_ref, to_ref)] + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + return proc.stdout.read().decode('utf-8') + + +def main(): + parser = argparse.ArgumentParser( + description='Helper for extracting git changelog', + fromfile_prefix_chars='@') + parser.add_argument('--git_path', required=True, help='path to git binary') + parser.add_argument('--git_root', required=True, help='path to git client') + parser.add_argument('--out', required=True, help='output path') + parser.add_argument('--from_ref', help='from REF') + parser.add_argument('--to_ref', help='to REF') + parser.add_argument('--verbose', action='store_true') + + options = parser.parse_args() + + with open(options.out, 'w', encoding='utf-8') as out: + os.chdir(options.git_root) + from_ref = options.from_ref + if not from_ref or from_ref == '_LATEST_TAG_': + from_ref = guess_previous_release_tag(options.git_path) + to_ref = options.to_ref or 'HEAD' + if options.verbose: + print('Getting changelog from %s to %s' % (from_ref, to_ref)) + changelog = git_changelog( + from_ref=from_ref, to_ref=to_ref, git_path=options.git_path) + out.write(changelog) + return 0 + +if __name__ == '__main__': + main() diff --git a/pkg/releasing/print_rel_notes.py b/pkg/releasing/print_rel_notes.py index 3d8d6d96..2e38008b 100644 --- a/pkg/releasing/print_rel_notes.py +++ b/pkg/releasing/print_rel_notes.py @@ -24,7 +24,8 @@ def print_notes(org, repo, version, tarball_path, mirror_host=None, - deps_method=None, setup_file=None, toolchains_method=None): + deps_method=None, setup_file=None, toolchains_method=None, + changelog=''): file_name = release_tools.package_basename(repo, version) sha256 = release_tools.get_package_sha256(tarball_path) @@ -42,6 +43,9 @@ def print_notes(org, repo, version, tarball_path, mirror_host=None, **Incompatible Changes** + **Change Log** + ${changelog} + **WORKSPACE setup** ``` @@ -55,6 +59,7 @@ def print_notes(org, repo, version, tarball_path, mirror_host=None, """).strip()) print(relnotes_template.substitute({ + 'changelog': changelog, 'org': org, 'repo': repo, 'version': version, @@ -100,7 +105,7 @@ def main(): required=True, help='path to release tarball') parser.add_argument( '--mirror_host', default=None, - help='If provider, the hostname of a mirror for the download url.') + help='If provider, the hostname of a mirror for the download url') parser.add_argument( '--setup_file', default=None, help='Alternate name for setup file. Default: deps.bzl') @@ -110,10 +115,20 @@ def main(): parser.add_argument( '--toolchains_method', default=None, help='Alternate name for toolchains method. Default: {repo}_toolchains') + parser.add_argument( + '--changelog', default=None, + help='Pre-fill release notes with changes from this file') options = parser.parse_args() + if options.changelog: + with open(options.changelog, 'r', encoding='utf-8') as f: + changelog = f.read() + else: + changelog = 'TBD' + print_notes(options.org, options.repo, options.version, options.tarball_path, deps_method=options.deps_method, + changelog=changelog, mirror_host=options.mirror_host, setup_file=options.setup_file, toolchains_method=options.toolchains_method) diff --git a/pkg/toolchains/git/BUILD b/pkg/toolchains/git/BUILD new file mode 100644 index 00000000..cb1b53e1 --- /dev/null +++ b/pkg/toolchains/git/BUILD @@ -0,0 +1,73 @@ +# Copyright 2021 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""toolchain to wrap the git binary. + +Type: @rules_pkg//toolchains/git:git_toolchain_type + +Toolchains: +- git_missing_toolchain: provides a fallback toolchain for exec plaforms + where git might not be available. + +- git_auto_toolchain: a toolchain that uses the installed git. See + git_configure.bzl%find_system_git for usage. +""" + +load("//toolchains/git:git.bzl", "git_toolchain", "is_git_available") + +filegroup( + name = "standard_package", + srcs = glob(["*"]), + visibility = ["//distro:__pkg__"], +) + +exports_files( + glob(["*"]), + visibility = ["//visibility:public"], +) + +# Expose the availabilty of an actual git as a config_setting, so we can +# select() on it. +config_setting( + name = "have_git", + flag_values = { + ":is_git_available": "1", + }, + visibility = ["//visibility:public"], +) + +# Expose the availabilty of an actual git as a feature flag, so we can +# create a config_setting from it. +is_git_available( + name = "is_git_available", + visibility = ["//:__subpackages__"], +) + +toolchain_type( + name = "git_toolchain_type", + visibility = ["//visibility:public"], +) + +# git_missing_toolchain provides a fallback toolchain so that toolchain +# resolution can succeed even on platforms that do not have a working git. +# If this toolchain is selected, the constraint ":have_git" will not be +# satistifed. +git_toolchain( + name = "no_git", +) + +toolchain( + name = "git_missing_toolchain", + toolchain = ":no_git", + toolchain_type = ":git_toolchain_type", +) diff --git a/pkg/toolchains/git/BUILD.tmpl b/pkg/toolchains/git/BUILD.tmpl new file mode 100644 index 00000000..f009c1b7 --- /dev/null +++ b/pkg/toolchains/git/BUILD.tmpl @@ -0,0 +1,16 @@ +# This content is generated by {GENERATOR} +load("@rules_pkg//toolchains/git:git.bzl", "git_toolchain") + +git_toolchain( + name = "git_auto", + # path to git executable + path = "{GIT_PATH}", + # path to the top of the git client (but really any folder under it) + client_top = "{GIT_ROOT}", +) + +toolchain( + name = "git_auto_toolchain", + toolchain = ":git_auto", + toolchain_type = "@rules_pkg//toolchains/git:git_toolchain_type", +) diff --git a/pkg/toolchains/git/git.bzl b/pkg/toolchains/git/git.bzl new file mode 100644 index 00000000..f58ef812 --- /dev/null +++ b/pkg/toolchains/git/git.bzl @@ -0,0 +1,81 @@ +# Copyright 2021 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""toolchain to provide access to git from Bazel. + +A git toolchain is somewhat unusual compared to other toolchains. +It not only provides a path to the git executable, but it also +holds the absolute path to our workspace, which is presumed to +be under revision control. This allows us to write helper tools +that escape the Bazel sandbox and pop back to the workspace/repo. +""" + +GitInfo = provider( + doc = """Information needed to invoke git.""", + fields = { + "name": "The name of the toolchain", + "valid": "Is this toolchain valid and usable?", + "label": "Label of a target providing a git binary", + "path": "The path to a pre-built git", + "client_top": "The path to the top of the git client." + + " In reality, we use the path to the WORKSPACE file as" + + " a proxy for a folder underneath the git client top.", + }, +) + +def _git_toolchain_impl(ctx): + if ctx.attr.label and ctx.attr.path: + fail("git_toolchain must not specify both label and path.") + valid = bool(ctx.attr.label) or bool(ctx.attr.path) + toolchain_info = platform_common.ToolchainInfo( + git = GitInfo( + name = str(ctx.label), + valid = valid, + label = ctx.attr.label, + path = ctx.attr.path, + client_top = ctx.attr.client_top, + ), + ) + return [toolchain_info] + +git_toolchain = rule( + implementation = _git_toolchain_impl, + attrs = { + "label": attr.label( + doc = "A valid label of a target to build or a prebuilt binary. Mutually exclusive with path.", + cfg = "exec", + executable = True, + allow_files = True, + ), + "path": attr.string( + doc = "The path to the git executable. Mutually exclusive with label.", + ), + "client_top": attr.string( + doc = "The top of your git client.", + ), + }, +) + +# Expose the presence of a git in the resolved toolchain as a flag. +def _is_git_available_impl(ctx): + toolchain = ctx.toolchains["@rules_pkg//toolchains/git:git_toolchain_type"] + available = toolchain and toolchain.git.valid + return [config_common.FeatureFlagInfo( + value = ("1" if available else "0"), + )] + +is_git_available = rule( + implementation = _is_git_available_impl, + attrs = {}, + toolchains = ["@rules_pkg//toolchains/git:git_toolchain_type"], +) diff --git a/pkg/toolchains/git/git_configure.bzl b/pkg/toolchains/git/git_configure.bzl new file mode 100644 index 00000000..ec33690a --- /dev/null +++ b/pkg/toolchains/git/git_configure.bzl @@ -0,0 +1,78 @@ +# Copyright 2021 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Repository rule to autoconfigure a toolchain using the system git.""" + +def _write_build(rctx, path, workspace_dir): + if not path: + path = "" + rctx.template( + "BUILD", + Label("//toolchains/git:BUILD.tmpl"), + substitutions = { + "{GENERATOR}": "@rules_pkg//toolchains/git/git_configure.bzl%find_system_git", + "{GIT_PATH}": str(path), + "{GIT_ROOT}": workspace_dir, + }, + executable = False, + ) + +def _find_system_git_impl(rctx): + git_path = rctx.which("git") + if rctx.attr.verbose: + if git_path: + print("Found git at '%s'" % git_path) # buildifier: disable=print + else: + print("No system git found.") # buildifier: disable=print + + # In a conventional setup the directory of the WORKSPACE file is under the git client. + # So we use the absolute path to WORKSPACE dir as a surrogate for git-ness. + ws_dir = str(rctx.path(rctx.attr.workspace_file).dirname.realpath) + if rctx.attr.verbose: + print("Found WORKSPACE in", ws_dir) # buildifier: disable=print + _write_build(rctx = rctx, path = git_path, workspace_dir = ws_dir) + +_find_system_git = repository_rule( + implementation = _find_system_git_impl, + doc = """Create a repository that defines an git toolchain based on the system git.""", + local = True, + attrs = { + "workspace_file": attr.label( + doc = "Referece to calling repository WORKSPACE file.", + allow_single_file = True, + mandatory = True, + ), + "verbose": attr.bool( + doc = "If true, print status messages.", + ), + }, +) + +def experimental_find_system_git(name, workspace_file = None, verbose = False): + """Create a toolchain that lets you run git. + + WARNING: This is experimental. The API and behavior are subject to change + at any time. + + This presumes that your Bazel WORKSPACE file is located under your git + client. That is often true, but might not be in a multi-repo where you + might weave together a Bazel workspace from several git repos that are + all rooted under the WORKSPACE file. + """ + if not workspace_file: + workspace_file = Label("//:WORKSPACE") + _find_system_git(name = name, workspace_file = workspace_file, verbose = verbose) + native.register_toolchains( + "@%s//:git_auto_toolchain" % name, + "@rules_pkg//toolchains/git:git_missing_toolchain", + )