From 0abb1a55fb8c6760b4d5834a6eeadbd324682d22 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 7 Mar 2022 12:46:43 +0100 Subject: [PATCH 01/25] move core `nixpkgs` rules to separate module this is in preparation to support `bzlmod` --- core/BUILD.bazel | 5 + core/MODULE.bazel | 7 + core/nixpkgs.bzl | 350 +++++++++++++++++++++++++++ core/util.bzl | 99 ++++++++ nixpkgs/BUILD.bazel | 2 + nixpkgs/nixpkgs.bzl | 464 ++---------------------------------- nixpkgs/toolchains/rust.bzl | 2 +- 7 files changed, 487 insertions(+), 442 deletions(-) create mode 100644 core/BUILD.bazel create mode 100644 core/MODULE.bazel create mode 100644 core/nixpkgs.bzl create mode 100644 core/util.bzl diff --git a/core/BUILD.bazel b/core/BUILD.bazel new file mode 100644 index 000000000..00d87a19e --- /dev/null +++ b/core/BUILD.bazel @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "nixpkgs.bzl", +]) diff --git a/core/MODULE.bazel b/core/MODULE.bazel new file mode 100644 index 000000000..34db8dd3c --- /dev/null +++ b/core/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "rules_nixpkgs_core", + version = "0.8.1", +) + +bazel_dep(name = "platforms", version = "0.0.4") +bazel_dep(name = "bazel_skylib", version = "1.0.3") diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl new file mode 100644 index 000000000..2a2e55222 --- /dev/null +++ b/core/nixpkgs.bzl @@ -0,0 +1,350 @@ +load("//nixpkgs:private/location_expansion.bzl", "expand_location") +load( + ":util.bzl", + "cp", + "executable_path", + "execute_or_fail", + "find_children", + "is_supported_platform", +) + +def _nixpkgs_git_repository_impl(repository_ctx): + repository_ctx.file( + "BUILD", + content = 'filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"])', + ) + + # Make "@nixpkgs" (syntactic sugar for "@nixpkgs//:nixpkgs") a valid + # label for default.nix. + repository_ctx.symlink("default.nix", repository_ctx.name) + + repository_ctx.download_and_extract( + url = "%s/archive/%s.tar.gz" % (repository_ctx.attr.remote, repository_ctx.attr.revision), + stripPrefix = "nixpkgs-" + repository_ctx.attr.revision, + sha256 = repository_ctx.attr.sha256, + ) + +nixpkgs_git_repository = repository_rule( + implementation = _nixpkgs_git_repository_impl, + attrs = { + "revision": attr.string( + mandatory = True, + doc = "Git commit hash or tag identifying the version of Nixpkgs to use.", + ), + "remote": attr.string( + default = "https://github.com/NixOS/nixpkgs", + doc = "The URI of the remote Git repository. This must be a HTTP URL. There is currently no support for authentication. Defaults to [upstream nixpkgs](https://github.com/NixOS/nixpkgs).", + ), + "sha256": attr.string(doc = "The SHA256 used to verify the integrity of the repository."), + }, + doc = """\ +Name a specific revision of Nixpkgs on GitHub or a local checkout. +""", +) + +def _nixpkgs_local_repository_impl(repository_ctx): + if not bool(repository_ctx.attr.nix_file) != \ + bool(repository_ctx.attr.nix_file_content): + fail("Specify one of 'nix_file' or 'nix_file_content' (but not both).") + if repository_ctx.attr.nix_file_content: + repository_ctx.file( + path = "default.nix", + content = repository_ctx.attr.nix_file_content, + executable = False, + ) + target = repository_ctx.path("default.nix") + else: + target = cp(repository_ctx, repository_ctx.attr.nix_file) + + repository_files = [target] + for dep in repository_ctx.attr.nix_file_deps: + dest = cp(repository_ctx, dep) + repository_files.append(dest) + + # Export all specified Nix files to make them dependencies of a + # nixpkgs_package rule. + export_files = "exports_files({})".format(repository_files) + repository_ctx.file("BUILD", content = export_files) + + # Create a file listing all Nix files of this repository. This + # file is used by the nixpgks_package rule to register all Nix + # files. + repository_ctx.file("nix-file-deps", content = "\n".join(repository_files)) + + # Make "@nixpkgs" (syntactic sugar for "@nixpkgs//:nixpkgs") a valid + # label for the target Nix file. + repository_ctx.symlink(target, repository_ctx.name) + +nixpkgs_local_repository = repository_rule( + implementation = _nixpkgs_local_repository_impl, + attrs = { + "nix_file": attr.label( + allow_single_file = [".nix"], + doc = "A file containing an expression for a Nix derivation.", + ), + "nix_file_deps": attr.label_list( + doc = "Dependencies of `nix_file` if any.", + ), + "nix_file_content": attr.string( + doc = "An expression for a Nix derivation.", + ), + }, + doc = """\ +Create an external repository representing the content of Nixpkgs, based on a Nix expression stored locally or provided inline. One of `nix_file` or `nix_file_content` must be provided. +""", +) + +def _nixpkgs_package_impl(repository_ctx): + repository = repository_ctx.attr.repository + repositories = repository_ctx.attr.repositories + + # Is nix supported on this platform? + not_supported = not is_supported_platform(repository_ctx) + + # Should we fail if Nix is not supported? + fail_not_supported = repository_ctx.attr.fail_not_supported + + if repository and repositories or not repository and not repositories: + fail("Specify one of 'repository' or 'repositories' (but not both).") + elif repository: + repositories = {repository_ctx.attr.repository: "nixpkgs"} + + # If true, a BUILD file will be created from a template if it does not + # exits. + # However this will happen AFTER the nix-build command. + create_build_file_if_needed = False + if repository_ctx.attr.build_file and repository_ctx.attr.build_file_content: + fail("Specify one of 'build_file' or 'build_file_content', but not both.") + elif repository_ctx.attr.build_file: + repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") + elif repository_ctx.attr.build_file_content: + repository_ctx.file("BUILD", content = repository_ctx.attr.build_file_content) + else: + # No user supplied build file, we may create the default one. + create_build_file_if_needed = True + + strFailureImplicitNixpkgs = ( + "One of 'repositories', 'nix_file' or 'nix_file_content' must be provided. " + + "The NIX_PATH environment variable is not inherited." + ) + + expr_args = [] + if repository_ctx.attr.nix_file and repository_ctx.attr.nix_file_content: + fail("Specify one of 'nix_file' or 'nix_file_content', but not both.") + elif repository_ctx.attr.nix_file: + nix_file = cp(repository_ctx, repository_ctx.attr.nix_file) + expr_args = [repository_ctx.path(nix_file)] + elif repository_ctx.attr.nix_file_content: + expr_args = ["-E", repository_ctx.attr.nix_file_content] + elif not repositories: + fail(strFailureImplicitNixpkgs) + else: + expr_args = ["-E", "import { config = {}; overlays = []; }"] + + nix_file_deps = {} + for dep in repository_ctx.attr.nix_file_deps: + nix_file_deps[dep] = cp(repository_ctx, dep) + + expr_args.extend([ + "-A", + repository_ctx.attr.attribute_path if repository_ctx.attr.nix_file or repository_ctx.attr.nix_file_content else repository_ctx.attr.attribute_path or repository_ctx.attr.name, + # Creating an out link prevents nix from garbage collecting the store path. + # nixpkgs uses `nix-support/` for such house-keeping files, so we mirror them + # and use `bazel-support/`, under the assumption that no nix package has + # a file named `bazel-support` in its root. + # A `bazel clean` deletes the symlink and thus nix is free to garbage collect + # the store path. + "--out-link", + "bazel-support/nix-out-link", + ]) + + expr_args.extend([ + expand_location( + repository_ctx = repository_ctx, + string = opt, + labels = nix_file_deps, + attr = "nixopts", + ) + for opt in repository_ctx.attr.nixopts + ]) + + for repo in repositories.keys(): + path = str(repository_ctx.path(repo).dirname) + "/nix-file-deps" + if repository_ctx.path(path).exists: + content = repository_ctx.read(path) + for f in content.splitlines(): + # Hack: this is to register all Nix files as dependencies + # of this rule (see issue #113) + repository_ctx.path(repo.relative(":{}".format(f))) + + # If repositories is not set, leave empty so nix will fail + # unless a pinned nixpkgs is set in the `nix_file` attribute. + nix_path = [ + "{}={}".format(prefix, repository_ctx.path(repo)) + for (repo, prefix) in repositories.items() + ] + if not (repositories or repository_ctx.attr.nix_file or repository_ctx.attr.nix_file_content): + fail(strFailureImplicitNixpkgs) + + for dir in nix_path: + expr_args.extend(["-I", dir]) + + if not_supported and fail_not_supported: + fail("Platform is not supported: nix-build not found in PATH. See attribute fail_not_supported if you don't want to use Nix.") + elif not_supported: + return + else: + nix_build_path = executable_path( + repository_ctx, + "nix-build", + extra_msg = "See: https://nixos.org/nix/", + ) + nix_build = [nix_build_path] + expr_args + + # Large enough integer that Bazel can still parse. We don't have + # access to MAX_INT and 0 is not a valid timeout so this is as good + # as we can do. The value shouldn't be too large to avoid errors on + # macOS, see https://github.com/tweag/rules_nixpkgs/issues/92. + timeout = 8640000 + repository_ctx.report_progress("Building Nix derivation") + exec_result = execute_or_fail( + repository_ctx, + nix_build, + failure_message = "Cannot build Nix attribute '{}'.".format( + repository_ctx.attr.attribute_path, + ), + quiet = repository_ctx.attr.quiet, + timeout = timeout, + ) + output_path = exec_result.stdout.splitlines()[-1] + + # ensure that the output is a directory + test_path = repository_ctx.which("test") + execute_or_fail( + repository_ctx, + [test_path, "-d", output_path], + failure_message = "nixpkgs_package '@{}' outputs a single file which is not supported by rules_nixpkgs. Please only use directories.".format( + repository_ctx.name, + ), + ) + + # Build a forest of symlinks (like new_local_package() does) to the + # Nix store. + for target in find_children(repository_ctx, output_path): + basename = target.rpartition("/")[-1] + repository_ctx.symlink(target, basename) + + # Create a default BUILD file only if it does not exists and is not + # provided by `build_file` or `build_file_content`. + if create_build_file_if_needed: + p = repository_ctx.path("BUILD") + if not p.exists: + repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) + +_nixpkgs_package = repository_rule( + implementation = _nixpkgs_package_impl, + attrs = { + "attribute_path": attr.string(), + "nix_file": attr.label(allow_single_file = [".nix"]), + "nix_file_deps": attr.label_list(), + "nix_file_content": attr.string(), + "repositories": attr.label_keyed_string_dict(), + "repository": attr.label(), + "build_file": attr.label(), + "build_file_content": attr.string(), + "nixopts": attr.string_list(), + "quiet": attr.bool(), + "fail_not_supported": attr.bool(default = True, doc = """ + If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. + """), + }, +) + +def nixpkgs_package( + name, + attribute_path = "", + nix_file = None, + nix_file_deps = [], + nix_file_content = "", + repository = None, + repositories = {}, + build_file = None, + build_file_content = "", + nixopts = [], + quiet = False, + fail_not_supported = True, + **kwargs): + """Make the content of a Nixpkgs package available in the Bazel workspace. + + If `repositories` is not specified, you must provide a nixpkgs clone in `nix_file` or `nix_file_content`. + + Args: + name: A unique name for this repository. + attribute_path: Select an attribute from the top-level Nix expression being evaluated. The attribute path is a sequence of attribute names separated by dots. + nix_file: A file containing an expression for a Nix derivation. + nix_file_deps: Dependencies of `nix_file` if any. + nix_file_content: An expression for a Nix derivation. + repository: A repository label identifying which Nixpkgs to use. Equivalent to `repositories = { "nixpkgs": ...}` + repositories: A dictionary mapping `NIX_PATH` entries to repository labels. + + Setting it to + ``` + repositories = { "myrepo" : "//:myrepo" } + ``` + for example would replace all instances of `` in the called nix code by the path to the target `"//:myrepo"`. See the [relevant section in the nix manual](https://nixos.org/nix/manual/#env-NIX_PATH) for more information. + + Specify one of `repository` or `repositories`. + build_file: The file to use as the BUILD file for this repository. + + Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. + + For common use cases we provide filegroups that expose certain files as targets: + +
+
:bin
+
Everything in the bin/ directory.
+
:lib
+
All .so and .a files that can be found in subdirectories of lib/.
+
:include
+
All .h files that can be found in subdirectories of bin/.
+
+ + If you need different files from the nix package, you can reference them like this: + ``` + package(default_visibility = [ "//visibility:public" ]) + filegroup( + name = "our-docs", + srcs = glob(["share/doc/ourpackage/**/*"]), + ) + ``` + See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). + build_file_content: Like `build_file`, but a string of the contents instead of a file name. + nixopts: Extra flags to pass when calling Nix. + quiet: Whether to hide the output of the Nix command. + fail_not_supported: If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. + """ + kwargs.update( + name = name, + attribute_path = attribute_path, + nix_file = nix_file, + nix_file_deps = nix_file_deps, + nix_file_content = nix_file_content, + repository = repository, + repositories = repositories, + build_file = build_file, + build_file_content = build_file_content, + nixopts = nixopts, + quiet = quiet, + fail_not_supported = fail_not_supported, + ) + + # Because of https://github.com/bazelbuild/bazel/issues/7989 we can't + # directly pass a dict from strings to labels to the rule (which we'd like + # for the `repositories` arguments), but we can pass a dict from labels to + # strings. So we swap the keys and the values (assuming they all are + # distinct). + if "repositories" in kwargs: + inversed_repositories = {value: key for (key, value) in kwargs["repositories"].items()} + kwargs["repositories"] = inversed_repositories + + _nixpkgs_package(**kwargs) diff --git a/core/util.bzl b/core/util.bzl new file mode 100644 index 000000000..8ba979c6c --- /dev/null +++ b/core/util.bzl @@ -0,0 +1,99 @@ +load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "get_cpu_value", +) + +def is_supported_platform(repository_ctx): + return repository_ctx.which("nix-build") != None + +def cp(repository_ctx, src, dest = None): + """Copy the given file into the external repository root. + + Args: + repository_ctx: The repository context of the current repository rule. + src: The source file. Must be a Label if dest is None. + dest: Optional, The target path within the current repository root. + By default the relative path to the repository root is preserved. + + Returns: + The dest value + """ + if dest == None: + if type(src) != "Label": + fail("src must be a Label if dest is not specified explicitly.") + dest = "/".join([ + component + for component in [src.workspace_root, src.package, src.name] + if component + ]) + + # Copy the file + repository_ctx.file( + repository_ctx.path(dest), + repository_ctx.read(repository_ctx.path(src)), + executable = False, + legacy_utf8 = False, + ) + + # Copy the executable bit of the source + # This is important to ensure that copied binaries are executable. + # Windows may not have chmod in path and doesn't have executable bits anyway. + if get_cpu_value(repository_ctx) != "x64_windows": + repository_ctx.execute([ + repository_ctx.which("chmod"), + "--reference", + repository_ctx.path(src), + repository_ctx.path(dest), + ]) + + return dest + +def execute_or_fail(repository_ctx, arguments, failure_message = "", *args, **kwargs): + """Call repository_ctx.execute() and fail if non-zero return code.""" + result = repository_ctx.execute(arguments, *args, **kwargs) + if result.return_code: + outputs = dict( + failure_message = failure_message, + arguments = arguments, + return_code = result.return_code, + stderr = result.stderr, + ) + fail(""" +{failure_message} +Command: {arguments} +Return code: {return_code} +Error output: +{stderr} +""".format(**outputs)) + return result + +def label_string(label): + """Convert the given (optional) Label to a string.""" + if not label: + return "None" + else: + return '"%s"' % label + +def executable_path(repository_ctx, exe_name, extra_msg = ""): + """Try to find the executable, fail with an error.""" + path = repository_ctx.which(exe_name) + if path == None: + fail("Could not find the `{}` executable in PATH.{}\n" + .format(exe_name, " " + extra_msg if extra_msg else "")) + return path + +def find_children(repository_ctx, target_dir): + find_args = [ + executable_path(repository_ctx, "find"), + "-L", + target_dir, + "-maxdepth", + "1", + # otherwise the directory is printed as well + "-mindepth", + "1", + # filenames can contain \n + "-print0", + ] + exec_result = execute_or_fail(repository_ctx, find_args) + return exec_result.stdout.rstrip("\000").split("\000") diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index f837b3ce5..7aeaae126 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -42,6 +42,8 @@ bzl_library( ], visibility = ["//visibility:public"], deps = [ + "//core:nixpkgs.bzl", + "//core:util.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 16fe59ce6..9a8b07c69 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -10,352 +10,26 @@ load( "write_builtin_include_directory_paths", ) load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load(":private/location_expansion.bzl", "expand_location") -load(":private/constraints.bzl", "ensure_constraints") - -def _nixpkgs_git_repository_impl(repository_ctx): - repository_ctx.file( - "BUILD", - content = 'filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"])', - ) - - # Make "@nixpkgs" (syntactic sugar for "@nixpkgs//:nixpkgs") a valid - # label for default.nix. - repository_ctx.symlink("default.nix", repository_ctx.name) - - repository_ctx.download_and_extract( - url = "%s/archive/%s.tar.gz" % (repository_ctx.attr.remote, repository_ctx.attr.revision), - stripPrefix = "nixpkgs-" + repository_ctx.attr.revision, - sha256 = repository_ctx.attr.sha256, - ) - -nixpkgs_git_repository = repository_rule( - implementation = _nixpkgs_git_repository_impl, - attrs = { - "revision": attr.string( - mandatory = True, - doc = "Git commit hash or tag identifying the version of Nixpkgs to use.", - ), - "remote": attr.string( - default = "https://github.com/NixOS/nixpkgs", - doc = "The URI of the remote Git repository. This must be a HTTP URL. There is currently no support for authentication. Defaults to [upstream nixpkgs](https://github.com/NixOS/nixpkgs).", - ), - "sha256": attr.string(doc = "The SHA256 used to verify the integrity of the repository."), - }, - doc = """\ -Name a specific revision of Nixpkgs on GitHub or a local checkout. -""", -) - -def _nixpkgs_local_repository_impl(repository_ctx): - if not bool(repository_ctx.attr.nix_file) != \ - bool(repository_ctx.attr.nix_file_content): - fail("Specify one of 'nix_file' or 'nix_file_content' (but not both).") - if repository_ctx.attr.nix_file_content: - repository_ctx.file( - path = "default.nix", - content = repository_ctx.attr.nix_file_content, - executable = False, - ) - target = repository_ctx.path("default.nix") - else: - target = _cp(repository_ctx, repository_ctx.attr.nix_file) - - repository_files = [target] - for dep in repository_ctx.attr.nix_file_deps: - dest = _cp(repository_ctx, dep) - repository_files.append(dest) - - # Export all specified Nix files to make them dependencies of a - # nixpkgs_package rule. - export_files = "exports_files({})".format(repository_files) - repository_ctx.file("BUILD", content = export_files) - - # Create a file listing all Nix files of this repository. This - # file is used by the nixpgks_package rule to register all Nix - # files. - repository_ctx.file("nix-file-deps", content = "\n".join(repository_files)) - - # Make "@nixpkgs" (syntactic sugar for "@nixpkgs//:nixpkgs") a valid - # label for the target Nix file. - repository_ctx.symlink(target, repository_ctx.name) - -nixpkgs_local_repository = repository_rule( - implementation = _nixpkgs_local_repository_impl, - attrs = { - "nix_file": attr.label( - allow_single_file = [".nix"], - doc = "A file containing an expression for a Nix derivation.", - ), - "nix_file_deps": attr.label_list( - doc = "Dependencies of `nix_file` if any.", - ), - "nix_file_content": attr.string( - doc = "An expression for a Nix derivation.", - ), - }, - doc = """\ -Create an external repository representing the content of Nixpkgs, based on a Nix expression stored locally or provided inline. One of `nix_file` or `nix_file_content` must be provided. -""", +load( + "//core:nixpkgs.bzl", + _nixpkgs_git_repository = "nixpkgs_git_repository", + _nixpkgs_local_repository = "nixpkgs_local_repository", + _nixpkgs_package = "nixpkgs_package", ) - -def _is_supported_platform(repository_ctx): - return repository_ctx.which("nix-build") != None - -def _nixpkgs_package_impl(repository_ctx): - repository = repository_ctx.attr.repository - repositories = repository_ctx.attr.repositories - - # Is nix supported on this platform? - not_supported = not _is_supported_platform(repository_ctx) - - # Should we fail if Nix is not supported? - fail_not_supported = repository_ctx.attr.fail_not_supported - - if repository and repositories or not repository and not repositories: - fail("Specify one of 'repository' or 'repositories' (but not both).") - elif repository: - repositories = {repository_ctx.attr.repository: "nixpkgs"} - - # If true, a BUILD file will be created from a template if it does not - # exits. - # However this will happen AFTER the nix-build command. - create_build_file_if_needed = False - if repository_ctx.attr.build_file and repository_ctx.attr.build_file_content: - fail("Specify one of 'build_file' or 'build_file_content', but not both.") - elif repository_ctx.attr.build_file: - repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") - elif repository_ctx.attr.build_file_content: - repository_ctx.file("BUILD", content = repository_ctx.attr.build_file_content) - else: - # No user supplied build file, we may create the default one. - create_build_file_if_needed = True - - strFailureImplicitNixpkgs = ( - "One of 'repositories', 'nix_file' or 'nix_file_content' must be provided. " + - "The NIX_PATH environment variable is not inherited." - ) - - expr_args = [] - if repository_ctx.attr.nix_file and repository_ctx.attr.nix_file_content: - fail("Specify one of 'nix_file' or 'nix_file_content', but not both.") - elif repository_ctx.attr.nix_file: - nix_file = _cp(repository_ctx, repository_ctx.attr.nix_file) - expr_args = [repository_ctx.path(nix_file)] - elif repository_ctx.attr.nix_file_content: - expr_args = ["-E", repository_ctx.attr.nix_file_content] - elif not repositories: - fail(strFailureImplicitNixpkgs) - else: - expr_args = ["-E", "import { config = {}; overlays = []; }"] - - nix_file_deps = {} - for dep in repository_ctx.attr.nix_file_deps: - nix_file_deps[dep] = _cp(repository_ctx, dep) - - expr_args.extend([ - "-A", - repository_ctx.attr.attribute_path if repository_ctx.attr.nix_file or repository_ctx.attr.nix_file_content else repository_ctx.attr.attribute_path or repository_ctx.attr.name, - # Creating an out link prevents nix from garbage collecting the store path. - # nixpkgs uses `nix-support/` for such house-keeping files, so we mirror them - # and use `bazel-support/`, under the assumption that no nix package has - # a file named `bazel-support` in its root. - # A `bazel clean` deletes the symlink and thus nix is free to garbage collect - # the store path. - "--out-link", - "bazel-support/nix-out-link", - ]) - - expr_args.extend([ - expand_location( - repository_ctx = repository_ctx, - string = opt, - labels = nix_file_deps, - attr = "nixopts", - ) - for opt in repository_ctx.attr.nixopts - ]) - - for repo in repositories.keys(): - path = str(repository_ctx.path(repo).dirname) + "/nix-file-deps" - if repository_ctx.path(path).exists: - content = repository_ctx.read(path) - for f in content.splitlines(): - # Hack: this is to register all Nix files as dependencies - # of this rule (see issue #113) - repository_ctx.path(repo.relative(":{}".format(f))) - - # If repositories is not set, leave empty so nix will fail - # unless a pinned nixpkgs is set in the `nix_file` attribute. - nix_path = [ - "{}={}".format(prefix, repository_ctx.path(repo)) - for (repo, prefix) in repositories.items() - ] - if not (repositories or repository_ctx.attr.nix_file or repository_ctx.attr.nix_file_content): - fail(strFailureImplicitNixpkgs) - - for dir in nix_path: - expr_args.extend(["-I", dir]) - - if not_supported and fail_not_supported: - fail("Platform is not supported: nix-build not found in PATH. See attribute fail_not_supported if you don't want to use Nix.") - elif not_supported: - return - else: - nix_build_path = _executable_path( - repository_ctx, - "nix-build", - extra_msg = "See: https://nixos.org/nix/", - ) - nix_build = [nix_build_path] + expr_args - - # Large enough integer that Bazel can still parse. We don't have - # access to MAX_INT and 0 is not a valid timeout so this is as good - # as we can do. The value shouldn't be too large to avoid errors on - # macOS, see https://github.com/tweag/rules_nixpkgs/issues/92. - timeout = 8640000 - repository_ctx.report_progress("Building Nix derivation") - exec_result = _execute_or_fail( - repository_ctx, - nix_build, - failure_message = "Cannot build Nix attribute '{}'.".format( - repository_ctx.attr.attribute_path, - ), - quiet = repository_ctx.attr.quiet, - timeout = timeout, - ) - output_path = exec_result.stdout.splitlines()[-1] - - # ensure that the output is a directory - test_path = repository_ctx.which("test") - _execute_or_fail( - repository_ctx, - [test_path, "-d", output_path], - failure_message = "nixpkgs_package '@{}' outputs a single file which is not supported by rules_nixpkgs. Please only use directories.".format( - repository_ctx.name, - ), - ) - - # Build a forest of symlinks (like new_local_package() does) to the - # Nix store. - for target in _find_children(repository_ctx, output_path): - basename = target.rpartition("/")[-1] - repository_ctx.symlink(target, basename) - - # Create a default BUILD file only if it does not exists and is not - # provided by `build_file` or `build_file_content`. - if create_build_file_if_needed: - p = repository_ctx.path("BUILD") - if not p.exists: - repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) - -_nixpkgs_package = repository_rule( - implementation = _nixpkgs_package_impl, - attrs = { - "attribute_path": attr.string(), - "nix_file": attr.label(allow_single_file = [".nix"]), - "nix_file_deps": attr.label_list(), - "nix_file_content": attr.string(), - "repositories": attr.label_keyed_string_dict(), - "repository": attr.label(), - "build_file": attr.label(), - "build_file_content": attr.string(), - "nixopts": attr.string_list(), - "quiet": attr.bool(), - "fail_not_supported": attr.bool(default = True, doc = """ - If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. - """), - }, +load( + "//core:util.bzl", + "ensure_constraints", + "execute_or_fail", + "find_children", + "is_supported_platform", + "label_string", ) +load(":private/constraints.bzl", "ensure_constraints") -def nixpkgs_package( - name, - attribute_path = "", - nix_file = None, - nix_file_deps = [], - nix_file_content = "", - repository = None, - repositories = {}, - build_file = None, - build_file_content = "", - nixopts = [], - quiet = False, - fail_not_supported = True, - **kwargs): - """Make the content of a Nixpkgs package available in the Bazel workspace. - - If `repositories` is not specified, you must provide a nixpkgs clone in `nix_file` or `nix_file_content`. - - Args: - name: A unique name for this repository. - attribute_path: Select an attribute from the top-level Nix expression being evaluated. The attribute path is a sequence of attribute names separated by dots. - nix_file: A file containing an expression for a Nix derivation. - nix_file_deps: Dependencies of `nix_file` if any. - nix_file_content: An expression for a Nix derivation. - repository: A repository label identifying which Nixpkgs to use. Equivalent to `repositories = { "nixpkgs": ...}` - repositories: A dictionary mapping `NIX_PATH` entries to repository labels. - - Setting it to - ``` - repositories = { "myrepo" : "//:myrepo" } - ``` - for example would replace all instances of `` in the called nix code by the path to the target `"//:myrepo"`. See the [relevant section in the nix manual](https://nixos.org/nix/manual/#env-NIX_PATH) for more information. - - Specify one of `repository` or `repositories`. - build_file: The file to use as the BUILD file for this repository. - - Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. - - For common use cases we provide filegroups that expose certain files as targets: - -
-
:bin
-
Everything in the bin/ directory.
-
:lib
-
All .so and .a files that can be found in subdirectories of lib/.
-
:include
-
All .h files that can be found in subdirectories of bin/.
-
- - If you need different files from the nix package, you can reference them like this: - ``` - package(default_visibility = [ "//visibility:public" ]) - filegroup( - name = "our-docs", - srcs = glob(["share/doc/ourpackage/**/*"]), - ) - ``` - See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). - build_file_content: Like `build_file`, but a string of the contents instead of a file name. - nixopts: Extra flags to pass when calling Nix. - quiet: Whether to hide the output of the Nix command. - fail_not_supported: If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. - """ - kwargs.update( - name = name, - attribute_path = attribute_path, - nix_file = nix_file, - nix_file_deps = nix_file_deps, - nix_file_content = nix_file_content, - repository = repository, - repositories = repositories, - build_file = build_file, - build_file_content = build_file_content, - nixopts = nixopts, - quiet = quiet, - fail_not_supported = fail_not_supported, - ) - - # Because of https://github.com/bazelbuild/bazel/issues/7989 we can't - # directly pass a dict from strings to labels to the rule (which we'd like - # for the `repositories` arguments), but we can pass a dict from labels to - # strings. So we swap the keys and the values (assuming they all are - # distinct). - if "repositories" in kwargs: - inversed_repositories = {value: key for (key, value) in kwargs["repositories"].items()} - kwargs["repositories"] = inversed_repositories - - _nixpkgs_package(**kwargs) +# aliases for backwards compatibility prior to `bzlmod` +nixpkgs_git_repository = _nixpkgs_git_repository +nixpkgs_local_repository = _nixpkgs_local_repository +nixpkgs_package = _nixpkgs_package def _parse_cc_toolchain_info(content, filename): """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. @@ -475,7 +149,7 @@ def _nixpkgs_cc_toolchain_config_impl(repository_ctx): ] repository_ctx.file( "module.modulemap", - _execute_or_fail( + execute_or_fail( repository_ctx, generate_system_module_map + info.cxx_builtin_include_directories, "Failed to generate system module map.", @@ -742,7 +416,7 @@ def _readlink(repository_ctx, path): def nixpkgs_cc_autoconf_impl(repository_ctx): cpu_value = get_cpu_value(repository_ctx) - if not _is_supported_platform(repository_ctx): + if not is_supported_platform(repository_ctx): cc_autoconf_impl(repository_ctx) return @@ -754,14 +428,14 @@ def nixpkgs_cc_autoconf_impl(repository_ctx): workspace_file_path = repository_ctx.path( Label("@nixpkgs_cc_toolchain//:WORKSPACE"), ) - workspace_root = _execute_or_fail( + workspace_root = execute_or_fail( repository_ctx, ["dirname", workspace_file_path], ).stdout.rstrip() # Make a list of all available tools in the Nix derivation. Override # the Bazel autoconfiguration with the tools we found. - bin_contents = _find_children(repository_ctx, workspace_root + "/bin") + bin_contents = find_children(repository_ctx, workspace_root + "/bin") overriden_tools = { tool: _readlink(repository_ctx, entry) for entry in bin_contents @@ -1114,8 +788,8 @@ toolchain( target_compatible_with = {target_constraints}, ) """.format( - python2_runtime = _label_string(repository_ctx.attr.python2_runtime), - python3_runtime = _label_string(repository_ctx.attr.python3_runtime), + python2_runtime = label_string(repository_ctx.attr.python2_runtime), + python3_runtime = label_string(repository_ctx.attr.python3_runtime), exec_constraints = exec_constraints, target_constraints = target_constraints, )) @@ -1407,95 +1081,3 @@ def nixpkgs_sh_posix_configure( native.register_toolchains( "@{}//:nixpkgs_sh_posix_toolchain".format(name + "_toolchain"), ) - -def _execute_or_fail(repository_ctx, arguments, failure_message = "", *args, **kwargs): - """Call repository_ctx.execute() and fail if non-zero return code.""" - result = repository_ctx.execute(arguments, *args, **kwargs) - if result.return_code: - outputs = dict( - failure_message = failure_message, - arguments = arguments, - return_code = result.return_code, - stderr = result.stderr, - ) - fail(""" -{failure_message} -Command: {arguments} -Return code: {return_code} -Error output: -{stderr} -""".format(**outputs)) - return result - -def _find_children(repository_ctx, target_dir): - find_args = [ - _executable_path(repository_ctx, "find"), - "-L", - target_dir, - "-maxdepth", - "1", - # otherwise the directory is printed as well - "-mindepth", - "1", - # filenames can contain \n - "-print0", - ] - exec_result = _execute_or_fail(repository_ctx, find_args) - return exec_result.stdout.rstrip("\000").split("\000") - -def _executable_path(repository_ctx, exe_name, extra_msg = ""): - """Try to find the executable, fail with an error.""" - path = repository_ctx.which(exe_name) - if path == None: - fail("Could not find the `{}` executable in PATH.{}\n" - .format(exe_name, " " + extra_msg if extra_msg else "")) - return path - -def _cp(repository_ctx, src, dest = None): - """Copy the given file into the external repository root. - - Args: - repository_ctx: The repository context of the current repository rule. - src: The source file. Must be a Label if dest is None. - dest: Optional, The target path within the current repository root. - By default the relative path to the repository root is preserved. - - Returns: - The dest value - """ - if dest == None: - if type(src) != "Label": - fail("src must be a Label if dest is not specified explicitly.") - dest = "/".join([ - component - for component in [src.workspace_root, src.package, src.name] - if component - ]) - - # Copy the file - repository_ctx.file( - repository_ctx.path(dest), - repository_ctx.read(repository_ctx.path(src)), - executable = False, - legacy_utf8 = False, - ) - - # Copy the executable bit of the source - # This is important to ensure that copied binaries are executable. - # Windows may not have chmod in path and doesn't have executable bits anyway. - if get_cpu_value(repository_ctx) != "x64_windows": - repository_ctx.execute([ - repository_ctx.which("chmod"), - "--reference", - repository_ctx.path(src), - repository_ctx.path(dest), - ]) - - return dest - -def _label_string(label): - """Convert the given (optional) Label to a string.""" - if not label: - return "None" - else: - return '"%s"' % label diff --git a/nixpkgs/toolchains/rust.bzl b/nixpkgs/toolchains/rust.bzl index 166abf2d8..cbd7e62fe 100644 --- a/nixpkgs/toolchains/rust.bzl +++ b/nixpkgs/toolchains/rust.bzl @@ -1,4 +1,4 @@ -load("//nixpkgs:nixpkgs.bzl", "nixpkgs_package") +load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") load("//nixpkgs:private/constraints.bzl", "ensure_constraints") # Adapted from rules_rust toolchain BUILD: From 389f8c3766a14b79f38443873aea84daf82fefa0 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 7 Mar 2022 13:23:36 +0100 Subject: [PATCH 02/25] docs generation: reset record separator after match otherwise the entire file will be consumed as the next line, skipping everything after the last match. --- docs/BUILD.bazel | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 31c2ef83d..f238734c7 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -42,11 +42,13 @@ genrule( if (/{{nixpkgs}}/) { RS="\\0"; getline content <"$(execpath nixpkgs.md)"; - print content + print content; + RS="\\n"; } else if (/{{toolchains_go}}/) { RS="\\0"; getline content <"$(execpath toolchains/go.md)"; - print content + print content; + RS="\\n"; } else { print } From 7268ea3d94db7e91cce8316770f70e4fcb07e3d7 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 7 Mar 2022 13:25:38 +0100 Subject: [PATCH 03/25] README: fix typos --- README.md | 31 ++++++++++++++++++++++++++++++- docs/README.md.tpl | 10 +++++----- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e7d15cae2..32081ac24 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository", " load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure") # optional ``` -If you use `rules_nixpkgs` to configure a toolchain then you will also need to +If you use `rules_nixpkgs` to configure a toolchain, then you will also need to configure the build platform to include the `@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix` constraint. For example by adding the following to `.bazelrc`: @@ -1394,6 +1394,7 @@ optional. + Rules for importing a Go toolchain from Nixpkgs. @@ -1625,3 +1626,31 @@ Whether to hide the output of the Nix command. + +## Migration from older releases + +### `path` Attribute (removed in 0.3) + +`path` was an attribute from the early days of `rules_nixpkgs`, and +its ability to reference arbitrary paths is a danger to build hermeticity. + +Replace it with either `nixpkgs_git_repository` if you need +a specific version of `nixpkgs`. If you absolutely *must* depend on a +local folder, use Bazel’s +[`local_repository` workspace rule](https://docs.bazel.build/versions/master/be/workspace.html#local_repository). +Both approaches work well with the `repositories` attribute of `nixpkgs_package`. + +```bzl +local_repository( + name = "local-nixpkgs", + path = "/path/to/nixpkgs", +) + +nixpkgs_package( + name = "somepackage", + repositories = { + "nixpkgs": "@local-nixpkgs//:default.nix", + }, + … +) +``` diff --git a/docs/README.md.tpl b/docs/README.md.tpl index 9f658ca5e..997f78b64 100644 --- a/docs/README.md.tpl +++ b/docs/README.md.tpl @@ -52,7 +52,7 @@ load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository", " load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure") # optional ``` -If you use `rules_nixpkgs` to configure a toolchain then you will also need to +If you use `rules_nixpkgs` to configure a toolchain, then you will also need to configure the build platform to include the `@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix` constraint. For example by adding the following to `.bazelrc`: @@ -82,16 +82,16 @@ nixpkgs_package( {{toolchains_go}} -## Migration +## Migration from older releases -### `path` Attribute +### `path` Attribute (removed in 0.3) `path` was an attribute from the early days of `rules_nixpkgs`, and -its ability to reference arbitrary paths a danger to build hermeticity. +its ability to reference arbitrary paths is a danger to build hermeticity. Replace it with either `nixpkgs_git_repository` if you need a specific version of `nixpkgs`. If you absolutely *must* depend on a -local folder, use bazel’s +local folder, use Bazel’s [`local_repository` workspace rule](https://docs.bazel.build/versions/master/be/workspace.html#local_repository). Both approaches work well with the `repositories` attribute of `nixpkgs_package`. From d19f8827c3720414b5833136b949a5eea8487534 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 7 Mar 2022 13:46:27 +0100 Subject: [PATCH 04/25] generate separate README for core module --- core/BUILD.bazel | 23 +++ core/README.md | 392 ++++++++++++++++++++++++++++++++++++++++ docs/BUILD.bazel | 38 +++- docs/README_core.md.tpl | 5 + docs/update-readme.sh | 3 + 5 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 core/README.md create mode 100644 docs/README_core.md.tpl diff --git a/core/BUILD.bazel b/core/BUILD.bazel index 00d87a19e..ecdfc7623 100644 --- a/core/BUILD.bazel +++ b/core/BUILD.bazel @@ -1,5 +1,28 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + package(default_visibility = ["//visibility:public"]) exports_files([ "nixpkgs.bzl", ]) + +bzl_library( + name = "bazel_tools", + srcs = [ + "@bazel_tools//tools:bzl_srcs", + ], +) + +bzl_library( + name = "core", + srcs = [ + "nixpkgs.bzl", + "util.bzl", + "//nixpkgs:private/location_expansion.bzl", + ], + visibility = ["//visibility:public"], + deps = [ + ":bazel_tools", + "@bazel_skylib//lib:paths", + ], +) \ No newline at end of file diff --git a/core/README.md b/core/README.md new file mode 100644 index 000000000..fcd14d527 --- /dev/null +++ b/core/README.md @@ -0,0 +1,392 @@ +# Nixpkgs for Bazel + +## Rules + + + + + + + +### nixpkgs_git_repository + +
+nixpkgs_git_repository(name, remote, revision, sha256)
+
+ +Name a specific revision of Nixpkgs on GitHub or a local checkout. + + +#### Attributes + + ++++ + + + + + + + + + + + + + + + + + + +
name + +Name; required + +

+ +A unique name for this repository. + +

+
remote + +String; optional + +

+ +The URI of the remote Git repository. This must be a HTTP URL. There is currently no support for authentication. Defaults to [upstream nixpkgs](https://github.com/NixOS/nixpkgs). + +

+
revision + +String; required + +

+ +Git commit hash or tag identifying the version of Nixpkgs to use. + +

+
sha256 + +String; optional + +

+ +The SHA256 used to verify the integrity of the repository. + +

+
+ + + + +### nixpkgs_local_repository + +
+nixpkgs_local_repository(name, nix_file, nix_file_content, nix_file_deps)
+
+ +Create an external repository representing the content of Nixpkgs, based on a Nix expression stored locally or provided inline. One of `nix_file` or `nix_file_content` must be provided. + + +#### Attributes + + ++++ + + + + + + + + + + + + + + + + + + +
name + +Name; required + +

+ +A unique name for this repository. + +

+
nix_file + +Label; optional + +

+ +A file containing an expression for a Nix derivation. + +

+
nix_file_content + +String; optional + +

+ +An expression for a Nix derivation. + +

+
nix_file_deps + +List of labels; optional + +

+ +Dependencies of `nix_file` if any. + +

+
+ + + + +### nixpkgs_package + +
+nixpkgs_package(name, attribute_path, nix_file, nix_file_deps, nix_file_content, repository,
+                repositories, build_file, build_file_content, nixopts, quiet, fail_not_supported,
+                kwargs)
+
+ +Make the content of a Nixpkgs package available in the Bazel workspace. + +If `repositories` is not specified, you must provide a nixpkgs clone in `nix_file` or `nix_file_content`. + + +#### Parameters + + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
name + +required. + +

+ +A unique name for this repository. + +

+
attribute_path + +optional. +default is "" + +

+ +Select an attribute from the top-level Nix expression being evaluated. The attribute path is a sequence of attribute names separated by dots. + +

+
nix_file + +optional. +default is None + +

+ +A file containing an expression for a Nix derivation. + +

+
nix_file_deps + +optional. +default is [] + +

+ +Dependencies of `nix_file` if any. + +

+
nix_file_content + +optional. +default is "" + +

+ +An expression for a Nix derivation. + +

+
repository + +optional. +default is None + +

+ +A repository label identifying which Nixpkgs to use. Equivalent to `repositories = { "nixpkgs": ...}` + +

+
repositories + +optional. +default is {} + +

+ +A dictionary mapping `NIX_PATH` entries to repository labels. + + Setting it to + ``` + repositories = { "myrepo" : "//:myrepo" } + ``` + for example would replace all instances of `` in the called nix code by the path to the target `"//:myrepo"`. See the [relevant section in the nix manual](https://nixos.org/nix/manual/#env-NIX_PATH) for more information. + + Specify one of `repository` or `repositories`. + +

+
build_file + +optional. +default is None + +

+ +The file to use as the BUILD file for this repository. + + Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. + + For common use cases we provide filegroups that expose certain files as targets: + +

+
:bin
+
Everything in the bin/ directory.
+
:lib
+
All .so and .a files that can be found in subdirectories of lib/.
+
:include
+
All .h files that can be found in subdirectories of bin/.
+
+ + If you need different files from the nix package, you can reference them like this: + ``` + package(default_visibility = [ "//visibility:public" ]) + filegroup( + name = "our-docs", + srcs = glob(["share/doc/ourpackage/**/*"]), + ) + ``` + See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). + +

+
build_file_content + +optional. +default is "" + +

+ +Like `build_file`, but a string of the contents instead of a file name. + +

+
nixopts + +optional. +default is [] + +

+ +Extra flags to pass when calling Nix. + +

+
quiet + +optional. +default is False + +

+ +Whether to hide the output of the Nix command. + +

+
fail_not_supported + +optional. +default is True + +

+ +If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. + +

+
kwargs + +optional. + +
+ + + diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index f238734c7..995b32013 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -17,6 +17,18 @@ stardoc( deps = ["//nixpkgs"], ) +stardoc( + name = "core", + out = "core.md", + input = "//core:nixpkgs.bzl", + symbol_names = [ + "nixpkgs_git_repository", + "nixpkgs_local_repository", + "nixpkgs_package", + ], + deps = ["//core"], +) + stardoc( name = "go", out = "toolchains/go.md", @@ -57,6 +69,30 @@ genrule( toolchains = ["@rules_sh//sh/posix:make_variables"], ) +genrule( + name = "readme_core", + srcs = [ + "README_core.md.tpl", + "core.md", + ], + outs = ["core/README.md"], + cmd = """$(POSIX_AWK) \\ + <$(execpath README_core.md.tpl) \\ + >$(OUTS) \\ + '{ + if (/{{core}}/) { + RS="\\0"; + getline content <"$(execpath core.md)"; + print content; + RS="\\n"; + } else { + print + } + }' +""", + toolchains = ["@rules_sh//sh/posix:make_variables"], +) + sh_test( name = "check-readme", srcs = ["check-readme.sh"], @@ -70,6 +106,6 @@ sh_test( sh_binary( name = "update-readme", srcs = ["update-readme.sh"], - data = ["README.md"], + data = ["README.md", "core/README.md"], deps = ["@bazel_tools//tools/bash/runfiles"], ) diff --git a/docs/README_core.md.tpl b/docs/README_core.md.tpl new file mode 100644 index 000000000..6b322bd4d --- /dev/null +++ b/docs/README_core.md.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for Bazel + +## Rules + +{{core}} \ No newline at end of file diff --git a/docs/update-readme.sh b/docs/update-readme.sh index 30ebc5aee..668fa28d5 100755 --- a/docs/update-readme.sh +++ b/docs/update-readme.sh @@ -11,6 +11,8 @@ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ set -euo pipefail new="$(rlocation io_tweag_rules_nixpkgs/docs/README.md)" +new2="$(rlocation io_tweag_rules_nixpkgs/docs/core/README.md)" + # this variable is set by `bazel run` if [ -z "$BUILD_WORKSPACE_DIRECTORY" ]; then @@ -19,3 +21,4 @@ if [ -z "$BUILD_WORKSPACE_DIRECTORY" ]; then fi cp "$new" "$BUILD_WORKSPACE_DIRECTORY/README.md" +cp "$new2" "$BUILD_WORKSPACE_DIRECTORY/core/README.md" From 7f91de8b9681c5d90691178a03391eda87785285 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 7 Mar 2022 13:57:22 +0100 Subject: [PATCH 05/25] move utilities to core module --- core/BUILD.bazel | 1 - core/nixpkgs.bzl | 2 +- core/util.bzl | 158 ++++++++++++++++++++++++- nixpkgs/BUILD.bazel | 2 - nixpkgs/nixpkgs.bzl | 1 - nixpkgs/private/constraints.bzl | 26 ---- nixpkgs/private/location_expansion.bzl | 128 -------------------- nixpkgs/toolchains/foreign_cc.bzl | 2 +- nixpkgs/toolchains/rust.bzl | 2 +- tests/location_expansion_unit_test.bzl | 2 +- 10 files changed, 158 insertions(+), 166 deletions(-) delete mode 100644 nixpkgs/private/constraints.bzl delete mode 100644 nixpkgs/private/location_expansion.bzl diff --git a/core/BUILD.bazel b/core/BUILD.bazel index ecdfc7623..7158bd251 100644 --- a/core/BUILD.bazel +++ b/core/BUILD.bazel @@ -18,7 +18,6 @@ bzl_library( srcs = [ "nixpkgs.bzl", "util.bzl", - "//nixpkgs:private/location_expansion.bzl", ], visibility = ["//visibility:public"], deps = [ diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl index 2a2e55222..847542afb 100644 --- a/core/nixpkgs.bzl +++ b/core/nixpkgs.bzl @@ -1,9 +1,9 @@ -load("//nixpkgs:private/location_expansion.bzl", "expand_location") load( ":util.bzl", "cp", "executable_path", "execute_or_fail", + "expand_location", "find_children", "is_supported_platform", ) diff --git a/core/util.bzl b/core/util.bzl index 8ba979c6c..9f1df6e1c 100644 --- a/core/util.bzl +++ b/core/util.bzl @@ -1,7 +1,5 @@ -load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "get_cpu_value", -) +load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") +load("@bazel_skylib//lib:paths.bzl", "paths") def is_supported_platform(repository_ctx): return repository_ctx.which("nix-build") != None @@ -97,3 +95,155 @@ def find_children(repository_ctx, target_dir): ] exec_result = execute_or_fail(repository_ctx, find_args) return exec_result.stdout.rstrip("\000").split("\000") + +def ensure_constraints(repository_ctx): + """Build exec and target constraints for repository rules. + + If these are user-provided, then they are passed through. Otherwise we build for x86_64 on the current OS. + In either case, exec_constraints always contain the support_nix constraint, so the toolchain can be rejected on non-Nix environments. + + Args: + repository_ctx: The repository context of the current repository rule. + + Returns: + exec_constraints, The generated list of exec constraints + target_constraints, The generated list of target constraints + """ + cpu = get_cpu_value(repository_ctx) + os = {"darwin": "osx"}.get(cpu, "linux") + if not repository_ctx.attr.target_constraints and not repository_ctx.attr.exec_constraints: + target_constraints = ["@platforms//cpu:x86_64"] + target_constraints.append("@platforms//os:{}".format(os)) + exec_constraints = target_constraints + else: + target_constraints = list(repository_ctx.attr.target_constraints) + exec_constraints = list(repository_ctx.attr.exec_constraints) + exec_constraints.append("@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix") + return exec_constraints, target_constraints + +def parse_expand_location(string): + """Parse a string that might contain location expansion commands. + + Generates a list of pairs of command and argument. + The command can have the following values: + - `string`: argument is a string, append it to the result. + - `location`: argument is a label, append its location to the result. + + Attrs: + string: string, The string to parse. + + Returns: + (result, error): + result: The generated list of pairs of command and argument. + error: string or None, This is set if an error occurred. + """ + result = [] + offset = 0 + len_string = len(string) + + # Step through occurrences of `$`. This is bounded by the length of the string. + for _ in range(len_string): + # Find the position of the next `$`. + position = string.find("$", offset) + if position == -1: + position = len_string + + # Append the in-between literal string. + if offset < position: + result.append(("string", string[offset:position])) + + # Terminate at the end of the string. + if position == len_string: + break + + # Parse the `$` command. + if string[position:].startswith("$$"): + # Insert verbatim '$'. + result.append(("string", "$")) + offset = position + 2 + elif string[position:].startswith("$("): + # Expand a location command. + group_start = position + 2 + group_end = string.find(")", group_start) + if group_end == -1: + return (None, "Unbalanced parentheses in location expansion for '{}'.".format(string[position:])) + + group = string[group_start:group_end] + command = None + if group.startswith("location "): + label_str = group[len("location "):] + command = ("location", label_str) + else: + return (None, "Unrecognized location expansion '$({})'.".format(group)) + + result.append(command) + offset = group_end + 1 + else: + return (None, "Unescaped '$' in location expansion at position {} of input.".format(position)) + + return (result, None) + +def resolve_label(label_str, labels): + """Find the label that corresponds to the given string. + + Attr: + label_str: string, String representation of a label. + labels: dict from Label to path: Known label to path mappings. + + Returns: + (path, error): + path: path, The path to the resolved label + error: string or None, This is set if an error occurred. + """ + label_candidates = [ + (lbl, path) + for (lbl, path) in labels.items() + if lbl.relative(label_str) == lbl + ] + + if len(label_candidates) == 0: + return (None, "Unknown label '{}' in location expansion.".format(label_str)) + elif len(label_candidates) > 1: + return (None, "Ambiguous label '{}' in location expansion. Candidates: {}".format( + label_str, + ", ".join([str(lbl) for (lbl, _) in label_candidates]), + )) + + return (label_candidates[0][1], None) + +def expand_location(repository_ctx, string, labels, attr = None): + """Expand `$(location label)` to a path. + + Raises an error on unexpected occurrences of `$`. + Use `$$` to insert a verbatim `$`. + + Attrs: + repository_ctx: The repository rule context. + string: string, Replace instances of `$(location )` in this string. + labels: dict from label to path: Known label to path mappings. + attr: string, The rule attribute to use for error reporting. + + Returns: + The string with all instances of `$(location )` replaced by paths. + """ + (parsed, error) = parse_expand_location(string) + if error != None: + fail(error, attr) + + result = "" + for (command, argument) in parsed: + if command == "string": + result += argument + elif command == "location": + (label, error) = resolve_label(argument, labels) + if error != None: + fail(error, attr) + + result += paths.join(".", paths.relativize( + str(repository_ctx.path(label)), + str(repository_ctx.path(".")), + )) + else: + fail("Internal error: Unknown location expansion command '{}'.".format(command), attr) + + return result diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 7aeaae126..29d31b54f 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -37,8 +37,6 @@ bzl_library( name = "nixpkgs", srcs = [ "nixpkgs.bzl", - "private/constraints.bzl", - "private/location_expansion.bzl", ], visibility = ["//visibility:public"], deps = [ diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 9a8b07c69..655777aa6 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -24,7 +24,6 @@ load( "is_supported_platform", "label_string", ) -load(":private/constraints.bzl", "ensure_constraints") # aliases for backwards compatibility prior to `bzlmod` nixpkgs_git_repository = _nixpkgs_git_repository diff --git a/nixpkgs/private/constraints.bzl b/nixpkgs/private/constraints.bzl deleted file mode 100644 index b35ec0bb9..000000000 --- a/nixpkgs/private/constraints.bzl +++ /dev/null @@ -1,26 +0,0 @@ -load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") - -def ensure_constraints(repository_ctx): - """Build exec and target constraints for repository rules. - - If these are user-provided, then they are passed through. Otherwise we build for x86_64 on the current OS. - In either case, exec_constraints always contain the support_nix constraint, so the toolchain can be rejected on non-Nix environments. - - Args: - repository_ctx: The repository context of the current repository rule. - - Returns: - exec_constraints, The generated list of exec constraints - target_constraints, The generated list of target constraints - """ - cpu = get_cpu_value(repository_ctx) - os = {"darwin": "osx"}.get(cpu, "linux") - if not repository_ctx.attr.target_constraints and not repository_ctx.attr.exec_constraints: - target_constraints = ["@platforms//cpu:x86_64"] - target_constraints.append("@platforms//os:{}".format(os)) - exec_constraints = target_constraints - else: - target_constraints = list(repository_ctx.attr.target_constraints) - exec_constraints = list(repository_ctx.attr.exec_constraints) - exec_constraints.append("@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix") - return exec_constraints, target_constraints diff --git a/nixpkgs/private/location_expansion.bzl b/nixpkgs/private/location_expansion.bzl deleted file mode 100644 index 069058643..000000000 --- a/nixpkgs/private/location_expansion.bzl +++ /dev/null @@ -1,128 +0,0 @@ -load("@bazel_skylib//lib:paths.bzl", "paths") - -def parse_expand_location(string): - """Parse a string that might contain location expansion commands. - - Generates a list of pairs of command and argument. - The command can have the following values: - - `string`: argument is a string, append it to the result. - - `location`: argument is a label, append its location to the result. - - Attrs: - string: string, The string to parse. - - Returns: - (result, error): - result: The generated list of pairs of command and argument. - error: string or None, This is set if an error occurred. - """ - result = [] - offset = 0 - len_string = len(string) - - # Step through occurrences of `$`. This is bounded by the length of the string. - for _ in range(len_string): - # Find the position of the next `$`. - position = string.find("$", offset) - if position == -1: - position = len_string - - # Append the in-between literal string. - if offset < position: - result.append(("string", string[offset:position])) - - # Terminate at the end of the string. - if position == len_string: - break - - # Parse the `$` command. - if string[position:].startswith("$$"): - # Insert verbatim '$'. - result.append(("string", "$")) - offset = position + 2 - elif string[position:].startswith("$("): - # Expand a location command. - group_start = position + 2 - group_end = string.find(")", group_start) - if group_end == -1: - return (None, "Unbalanced parentheses in location expansion for '{}'.".format(string[position:])) - - group = string[group_start:group_end] - command = None - if group.startswith("location "): - label_str = group[len("location "):] - command = ("location", label_str) - else: - return (None, "Unrecognized location expansion '$({})'.".format(group)) - - result.append(command) - offset = group_end + 1 - else: - return (None, "Unescaped '$' in location expansion at position {} of input.".format(position)) - - return (result, None) - -def resolve_label(label_str, labels): - """Find the label that corresponds to the given string. - - Attr: - label_str: string, String representation of a label. - labels: dict from Label to path: Known label to path mappings. - - Returns: - (path, error): - path: path, The path to the resolved label - error: string or None, This is set if an error occurred. - """ - label_candidates = [ - (lbl, path) - for (lbl, path) in labels.items() - if lbl.relative(label_str) == lbl - ] - - if len(label_candidates) == 0: - return (None, "Unknown label '{}' in location expansion.".format(label_str)) - elif len(label_candidates) > 1: - return (None, "Ambiguous label '{}' in location expansion. Candidates: {}".format( - label_str, - ", ".join([str(lbl) for (lbl, _) in label_candidates]), - )) - - return (label_candidates[0][1], None) - -def expand_location(repository_ctx, string, labels, attr = None): - """Expand `$(location label)` to a path. - - Raises an error on unexpected occurrences of `$`. - Use `$$` to insert a verbatim `$`. - - Attrs: - repository_ctx: The repository rule context. - string: string, Replace instances of `$(location )` in this string. - labels: dict from label to path: Known label to path mappings. - attr: string, The rule attribute to use for error reporting. - - Returns: - The string with all instances of `$(location )` replaced by paths. - """ - (parsed, error) = parse_expand_location(string) - if error != None: - fail(error, attr) - - result = "" - for (command, argument) in parsed: - if command == "string": - result += argument - elif command == "location": - (label, error) = resolve_label(argument, labels) - if error != None: - fail(error, attr) - - result += paths.join(".", paths.relativize( - str(repository_ctx.path(label)), - str(repository_ctx.path(".")), - )) - else: - fail("Internal error: Unknown location expansion command '{}'.".format(command), attr) - - return result diff --git a/nixpkgs/toolchains/foreign_cc.bzl b/nixpkgs/toolchains/foreign_cc.bzl index 37b76c8d3..fd262bfb1 100644 --- a/nixpkgs/toolchains/foreign_cc.bzl +++ b/nixpkgs/toolchains/foreign_cc.bzl @@ -1,5 +1,5 @@ load("//nixpkgs:nixpkgs.bzl", "nixpkgs_package") -load("//nixpkgs:private/constraints.bzl", "ensure_constraints") +load("//core:util.bzl", "ensure_constraints") _foreign_cc_nix_build = """ load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain") diff --git a/nixpkgs/toolchains/rust.bzl b/nixpkgs/toolchains/rust.bzl index cbd7e62fe..f36a78a1b 100644 --- a/nixpkgs/toolchains/rust.bzl +++ b/nixpkgs/toolchains/rust.bzl @@ -1,5 +1,5 @@ load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") -load("//nixpkgs:private/constraints.bzl", "ensure_constraints") +load("//core:util.bzl", "ensure_constraints") # Adapted from rules_rust toolchain BUILD: # https://github.com/bazelbuild/rules_rust/blob/fd436df9e2d4ac1b234ca5e969e34a4cb5891910/rust/private/repository_utils.bzl#L17-L46 diff --git a/tests/location_expansion_unit_test.bzl b/tests/location_expansion_unit_test.bzl index a924fe57c..f51544a66 100644 --- a/tests/location_expansion_unit_test.bzl +++ b/tests/location_expansion_unit_test.bzl @@ -1,6 +1,6 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") load( - "//nixpkgs:private/location_expansion.bzl", + "//core:util.bzl", "parse_expand_location", "resolve_label", ) From b143b595a1ca3d5a8af88e625b7d3ec8d229b5f0 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Tue, 8 Mar 2022 09:01:09 +0100 Subject: [PATCH 06/25] extract go toolchain into separate module --- nixpkgs/BUILD.bazel | 1 + nixpkgs/toolchains/go.bzl | 359 +------------------------------------ toolchains/go/BUILD.bazel | 5 + toolchains/go/MODULE.bazel | 7 + toolchains/go/go.bzl | 359 +++++++++++++++++++++++++++++++++++++ 5 files changed, 376 insertions(+), 355 deletions(-) create mode 100644 toolchains/go/BUILD.bazel create mode 100644 toolchains/go/MODULE.bazel create mode 100644 toolchains/go/go.bzl diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 29d31b54f..0bb93d91c 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -59,6 +59,7 @@ bzl_library( deps = [ ":bazel_tools", ":nixpkgs", + "//toolchains/go:go.bzl", "@io_bazel_rules_go//go:deps", ], ) diff --git a/nixpkgs/toolchains/go.bzl b/nixpkgs/toolchains/go.bzl index 0384a641c..2fb1519f2 100644 --- a/nixpkgs/toolchains/go.bzl +++ b/nixpkgs/toolchains/go.bzl @@ -7,360 +7,9 @@ dependencies on rules_go for those who don't need go toolchain. `@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl`.** """ -load("//nixpkgs:nixpkgs.bzl", "nixpkgs_package") -load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") -load("@io_bazel_rules_go//go/private:platforms.bzl", "PLATFORMS") - -def _detect_host_platform(ctx): - """Copied from `rules_go`, since we have no other way to determine the proper `goarch` value. - https://github.com/bazelbuild/rules_go/blob/728a9e1874bc965b05c415d7f6b332a86ac35102/go/private/sdk.bzl#L239 - """ - if ctx.os.name == "linux": - goos, goarch = "linux", "amd64" - res = ctx.execute(["uname", "-p"]) - if res.return_code == 0: - uname = res.stdout.strip() - if uname == "s390x": - goarch = "s390x" - elif uname == "i686": - goarch = "386" - - # uname -p is not working on Aarch64 boards - # or for ppc64le on some distros - res = ctx.execute(["uname", "-m"]) - if res.return_code == 0: - uname = res.stdout.strip() - if uname == "aarch64": - goarch = "arm64" - elif uname == "armv6l": - goarch = "arm" - elif uname == "armv7l": - goarch = "arm" - elif uname == "ppc64le": - goarch = "ppc64le" - - # Default to amd64 when uname doesn't return a known value. - - elif ctx.os.name == "mac os x": - goos, goarch = "darwin", "amd64" - - res = ctx.execute(["uname", "-m"]) - if res.return_code == 0: - uname = res.stdout.strip() - if uname == "arm64": - goarch = "arm64" - - # Default to amd64 when uname doesn't return a known value. - - elif ctx.os.name.startswith("windows"): - goos, goarch = "windows", "amd64" - elif ctx.os.name == "freebsd": - goos, goarch = "freebsd", "amd64" - else: - fail("Unsupported operating system: " + ctx.os.name) - - return goos, goarch - -go_helpers_build = """ -load("@io_bazel_rules_go//go:def.bzl", "go_sdk") - -def go_sdk_for_arch(): - native.filegroup( - name = "libs", - srcs = native.glob( - ["pkg/{goos}_{goarch}/**/*.a"], - exclude = ["pkg/{goos}_{goarch}/**/cmd/**"], - ), - ) - - go_sdk( - name = "go_sdk", - goos = "{goos}", - goarch = "{goarch}", - root_file = "ROOT", - package_list = ":package_list", - libs = [":libs"], - headers = [":headers"], - srcs = [":srcs"], - tools = [":tools"], - go = "bin/go{exe}", - ) -""" - -def _nixpkgs_go_helpers_impl(repository_ctx): - repository_ctx.file("BUILD.bazel", executable = False, content = "") - goos, goarch = _detect_host_platform(repository_ctx) - content = go_helpers_build.format( - goos = goos, - goarch = goarch, - exe = ".exe" if goos == "windows" else "", - ) - repository_ctx.file("go_sdk.bzl", executable = False, content = content) - -nixpkgs_go_helpers = repository_rule( - implementation = _nixpkgs_go_helpers_impl -) - -go_toolchain_func = """ -load("@io_bazel_rules_go//go/private:platforms.bzl", "PLATFORMS") -load("@io_bazel_rules_go//go:def.bzl", "go_toolchain") - -def declare_toolchains(host_goos, host_goarch): - for p in [p for p in PLATFORMS if not p.cgo]: - link_flags = [] - cgo_link_flags = [] - if host_goos == "darwin": - cgo_link_flags.extend(["-shared", "-Wl,-all_load"]) - if host_goos == "linux": - cgo_link_flags.append("-Wl,-whole-archive") - toolchain_name = "toolchain_go_" + p.name - impl_name = toolchain_name + "-impl" - cgo_constraints = ( - "@io_bazel_rules_go//go/toolchain:cgo_off", - "@io_bazel_rules_go//go/toolchain:cgo_on", - ) - constraints = [c for c in p.constraints if c not in cgo_constraints] - go_toolchain( - name = impl_name, - goos = p.goos, - goarch = p.goarch, - sdk = "@{sdk_repo}//:go_sdk", - builder = "@{sdk_repo}//:builder", - link_flags = link_flags, - cgo_link_flags = cgo_link_flags, - visibility = ["//visibility:public"], - ) - native.toolchain( - name = toolchain_name, - toolchain_type = "@io_bazel_rules_go//go:toolchain", - exec_compatible_with = [ - "@io_bazel_rules_go//go/toolchain:" + host_goos, - "@io_bazel_rules_go//go/toolchain:" + host_goarch, - "@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix", - ], - target_compatible_with = constraints, - toolchain = ":" + impl_name, - ) -""" - -go_toolchain_build = """ -load("//:toolchain.bzl", "declare_toolchains") - -declare_toolchains("{goos}", "{goarch}") -""" - -def _nixpkgs_go_toolchain_impl(repository_ctx): - cpu = get_cpu_value(repository_ctx) - goos, goarch = _detect_host_platform(repository_ctx) - content = go_toolchain_func.format( - sdk_repo = repository_ctx.attr.sdk_repo, - ) - build_content = go_toolchain_build.format( - goos = goos, - goarch = goarch, - ) - repository_ctx.file("toolchain.bzl", executable = False, content = content) - repository_ctx.file("BUILD.bazel", executable = False, content = build_content) - -nixpkgs_go_toolchain = repository_rule( - implementation = _nixpkgs_go_toolchain_impl, - attrs = { - "sdk_repo": attr.string( - doc = "name of the nixpkgs_package repository defining the go sdk" - ), - }, - doc = """ - Set up the Go SDK - """ -) - -go_sdk_build = """ -load("@io_bazel_rules_go//go/private/rules:binary.bzl", "go_tool_binary") -load("@io_bazel_rules_go//go/private/rules:sdk.bzl", "package_list") -load("@io_bazel_rules_go//go:def.bzl", "go_sdk") -load("@{helpers}//:go_sdk.bzl", "go_sdk_for_arch") - -package(default_visibility = ["//visibility:public"]) - -go_sdk_for_arch() - -filegroup( - name = "headers", - srcs = glob(["pkg/include/*.h"]), -) - -filegroup( - name = "srcs", - srcs = glob(["src/**"]), +load( + "//toolchains/go:go.bzl", + _nixpkgs_go_configure = "nixpkgs_go_configure", ) -filegroup( - name = "tools", - srcs = glob(["pkg/tool/**", "bin/gofmt*"]) -) - -go_tool_binary( - name = "builder", - srcs = ["@io_bazel_rules_go//go/tools/builders:builder_srcs"], - sdk = ":go_sdk", -) - -package_list( - name = "package_list", - srcs = [":srcs"], - root_file = "ROOT", - out = "packages.txt", -) - -filegroup( - name = "files", - srcs = glob([ - "bin/go*", - "src/**", - "pkg/**", - ]), -) - -exports_files( - ["lib/time/zoneinfo.zip"], - visibility = ["//visibility:public"], -) -""" - -def nixpkgs_go_configure( - sdk_name = "go_sdk", - repository = None, - repositories = {}, - nix_file = None, - nix_file_deps = None, - nix_file_content = None, - nixopts = [], - fail_not_supported = True, - quiet = False, - ): - """Use go toolchain from Nixpkgs. - - By default rules_go configures the go toolchain to be downloaded as binaries (which doesn't work on NixOS), - there is a way to tell rules_go to look into environment and find local go binary which is not hermetic. - This command allows to setup a hermetic go sdk from Nixpkgs, which should be considered as best practice. - Cross toolchains are declared and registered for each entry in the `PLATFORMS` constant in `rules_go`. - - Note that the nix package must provide a full go sdk at the root of the package instead of in `$out/share/go`, - and also provide an empty normal file named `ROOT` at the root of package. - - #### Example - - ```bzl - nixpkgs_go_configure(repository = "@nixpkgs//:default.nix") - ``` - - Example (optional nix support when go is a transitive dependency): - - ```bzl - # .bazel-lib/nixos-support.bzl - def _has_nix(ctx): - return ctx.which("nix-build") != None - - def _gen_imports_impl(ctx): - ctx.file("BUILD", "") - - imports_for_nix = \""" - load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure") - - def fix_go(): - nixpkgs_go_configure(repository = "@nixpkgs") - \""" - imports_for_non_nix = \""" - def fix_go(): - # if go isn't transitive you'll need to add call to go_register_toolchains here - pass - \""" - - if _has_nix(ctx): - ctx.file("imports.bzl", imports_for_nix) - else: - ctx.file("imports.bzl", imports_for_non_nix) - - _gen_imports = repository_rule( - implementation = _gen_imports_impl, - attrs = dict(), - ) - - def gen_imports(): - _gen_imports( - name = "nixos_support", - ) - - # WORKSPACE - - // ... - http_archive(name = "io_tweag_rules_nixpkgs", ...) - // ... - local_repository( - name = "bazel_lib", - path = ".bazel-lib", - ) - load("@bazel_lib//:nixos-support.bzl", "gen_imports") - gen_imports() - load("@nixos_support//:imports.bzl", "fix_go") - fix_go() - ``` - - Args: - sdk_name: Go sdk name to pass to rules_go - nix_file: An expression for a Nix environment derivation. The environment should expose the whole go SDK (`bin`, `src`, ...) at the root of package. It also must contain a `ROOT` file in the root of pacakge. - nix_file_deps: Dependencies of `nix_file` if any. - nix_file_content: An expression for a Nix environment derivation. - repository: A repository label identifying which Nixpkgs to use. Equivalent to `repositories = { "nixpkgs": ...}`. - repositories: A dictionary mapping `NIX_PATH` entries to repository labels. - - Setting it to - ``` - repositories = { "myrepo" : "//:myrepo" } - ``` - for example would replace all instances of `` in the called nix code by the path to the target `"//:myrepo"`. See the [relevant section in the nix manual](https://nixos.org/nix/manual/#env-NIX_PATH) for more information. - - Specify one of `path` or `repositories`. - fail_not_supported: See [`nixpkgs_package`](#nixpkgs_package-fail_not_supported). - quiet: Whether to hide the output of the Nix command. - """ - - if not nix_file and not nix_file_content: - nix_file_content = """ - with import { config = {}; overlays = []; }; buildEnv { - name = "bazel-go-toolchain"; - paths = [ - go - ]; - postBuild = '' - touch $out/ROOT - ln -s $out/share/go/{api,doc,lib,misc,pkg,src} $out/ - ''; - } - """ - - helpers_repo = sdk_name + "_helpers" - nixpkgs_go_helpers( - name = helpers_repo, - ) - nixpkgs_package( - name = sdk_name, - repository = repository, - repositories = repositories, - nix_file = nix_file, - nix_file_deps = nix_file_deps, - nix_file_content = nix_file_content, - build_file_content = go_sdk_build.format( - helpers = helpers_repo, - ), - nixopts = nixopts, - fail_not_supported = fail_not_supported, - quiet = quiet, - ) - toolchains_repo = sdk_name + "_toolchains" - nixpkgs_go_toolchain( - name = toolchains_repo, - sdk_repo = sdk_name, - ) - for p in [p for p in PLATFORMS if not p.cgo]: - native.register_toolchains("@{}//:toolchain_go_{}".format(toolchains_repo, p.name)) +nixpkgs_go_configure = _nixpkgs_go_configure diff --git a/toolchains/go/BUILD.bazel b/toolchains/go/BUILD.bazel new file mode 100644 index 000000000..7f23eb1a5 --- /dev/null +++ b/toolchains/go/BUILD.bazel @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "go.bzl", +]) \ No newline at end of file diff --git a/toolchains/go/MODULE.bazel b/toolchains/go/MODULE.bazel new file mode 100644 index 000000000..d03156951 --- /dev/null +++ b/toolchains/go/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "rules_nixpkgs_go", + version = "0.8.1", +) + +bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") +bazel_dep(name = "io_bazel_rules_go", repo_name = "rules_go", version = "0.26.0") \ No newline at end of file diff --git a/toolchains/go/go.bzl b/toolchains/go/go.bzl new file mode 100644 index 000000000..234d6d3b3 --- /dev/null +++ b/toolchains/go/go.bzl @@ -0,0 +1,359 @@ +"""Rules for importing a Go toolchain from Nixpkgs. +""" + +load("//core:nixpkgs.bzl", "nixpkgs_package") +load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") +load("@io_bazel_rules_go//go/private:platforms.bzl", "PLATFORMS") + +def _detect_host_platform(ctx): + """Copied from `rules_go`, since we have no other way to determine the proper `goarch` value. + https://github.com/bazelbuild/rules_go/blob/728a9e1874bc965b05c415d7f6b332a86ac35102/go/private/sdk.bzl#L239 + """ + if ctx.os.name == "linux": + goos, goarch = "linux", "amd64" + res = ctx.execute(["uname", "-p"]) + if res.return_code == 0: + uname = res.stdout.strip() + if uname == "s390x": + goarch = "s390x" + elif uname == "i686": + goarch = "386" + + # uname -p is not working on Aarch64 boards + # or for ppc64le on some distros + res = ctx.execute(["uname", "-m"]) + if res.return_code == 0: + uname = res.stdout.strip() + if uname == "aarch64": + goarch = "arm64" + elif uname == "armv6l": + goarch = "arm" + elif uname == "armv7l": + goarch = "arm" + elif uname == "ppc64le": + goarch = "ppc64le" + + # Default to amd64 when uname doesn't return a known value. + + elif ctx.os.name == "mac os x": + goos, goarch = "darwin", "amd64" + + res = ctx.execute(["uname", "-m"]) + if res.return_code == 0: + uname = res.stdout.strip() + if uname == "arm64": + goarch = "arm64" + + # Default to amd64 when uname doesn't return a known value. + + elif ctx.os.name.startswith("windows"): + goos, goarch = "windows", "amd64" + elif ctx.os.name == "freebsd": + goos, goarch = "freebsd", "amd64" + else: + fail("Unsupported operating system: " + ctx.os.name) + + return goos, goarch + +go_helpers_build = """ +load("@io_bazel_rules_go//go:def.bzl", "go_sdk") + +def go_sdk_for_arch(): + native.filegroup( + name = "libs", + srcs = native.glob( + ["pkg/{goos}_{goarch}/**/*.a"], + exclude = ["pkg/{goos}_{goarch}/**/cmd/**"], + ), + ) + + go_sdk( + name = "go_sdk", + goos = "{goos}", + goarch = "{goarch}", + root_file = "ROOT", + package_list = ":package_list", + libs = [":libs"], + headers = [":headers"], + srcs = [":srcs"], + tools = [":tools"], + go = "bin/go{exe}", + ) +""" + +def _nixpkgs_go_helpers_impl(repository_ctx): + repository_ctx.file("BUILD.bazel", executable = False, content = "") + goos, goarch = _detect_host_platform(repository_ctx) + content = go_helpers_build.format( + goos = goos, + goarch = goarch, + exe = ".exe" if goos == "windows" else "", + ) + repository_ctx.file("go_sdk.bzl", executable = False, content = content) + +nixpkgs_go_helpers = repository_rule( + implementation = _nixpkgs_go_helpers_impl, +) + +go_toolchain_func = """ +load("@io_bazel_rules_go//go/private:platforms.bzl", "PLATFORMS") +load("@io_bazel_rules_go//go:def.bzl", "go_toolchain") + +def declare_toolchains(host_goos, host_goarch): + for p in [p for p in PLATFORMS if not p.cgo]: + link_flags = [] + cgo_link_flags = [] + if host_goos == "darwin": + cgo_link_flags.extend(["-shared", "-Wl,-all_load"]) + if host_goos == "linux": + cgo_link_flags.append("-Wl,-whole-archive") + toolchain_name = "toolchain_go_" + p.name + impl_name = toolchain_name + "-impl" + cgo_constraints = ( + "@io_bazel_rules_go//go/toolchain:cgo_off", + "@io_bazel_rules_go//go/toolchain:cgo_on", + ) + constraints = [c for c in p.constraints if c not in cgo_constraints] + go_toolchain( + name = impl_name, + goos = p.goos, + goarch = p.goarch, + sdk = "@{sdk_repo}//:go_sdk", + builder = "@{sdk_repo}//:builder", + link_flags = link_flags, + cgo_link_flags = cgo_link_flags, + visibility = ["//visibility:public"], + ) + native.toolchain( + name = toolchain_name, + toolchain_type = "@io_bazel_rules_go//go:toolchain", + exec_compatible_with = [ + "@io_bazel_rules_go//go/toolchain:" + host_goos, + "@io_bazel_rules_go//go/toolchain:" + host_goarch, + "@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix", + ], + target_compatible_with = constraints, + toolchain = ":" + impl_name, + ) +""" + +go_toolchain_build = """ +load("//:toolchain.bzl", "declare_toolchains") + +declare_toolchains("{goos}", "{goarch}") +""" + +def _nixpkgs_go_toolchain_impl(repository_ctx): + cpu = get_cpu_value(repository_ctx) + goos, goarch = _detect_host_platform(repository_ctx) + content = go_toolchain_func.format( + sdk_repo = repository_ctx.attr.sdk_repo, + ) + build_content = go_toolchain_build.format( + goos = goos, + goarch = goarch, + ) + repository_ctx.file("toolchain.bzl", executable = False, content = content) + repository_ctx.file("BUILD.bazel", executable = False, content = build_content) + +nixpkgs_go_toolchain = repository_rule( + implementation = _nixpkgs_go_toolchain_impl, + attrs = { + "sdk_repo": attr.string( + doc = "name of the nixpkgs_package repository defining the go sdk", + ), + }, + doc = """ + Set up the Go SDK + """, +) + +go_sdk_build = """ +load("@io_bazel_rules_go//go/private/rules:binary.bzl", "go_tool_binary") +load("@io_bazel_rules_go//go/private/rules:sdk.bzl", "package_list") +load("@io_bazel_rules_go//go:def.bzl", "go_sdk") +load("@{helpers}//:go_sdk.bzl", "go_sdk_for_arch") + +package(default_visibility = ["//visibility:public"]) + +go_sdk_for_arch() + +filegroup( + name = "headers", + srcs = glob(["pkg/include/*.h"]), +) + +filegroup( + name = "srcs", + srcs = glob(["src/**"]), +) + +filegroup( + name = "tools", + srcs = glob(["pkg/tool/**", "bin/gofmt*"]) +) + +go_tool_binary( + name = "builder", + srcs = ["@io_bazel_rules_go//go/tools/builders:builder_srcs"], + sdk = ":go_sdk", +) + +package_list( + name = "package_list", + srcs = [":srcs"], + root_file = "ROOT", + out = "packages.txt", +) + +filegroup( + name = "files", + srcs = glob([ + "bin/go*", + "src/**", + "pkg/**", + ]), +) + +exports_files( + ["lib/time/zoneinfo.zip"], + visibility = ["//visibility:public"], +) +""" + +def nixpkgs_go_configure( + sdk_name = "go_sdk", + repository = None, + repositories = {}, + nix_file = None, + nix_file_deps = None, + nix_file_content = None, + nixopts = [], + fail_not_supported = True, + quiet = False): + """Use go toolchain from Nixpkgs. + + By default rules_go configures the go toolchain to be downloaded as binaries (which doesn't work on NixOS), + there is a way to tell rules_go to look into environment and find local go binary which is not hermetic. + This command allows to setup a hermetic go sdk from Nixpkgs, which should be considered as best practice. + Cross toolchains are declared and registered for each entry in the `PLATFORMS` constant in `rules_go`. + + Note that the nix package must provide a full go sdk at the root of the package instead of in `$out/share/go`, + and also provide an empty normal file named `ROOT` at the root of package. + + #### Example + + ```bzl + nixpkgs_go_configure(repository = "@nixpkgs//:default.nix") + ``` + + Example (optional nix support when go is a transitive dependency): + + ```bzl + # .bazel-lib/nixos-support.bzl + def _has_nix(ctx): + return ctx.which("nix-build") != None + + def _gen_imports_impl(ctx): + ctx.file("BUILD", "") + + imports_for_nix = \""" + load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure") + + def fix_go(): + nixpkgs_go_configure(repository = "@nixpkgs") + \""" + imports_for_non_nix = \""" + def fix_go(): + # if go isn't transitive you'll need to add call to go_register_toolchains here + pass + \""" + + if _has_nix(ctx): + ctx.file("imports.bzl", imports_for_nix) + else: + ctx.file("imports.bzl", imports_for_non_nix) + + _gen_imports = repository_rule( + implementation = _gen_imports_impl, + attrs = dict(), + ) + + def gen_imports(): + _gen_imports( + name = "nixos_support", + ) + + # WORKSPACE + + // ... + http_archive(name = "io_tweag_rules_nixpkgs", ...) + // ... + local_repository( + name = "bazel_lib", + path = ".bazel-lib", + ) + load("@bazel_lib//:nixos-support.bzl", "gen_imports") + gen_imports() + load("@nixos_support//:imports.bzl", "fix_go") + fix_go() + ``` + + Args: + sdk_name: Go sdk name to pass to rules_go + nix_file: An expression for a Nix environment derivation. The environment should expose the whole go SDK (`bin`, `src`, ...) at the root of package. It also must contain a `ROOT` file in the root of pacakge. + nix_file_deps: Dependencies of `nix_file` if any. + nix_file_content: An expression for a Nix environment derivation. + repository: A repository label identifying which Nixpkgs to use. Equivalent to `repositories = { "nixpkgs": ...}`. + repositories: A dictionary mapping `NIX_PATH` entries to repository labels. + + Setting it to + ``` + repositories = { "myrepo" : "//:myrepo" } + ``` + for example would replace all instances of `` in the called nix code by the path to the target `"//:myrepo"`. See the [relevant section in the nix manual](https://nixos.org/nix/manual/#env-NIX_PATH) for more information. + + Specify one of `path` or `repositories`. + fail_not_supported: See [`nixpkgs_package`](#nixpkgs_package-fail_not_supported). + quiet: Whether to hide the output of the Nix command. + """ + + if not nix_file and not nix_file_content: + nix_file_content = """ + with import { config = {}; overlays = []; }; buildEnv { + name = "bazel-go-toolchain"; + paths = [ + go + ]; + postBuild = '' + touch $out/ROOT + ln -s $out/share/go/{api,doc,lib,misc,pkg,src} $out/ + ''; + } + """ + + helpers_repo = sdk_name + "_helpers" + nixpkgs_go_helpers( + name = helpers_repo, + ) + nixpkgs_package( + name = sdk_name, + repository = repository, + repositories = repositories, + nix_file = nix_file, + nix_file_deps = nix_file_deps, + nix_file_content = nix_file_content, + build_file_content = go_sdk_build.format( + helpers = helpers_repo, + ), + nixopts = nixopts, + fail_not_supported = fail_not_supported, + quiet = quiet, + ) + toolchains_repo = sdk_name + "_toolchains" + nixpkgs_go_toolchain( + name = toolchains_repo, + sdk_repo = sdk_name, + ) + for p in [p for p in PLATFORMS if not p.cgo]: + native.register_toolchains("@{}//:toolchain_go_{}".format(toolchains_repo, p.name)) From 607320455010d5901547a837b21b41222f36205d Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Tue, 8 Mar 2022 13:50:24 +0100 Subject: [PATCH 07/25] make `rules_nixpkgs_core` independent repository tl;dr: end users of `rules_nixpkgs` should not be affected by this change at all. the additional code is temporary and purely for migration reasons. migrating the go toolchain to `bzlmod` necessitates accessing `rules_nixpkgs_core` as an external repository (later module). therefore we have to incorporate that new repository into the `WORKSPACE` setup. to maintain compatibility with existing installations, `rules_nixpkgs_dependencies()` automatically fetches the new sub-repository from a remote archive. but local testing requires taking it from the working directory! therefore we temporary introduce a `local=` parameter which lets the caller point optionally to a directory for local sub-repositories. the option will be removed when migration to `bzlmod` is complete. it must be configurable, because the call to `local_repository()` takes its path starting from the workspace directory where `rules_nixpkgs_dependencies()` is called: - `bazel test //...` needs `local="."` - `cd examples/toolchains/{go, java, ...}; bazel run :hello` needs `local="../../.."` --- WORKSPACE | 2 +- core/WORKSPACE | 2 ++ docs/BUILD.bazel | 4 ++-- examples/toolchains/cc/WORKSPACE | 2 +- examples/toolchains/cc_with_deps/WORKSPACE | 3 ++- examples/toolchains/go/WORKSPACE | 3 ++- examples/toolchains/java/WORKSPACE | 3 ++- examples/toolchains/python/WORKSPACE | 3 ++- examples/toolchains/rust/WORKSPACE | 4 ++-- nixpkgs/BUILD.bazel | 3 +-- nixpkgs/nixpkgs.bzl | 4 ++-- nixpkgs/repositories.bzl | 24 ++++++++++++++++++++-- nixpkgs/toolchains/rust.bzl | 2 +- tests/location_expansion_unit_test.bzl | 2 +- toolchains/go/go.bzl | 2 +- 15 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 core/WORKSPACE diff --git a/WORKSPACE b/WORKSPACE index da8457af3..c5b9c387a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,7 +3,7 @@ workspace(name = "io_tweag_rules_nixpkgs") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +rules_nixpkgs_dependencies(local=".") load( "//nixpkgs:nixpkgs.bzl", diff --git a/core/WORKSPACE b/core/WORKSPACE new file mode 100644 index 000000000..a69f72f2c --- /dev/null +++ b/core/WORKSPACE @@ -0,0 +1,2 @@ +# only temporary for compatibility with `WORKSPACE` setup +# TODO: remove when migration to `bzlmod` is complete \ No newline at end of file diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 995b32013..2de42ef43 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -20,13 +20,13 @@ stardoc( stardoc( name = "core", out = "core.md", - input = "//core:nixpkgs.bzl", + input = "@rules_nixpkgs_core//:nixpkgs.bzl", symbol_names = [ "nixpkgs_git_repository", "nixpkgs_local_repository", "nixpkgs_package", ], - deps = ["//core"], + deps = ["@rules_nixpkgs_core//:core"], ) stardoc( diff --git a/examples/toolchains/cc/WORKSPACE b/examples/toolchains/cc/WORKSPACE index 2e3d750f3..dd67de6d7 100644 --- a/examples/toolchains/cc/WORKSPACE +++ b/examples/toolchains/cc/WORKSPACE @@ -16,7 +16,7 @@ http_archive( load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +rules_nixpkgs_dependencies(local="../../..") load( "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", diff --git a/examples/toolchains/cc_with_deps/WORKSPACE b/examples/toolchains/cc_with_deps/WORKSPACE index 7f9588dea..d5ec27954 100644 --- a/examples/toolchains/cc_with_deps/WORKSPACE +++ b/examples/toolchains/cc_with_deps/WORKSPACE @@ -6,7 +6,8 @@ local_repository( load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +# Remove `local=...` +rules_nixpkgs_dependencies(local="../../..") load( "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", diff --git a/examples/toolchains/go/WORKSPACE b/examples/toolchains/go/WORKSPACE index 4a7b625b7..ebeb7ef77 100644 --- a/examples/toolchains/go/WORKSPACE +++ b/examples/toolchains/go/WORKSPACE @@ -21,7 +21,8 @@ go_rules_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +# Remove `local=...` +rules_nixpkgs_dependencies(local="../../..") load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/examples/toolchains/java/WORKSPACE b/examples/toolchains/java/WORKSPACE index 7c0d30f10..e30a9720d 100644 --- a/examples/toolchains/java/WORKSPACE +++ b/examples/toolchains/java/WORKSPACE @@ -19,7 +19,8 @@ rules_java_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +# Remove `local=...` +rules_nixpkgs_dependencies(local="../../..") load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/examples/toolchains/python/WORKSPACE b/examples/toolchains/python/WORKSPACE index 10e6b4df5..03731b7b8 100644 --- a/examples/toolchains/python/WORKSPACE +++ b/examples/toolchains/python/WORKSPACE @@ -7,7 +7,8 @@ local_repository( load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +# Remove `local=...` +rules_nixpkgs_dependencies(local="../../..") load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/examples/toolchains/rust/WORKSPACE b/examples/toolchains/rust/WORKSPACE index 2e5084828..c42f95617 100644 --- a/examples/toolchains/rust/WORKSPACE +++ b/examples/toolchains/rust/WORKSPACE @@ -1,4 +1,3 @@ -# Replace with http_archive: https://github.com/tweag/rules_nixpkgs/#setup local_repository( name = "io_tweag_rules_nixpkgs", path = "../../../", @@ -21,7 +20,8 @@ http_archive( load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies() +# Remove `local=...` +rules_nixpkgs_dependencies(local="../../..") load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 0bb93d91c..6d6eadbcd 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -40,8 +40,7 @@ bzl_library( ], visibility = ["//visibility:public"], deps = [ - "//core:nixpkgs.bzl", - "//core:util.bzl", + "@rules_nixpkgs_core//:core", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 655777aa6..46e2bd788 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -11,13 +11,13 @@ load( ) load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load( - "//core:nixpkgs.bzl", + "@rules_nixpkgs_core//:nixpkgs.bzl", _nixpkgs_git_repository = "nixpkgs_git_repository", _nixpkgs_local_repository = "nixpkgs_local_repository", _nixpkgs_package = "nixpkgs_package", ) load( - "//core:util.bzl", + "@rules_nixpkgs_core//:util.bzl", "ensure_constraints", "execute_or_fail", "find_children", diff --git a/nixpkgs/repositories.bzl b/nixpkgs/repositories.bzl index 25dcba029..6df21c54d 100644 --- a/nixpkgs/repositories.bzl +++ b/nixpkgs/repositories.bzl @@ -1,8 +1,14 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -def rules_nixpkgs_dependencies(): - """Load repositories required by rules_nixpkgs.""" +def rules_nixpkgs_dependencies(local = None): + """Load repositories required by rules_nixpkgs. + + Args: + local: path to local `rules_nixpkgs` repository. + use for testing and CI. + TODO: remove when migration to `bzlmod` is complete. + """ maybe( http_archive, "platforms", @@ -27,3 +33,17 @@ def rules_nixpkgs_dependencies(): url = "https://github.com/bazelbuild/rules_java/releases/download/4.0.0/rules_java-4.0.0.tar.gz", sha256 = "34b41ec683e67253043ab1a3d1e8b7c61e4e8edefbcad485381328c934d072fe", ) + if not local: + maybe( + http_archive, + "rules_nixpkgs_core", + # XXX: no way to use `sha256` here, but if this surrounding repo comes + # from that URL, Bazel should hit the cache for the sub-workspace + url = "https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.8.1.tar.gz", + strip_prefix = "core", + ) + else: + native.local_repository( + name = "rules_nixpkgs_core", + path = local + "/core", + ) diff --git a/nixpkgs/toolchains/rust.bzl b/nixpkgs/toolchains/rust.bzl index f36a78a1b..b1f6ea4e8 100644 --- a/nixpkgs/toolchains/rust.bzl +++ b/nixpkgs/toolchains/rust.bzl @@ -1,5 +1,5 @@ load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") -load("//core:util.bzl", "ensure_constraints") +load("@rules_nixpkgs_core//:util.bzl", "ensure_constraints") # Adapted from rules_rust toolchain BUILD: # https://github.com/bazelbuild/rules_rust/blob/fd436df9e2d4ac1b234ca5e969e34a4cb5891910/rust/private/repository_utils.bzl#L17-L46 diff --git a/tests/location_expansion_unit_test.bzl b/tests/location_expansion_unit_test.bzl index f51544a66..1fe8d1f9d 100644 --- a/tests/location_expansion_unit_test.bzl +++ b/tests/location_expansion_unit_test.bzl @@ -1,6 +1,6 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") load( - "//core:util.bzl", + "@rules_nixpkgs_core//:util.bzl", "parse_expand_location", "resolve_label", ) diff --git a/toolchains/go/go.bzl b/toolchains/go/go.bzl index 234d6d3b3..adaea01dd 100644 --- a/toolchains/go/go.bzl +++ b/toolchains/go/go.bzl @@ -1,7 +1,7 @@ """Rules for importing a Go toolchain from Nixpkgs. """ -load("//core:nixpkgs.bzl", "nixpkgs_package") +load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") load("@io_bazel_rules_go//go/private:platforms.bzl", "PLATFORMS") From 2f6b91f6d0bf306b3ceb315fbe396dcb42f701d1 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Wed, 9 Mar 2022 09:10:21 +0100 Subject: [PATCH 08/25] extract python toolchain into module --- nixpkgs/BUILD.bazel | 1 + nixpkgs/nixpkgs.bzl | 166 +------------------------------ toolchains/python/BUILD.bazel | 5 + toolchains/python/MODULE.bazel | 6 ++ toolchains/python/python.bzl | 173 +++++++++++++++++++++++++++++++++ 5 files changed, 190 insertions(+), 161 deletions(-) create mode 100644 toolchains/python/BUILD.bazel create mode 100644 toolchains/python/MODULE.bazel create mode 100644 toolchains/python/python.bzl diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 6d6eadbcd..46bff4a0a 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -41,6 +41,7 @@ bzl_library( visibility = ["//visibility:public"], deps = [ "@rules_nixpkgs_core//:core", + "//toolchains/python:python.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 46e2bd788..ae0ec7b38 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -16,19 +16,23 @@ load( _nixpkgs_local_repository = "nixpkgs_local_repository", _nixpkgs_package = "nixpkgs_package", ) +load( + "//toolchains/python:python.bzl", + _nixpkgs_python_configure = "nixpkgs_python_configure", +) load( "@rules_nixpkgs_core//:util.bzl", "ensure_constraints", "execute_or_fail", "find_children", "is_supported_platform", - "label_string", ) # aliases for backwards compatibility prior to `bzlmod` nixpkgs_git_repository = _nixpkgs_git_repository nixpkgs_local_repository = _nixpkgs_local_repository nixpkgs_package = _nixpkgs_package +nixpkgs_python_configure = _nixpkgs_python_configure def _parse_cc_toolchain_info(content, filename): """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. @@ -769,166 +773,6 @@ def nixpkgs_java_configure( ) native.register_toolchains("@{}_toolchain//:all".format(name)) -def _nixpkgs_python_toolchain_impl(repository_ctx): - exec_constraints, target_constraints = ensure_constraints(repository_ctx) - - repository_ctx.file("BUILD.bazel", executable = False, content = """ -load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") -py_runtime_pair( - name = "py_runtime_pair", - py2_runtime = {python2_runtime}, - py3_runtime = {python3_runtime}, -) -toolchain( - name = "toolchain", - toolchain = ":py_runtime_pair", - toolchain_type = "@bazel_tools//tools/python:toolchain_type", - exec_compatible_with = {exec_constraints}, - target_compatible_with = {target_constraints}, -) -""".format( - python2_runtime = label_string(repository_ctx.attr.python2_runtime), - python3_runtime = label_string(repository_ctx.attr.python3_runtime), - exec_constraints = exec_constraints, - target_constraints = target_constraints, - )) - -_nixpkgs_python_toolchain = repository_rule( - _nixpkgs_python_toolchain_impl, - attrs = { - # Using attr.string instead of attr.label, so that the repository rule - # does not explicitly depend on the nixpkgs_package instances. This is - # necessary, so that builds don't fail on platforms without nixpkgs. - "python2_runtime": attr.string(), - "python3_runtime": attr.string(), - "exec_constraints": attr.string_list(), - "target_constraints": attr.string_list(), - }, -) - -def _python_nix_file_content(attribute_path, bin_path, version): - add_shebang = versions.is_at_least("4.2.0", versions.get()) - - return """ -with import {{ config = {{}}; overlays = []; }}; -let - addShebang = {add_shebang}; - interpreterPath = "${{{attribute_path}}}/{bin_path}"; - shebangLine = interpreter: writers.makeScriptWriter {{ inherit interpreter; }} "shebang" ""; -in -runCommand "bazel-nixpkgs-python-toolchain" - {{ executable = false; - # Pointless to do this on a remote machine. - preferLocalBuild = true; - allowSubstitutes = false; - }} - '' - n=$out/BUILD.bazel - mkdir -p "$(dirname "$n")" - - cat >>$n < {{ config = {{}}; overlays = []; }}; +let + addShebang = {add_shebang}; + interpreterPath = "${{{attribute_path}}}/{bin_path}"; + shebangLine = interpreter: writers.makeScriptWriter {{ inherit interpreter; }} "shebang" ""; +in +runCommand "bazel-nixpkgs-python-toolchain" + {{ executable = false; + # Pointless to do this on a remote machine. + preferLocalBuild = true; + allowSubstitutes = false; + }} + '' + n=$out/BUILD.bazel + mkdir -p "$(dirname "$n")" + + cat >>$n < Date: Wed, 9 Mar 2022 09:33:25 +0100 Subject: [PATCH 09/25] extract java toolchain into module no alias for `default_java_toolchain`, since that seems to be used only internally, and a copy of what is usually distributed with Bazel. --- nixpkgs/BUILD.bazel | 1 + nixpkgs/nixpkgs.bzl | 220 +---------------- toolchains/java/BUILD.bazel | 5 + toolchains/java/MODULE.bazel | 6 + .../java/default_java_toolchain.bzl | 0 toolchains/java/java.bzl | 230 ++++++++++++++++++ .../java/local_java_repository.bzl | 2 +- 7 files changed, 248 insertions(+), 216 deletions(-) create mode 100644 toolchains/java/BUILD.bazel create mode 100644 toolchains/java/MODULE.bazel rename {nixpkgs/toolchains => toolchains}/java/default_java_toolchain.bzl (100%) create mode 100644 toolchains/java/java.bzl rename {nixpkgs/toolchains => toolchains}/java/local_java_repository.bzl (98%) diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 46bff4a0a..65f1539a2 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -42,6 +42,7 @@ bzl_library( deps = [ "@rules_nixpkgs_core//:core", "//toolchains/python:python.bzl", + "//toolchains/java:java.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index ae0ec7b38..1dab37804 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -20,6 +20,10 @@ load( "//toolchains/python:python.bzl", _nixpkgs_python_configure = "nixpkgs_python_configure", ) +load( + "//toolchains/java:java.bzl", + _nixpkgs_java_configure = "nixpkgs_java_configure", +) load( "@rules_nixpkgs_core//:util.bzl", "ensure_constraints", @@ -33,6 +37,7 @@ nixpkgs_git_repository = _nixpkgs_git_repository nixpkgs_local_repository = _nixpkgs_local_repository nixpkgs_package = _nixpkgs_package nixpkgs_python_configure = _nixpkgs_python_configure +nixpkgs_java_configure = _nixpkgs_java_configure def _parse_cc_toolchain_info(content, filename): """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. @@ -558,221 +563,6 @@ def nixpkgs_cc_configure_deprecated( native.bind(name = "cc_toolchain", actual = "@local_config_cc//:toolchain") native.register_toolchains("@local_config_cc//:all") -_java_nix_file_content = """\ -with import { config = {}; overlays = []; }; - -{ attrPath -, attrSet -, filePath -}: - -let - javaHome = - if attrSet == null then - pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." attrPath) pkgs - else - pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." attrPath) attrSet - ; - javaHomePath = - if filePath == "" then - "${javaHome}" - else - "${javaHome}/${filePath}" - ; -in - -pkgs.runCommand "bazel-nixpkgs-java-runtime" - { executable = false; - # Pointless to do this on a remote machine. - preferLocalBuild = true; - allowSubstitutes = false; - } - '' - n=$out/BUILD.bazel - mkdir -p "$(dirname "$n")" - - cat >>$n <= 5.0 - Bazel can use this instance to run JVM binaries and tests, refer to the - [Bazel documentation](https://docs.bazel.build/versions/4.0.0/bazel-and-java.html#configuring-the-jdk) for details. - - #### Example - - ##### Bazel 4 - - Add the following to your `WORKSPACE` file to import a JDK from nixpkgs: - ```bzl - load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_java_configure") - nixpkgs_java_configure( - attribute_path = "jdk11.home", - repository = "@nixpkgs", - ) - ``` - - Add the following configuration to `.bazelrc` to enable this Java runtime: - ``` - build --javabase=@nixpkgs_java_runtime//:runtime - build --host_javabase=@nixpkgs_java_runtime//:runtime - # Adjust this to match the Java version provided by this runtime. - # See `bazel query 'kind(java_toolchain, @bazel_tools//tools/jdk:all)'` for available options. - build --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 - build --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 - ``` - - ##### Bazel 5 - - Add the following to your `WORKSPACE` file to import a JDK from nixpkgs: - ```bzl - load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_java_configure") - nixpkgs_java_configure( - attribute_path = "jdk11.home", - repository = "@nixpkgs", - toolchain = True, - toolchain_name = "nixpkgs_java", - toolchain_version = "11", - ) - ``` - - Add the following configuration to `.bazelrc` to enable this Java runtime: - ``` - build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host - build --java_runtime_version=nixpkgs_java - build --tool_java_runtime_version=nixpkgs_java - ``` - - Args: - name: The name-prefix for the created external repositories. - attribute_path: string, The nixpkgs attribute path for `jdk.home`. - java_home_path: optional, string, The path to `JAVA_HOME` within the package. - repository: See [`nixpkgs_package`](#nixpkgs_package-repository). - repositories: See [`nixpkgs_package`](#nixpkgs_package-repositories). - nix_file: optional, Label, Obtain the runtime from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. - nix_file_content: optional, string, Obtain the runtime from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. - nix_file_deps: See [`nixpkgs_package`](#nixpkgs_package-nix_file_deps). - nixopts: See [`nixpkgs_package`](#nixpkgs_package-nixopts). - fail_not_supported: See [`nixpkgs_package`](#nixpkgs_package-fail_not_supported). - quiet: See [`nixpkgs_package`](#nixpkgs_package-quiet). - toolchain: Create & register a Bazel toolchain based on the Java runtime. - toolchain_name: The name of the toolchain that can be used in --java_runtime_version. - toolchain_version: The version of the toolchain that can be used in --java_runtime_version. - exec_constraints: Constraints for the execution platform. - target_constraints: Constraints for the target platform. - """ - if attribute_path == None: - fail("'attribute_path' is required.", "attribute_path") - - nix_expr = None - if nix_file and nix_file_content: - fail("Cannot specify both 'nix_file' and 'nix_file_content'.") - elif nix_file: - nix_expr = "import $(location {}) {{}}".format(nix_file) - nix_file_deps = depset(direct = [nix_file] + nix_file_deps).to_list() - elif nix_file_content: - nix_expr = nix_file_content - else: - nix_expr = "null" - - nixopts = list(nixopts) - nixopts.extend([ - "--argstr", - "attrPath", - attribute_path, - "--arg", - "attrSet", - nix_expr, - "--argstr", - "filePath", - java_home_path, - ]) - - nixpkgs_package( - name = name, - nix_file_content = _java_nix_file_content, - repository = repository, - repositories = repositories, - nix_file_deps = nix_file_deps, - nixopts = nixopts, - fail_not_supported = fail_not_supported, - quiet = quiet, - ) - if toolchain: - _nixpkgs_java_toolchain( - name = "{}_toolchain".format(name), - runtime_repo = name, - runtime_version = toolchain_version, - runtime_name = toolchain_name, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ) - native.register_toolchains("@{}_toolchain//:all".format(name)) - def nixpkgs_sh_posix_config(name, packages, **kwargs): nixpkgs_package( name = name, diff --git a/toolchains/java/BUILD.bazel b/toolchains/java/BUILD.bazel new file mode 100644 index 000000000..d7e2b0b13 --- /dev/null +++ b/toolchains/java/BUILD.bazel @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "java.bzl", +]) \ No newline at end of file diff --git a/toolchains/java/MODULE.bazel b/toolchains/java/MODULE.bazel new file mode 100644 index 000000000..784c3eb40 --- /dev/null +++ b/toolchains/java/MODULE.bazel @@ -0,0 +1,6 @@ +module( + name = "rules_nixpkgs_java", + version = "0.8.1", +) + +bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") diff --git a/nixpkgs/toolchains/java/default_java_toolchain.bzl b/toolchains/java/default_java_toolchain.bzl similarity index 100% rename from nixpkgs/toolchains/java/default_java_toolchain.bzl rename to toolchains/java/default_java_toolchain.bzl diff --git a/toolchains/java/java.bzl b/toolchains/java/java.bzl new file mode 100644 index 000000000..731a08cc8 --- /dev/null +++ b/toolchains/java/java.bzl @@ -0,0 +1,230 @@ +"""Rules for importing a Java toolchain from Nixpkgs. +""" + +load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "get_cpu_value", +) +load( + "@rules_nixpkgs_core//:nixpkgs.bzl", + "nixpkgs_package", +) +load( + "@rules_nixpkgs_core//:util.bzl", + "ensure_constraints", +) + +_java_nix_file_content = """\ +with import { config = {}; overlays = []; }; + +{ attrPath +, attrSet +, filePath +}: + +let + javaHome = + if attrSet == null then + pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." attrPath) pkgs + else + pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." attrPath) attrSet + ; + javaHomePath = + if filePath == "" then + "${javaHome}" + else + "${javaHome}/${filePath}" + ; +in + +pkgs.runCommand "bazel-nixpkgs-java-runtime" + { executable = false; + # Pointless to do this on a remote machine. + preferLocalBuild = true; + allowSubstitutes = false; + } + '' + n=$out/BUILD.bazel + mkdir -p "$(dirname "$n")" + + cat >>$n <= 5.0 + Bazel can use this instance to run JVM binaries and tests, refer to the + [Bazel documentation](https://docs.bazel.build/versions/4.0.0/bazel-and-java.html#configuring-the-jdk) for details. + + #### Example + + ##### Bazel 4 + + Add the following to your `WORKSPACE` file to import a JDK from nixpkgs: + ```bzl + load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_java_configure") + nixpkgs_java_configure( + attribute_path = "jdk11.home", + repository = "@nixpkgs", + ) + ``` + + Add the following configuration to `.bazelrc` to enable this Java runtime: + ``` + build --javabase=@nixpkgs_java_runtime//:runtime + build --host_javabase=@nixpkgs_java_runtime//:runtime + # Adjust this to match the Java version provided by this runtime. + # See `bazel query 'kind(java_toolchain, @bazel_tools//tools/jdk:all)'` for available options. + build --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 + build --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 + ``` + + ##### Bazel 5 + + Add the following to your `WORKSPACE` file to import a JDK from nixpkgs: + ```bzl + load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_java_configure") + nixpkgs_java_configure( + attribute_path = "jdk11.home", + repository = "@nixpkgs", + toolchain = True, + toolchain_name = "nixpkgs_java", + toolchain_version = "11", + ) + ``` + + Add the following configuration to `.bazelrc` to enable this Java runtime: + ``` + build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host + build --java_runtime_version=nixpkgs_java + build --tool_java_runtime_version=nixpkgs_java + ``` + + Args: + name: The name-prefix for the created external repositories. + attribute_path: string, The nixpkgs attribute path for `jdk.home`. + java_home_path: optional, string, The path to `JAVA_HOME` within the package. + repository: See [`nixpkgs_package`](#nixpkgs_package-repository). + repositories: See [`nixpkgs_package`](#nixpkgs_package-repositories). + nix_file: optional, Label, Obtain the runtime from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. + nix_file_content: optional, string, Obtain the runtime from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. + nix_file_deps: See [`nixpkgs_package`](#nixpkgs_package-nix_file_deps). + nixopts: See [`nixpkgs_package`](#nixpkgs_package-nixopts). + fail_not_supported: See [`nixpkgs_package`](#nixpkgs_package-fail_not_supported). + quiet: See [`nixpkgs_package`](#nixpkgs_package-quiet). + toolchain: Create & register a Bazel toolchain based on the Java runtime. + toolchain_name: The name of the toolchain that can be used in --java_runtime_version. + toolchain_version: The version of the toolchain that can be used in --java_runtime_version. + exec_constraints: Constraints for the execution platform. + target_constraints: Constraints for the target platform. + """ + if attribute_path == None: + fail("'attribute_path' is required.", "attribute_path") + + nix_expr = None + if nix_file and nix_file_content: + fail("Cannot specify both 'nix_file' and 'nix_file_content'.") + elif nix_file: + nix_expr = "import $(location {}) {{}}".format(nix_file) + nix_file_deps = depset(direct = [nix_file] + nix_file_deps).to_list() + elif nix_file_content: + nix_expr = nix_file_content + else: + nix_expr = "null" + + nixopts = list(nixopts) + nixopts.extend([ + "--argstr", + "attrPath", + attribute_path, + "--arg", + "attrSet", + nix_expr, + "--argstr", + "filePath", + java_home_path, + ]) + + nixpkgs_package( + name = name, + nix_file_content = _java_nix_file_content, + repository = repository, + repositories = repositories, + nix_file_deps = nix_file_deps, + nixopts = nixopts, + fail_not_supported = fail_not_supported, + quiet = quiet, + ) + if toolchain: + _nixpkgs_java_toolchain( + name = "{}_toolchain".format(name), + runtime_repo = name, + runtime_version = toolchain_version, + runtime_name = toolchain_name, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ) + native.register_toolchains("@{}_toolchain//:all".format(name)) diff --git a/nixpkgs/toolchains/java/local_java_repository.bzl b/toolchains/java/local_java_repository.bzl similarity index 98% rename from nixpkgs/toolchains/java/local_java_repository.bzl rename to toolchains/java/local_java_repository.bzl index a035d928c..53c19775c 100644 --- a/nixpkgs/toolchains/java/local_java_repository.bzl +++ b/toolchains/java/local_java_repository.bzl @@ -14,7 +14,7 @@ """Rules for importing and registering a local JDK.""" -load(":toolchains/java/default_java_toolchain.bzl", "JVM8_TOOLCHAIN_CONFIGURATION", "default_java_toolchain") +load("//toolchains/java:default_java_toolchain.bzl", "JVM8_TOOLCHAIN_CONFIGURATION", "default_java_toolchain") def _detect_java_version(repository_ctx, java_bin): properties_out = repository_ctx.execute([java_bin, "-XshowSettings:properties"]).stderr From b08b196e121b27f9382c0be000194a0245e91fd0 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Wed, 9 Mar 2022 10:05:17 +0100 Subject: [PATCH 10/25] extract CC toolchain into module leave aliases to existing files to ensure backwards compatibility. put CC toolchain in separate workspace, which is required because `nixpkgs_cc_configure()` references a file `cc.nix` from its own repository, which will be external at the call site. --- nixpkgs/BUILD.bazel | 3 +- nixpkgs/nixpkgs.bzl | 406 +------------------ nixpkgs/repositories.bzl | 17 +- nixpkgs/toolchains/foreign_cc.bzl | 116 +----- tests/BUILD.bazel | 1 + toolchains/cc/BUILD.bazel | 7 + toolchains/cc/MODULE.bazel | 6 + toolchains/cc/WORKSPACE | 2 + toolchains/cc/cc.bzl | 398 ++++++++++++++++++ {nixpkgs/toolchains => toolchains/cc}/cc.nix | 0 toolchains/cc/foreign_cc.bzl | 114 ++++++ 11 files changed, 558 insertions(+), 512 deletions(-) create mode 100644 toolchains/cc/BUILD.bazel create mode 100644 toolchains/cc/MODULE.bazel create mode 100644 toolchains/cc/WORKSPACE create mode 100644 toolchains/cc/cc.bzl rename {nixpkgs/toolchains => toolchains/cc}/cc.nix (100%) create mode 100644 toolchains/cc/foreign_cc.bzl diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 65f1539a2..220058288 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -41,8 +41,9 @@ bzl_library( visibility = ["//visibility:public"], deps = [ "@rules_nixpkgs_core//:core", - "//toolchains/python:python.bzl", + "@rules_nixpkgs_cc//:cc.bzl", "//toolchains/java:java.bzl", + "//toolchains/python:python.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 1dab37804..ad87b7b06 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -1,21 +1,22 @@ """Rules for importing Nixpkgs packages.""" -load("@bazel_skylib//lib:sets.bzl", "sets") -load("@bazel_skylib//lib:versions.bzl", "versions") -load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl") load( "@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value", - "get_starlark_list", - "write_builtin_include_directory_paths", ) -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load( "@rules_nixpkgs_core//:nixpkgs.bzl", _nixpkgs_git_repository = "nixpkgs_git_repository", _nixpkgs_local_repository = "nixpkgs_local_repository", _nixpkgs_package = "nixpkgs_package", ) +load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl") +load( + "@rules_nixpkgs_core//:util.bzl", + "execute_or_fail", + "find_children", + "is_supported_platform", +) load( "//toolchains/python:python.bzl", _nixpkgs_python_configure = "nixpkgs_python_configure", @@ -25,11 +26,8 @@ load( _nixpkgs_java_configure = "nixpkgs_java_configure", ) load( - "@rules_nixpkgs_core//:util.bzl", - "ensure_constraints", - "execute_or_fail", - "find_children", - "is_supported_platform", + "@rules_nixpkgs_cc//:cc.bzl", + _nixpkgs_cc_configure = "nixpkgs_cc_configure", ) # aliases for backwards compatibility prior to `bzlmod` @@ -38,389 +36,7 @@ nixpkgs_local_repository = _nixpkgs_local_repository nixpkgs_package = _nixpkgs_package nixpkgs_python_configure = _nixpkgs_python_configure nixpkgs_java_configure = _nixpkgs_java_configure - -def _parse_cc_toolchain_info(content, filename): - """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. - - Attrs: - content: string, The content of the `CC_TOOLCHAIN_INFO` file. - filename: string, The path to the `CC_TOOLCHAIN_INFO` file, used for error reporting. - - Returns: - struct, The substitutions for `@bazel_tools//tools/cpp:BUILD.tpl`. - """ - - # Parse the content of CC_TOOLCHAIN_INFO. - # - # Each line has the form - # - # :::... - info = {} - for line in content.splitlines(): - fields = line.split(":") - if len(fields) == 0: - fail( - "Malformed CC_TOOLCHAIN_INFO '{}': Empty line encountered.".format(filename), - "cc_toolchain_info", - ) - info[fields[0]] = fields[1:] - - # Validate the keys in CC_TOOLCHAIN_INFO. - expected_keys = sets.make([ - "TOOL_NAMES", - "TOOL_PATHS", - "CXX_BUILTIN_INCLUDE_DIRECTORIES", - "COMPILE_FLAGS", - "CXX_FLAGS", - "LINK_FLAGS", - "LINK_LIBS", - "OPT_COMPILE_FLAGS", - "OPT_LINK_FLAGS", - "UNFILTERED_COMPILE_FLAGS", - "DBG_COMPILE_FLAGS", - "COVERAGE_COMPILE_FLAGS", - "COVERAGE_LINK_FLAGS", - "SUPPORTS_START_END_LIB", - "IS_CLANG", - ]) - actual_keys = sets.make(info.keys()) - missing_keys = sets.difference(expected_keys, actual_keys) - unexpected_keys = sets.difference(actual_keys, expected_keys) - if sets.length(missing_keys) > 0: - fail( - "Malformed CC_TOOLCHAIN_INFO '{}': Missing entries '{}'.".format( - filename, - "', '".join(sets.to_list(missing_keys)), - ), - "cc_toolchain_info", - ) - if sets.length(unexpected_keys) > 0: - fail( - "Malformed CC_TOOLCHAIN_INFO '{}': Unexpected entries '{}'.".format( - filename, - "', '".join(sets.to_list(unexpected_keys)), - ), - "cc_toolchain_info", - ) - - return struct( - tool_paths = { - tool: path - for (tool, path) in zip(info["TOOL_NAMES"], info["TOOL_PATHS"]) - }, - cxx_builtin_include_directories = info["CXX_BUILTIN_INCLUDE_DIRECTORIES"], - compile_flags = info["COMPILE_FLAGS"], - cxx_flags = info["CXX_FLAGS"], - link_flags = info["LINK_FLAGS"], - link_libs = info["LINK_LIBS"], - opt_compile_flags = info["OPT_COMPILE_FLAGS"], - opt_link_flags = info["OPT_LINK_FLAGS"], - unfiltered_compile_flags = info["UNFILTERED_COMPILE_FLAGS"], - dbg_compile_flags = info["DBG_COMPILE_FLAGS"], - coverage_compile_flags = info["COVERAGE_COMPILE_FLAGS"], - coverage_link_flags = info["COVERAGE_LINK_FLAGS"], - supports_start_end_lib = info["SUPPORTS_START_END_LIB"] == ["True"], - is_clang = info["IS_CLANG"] == ["True"], - ) - -def _nixpkgs_cc_toolchain_config_impl(repository_ctx): - cpu_value = get_cpu_value(repository_ctx) - darwin = cpu_value == "darwin" - - cc_toolchain_info_file = repository_ctx.path(repository_ctx.attr.cc_toolchain_info) - if not cc_toolchain_info_file.exists and not repository_ctx.attr.fail_not_supported: - return - info = _parse_cc_toolchain_info( - repository_ctx.read(cc_toolchain_info_file), - cc_toolchain_info_file, - ) - - # Generate the cc_toolchain workspace following the example from - # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. - # Uses the corresponding templates from `@bazel_tools` as well, see the - # private attributes of the `_nixpkgs_cc_toolchain_config` rule. - repository_ctx.symlink( - repository_ctx.path(repository_ctx.attr._unix_cc_toolchain_config), - "cc_toolchain_config.bzl", - ) - repository_ctx.symlink( - repository_ctx.path(repository_ctx.attr._armeabi_cc_toolchain_config), - "armeabi_cc_toolchain_config.bzl", - ) - - # A module map is required for clang starting from Bazel version 3.3.0. - # https://github.com/bazelbuild/bazel/commit/8b9f74649512ee17ac52815468bf3d7e5e71c9fa - needs_module_map = info.is_clang and versions.is_at_least("3.3.0", versions.get()) - if needs_module_map: - generate_system_module_map = [ - repository_ctx.path(repository_ctx.attr._generate_system_module_map), - ] - repository_ctx.file( - "module.modulemap", - execute_or_fail( - repository_ctx, - generate_system_module_map + info.cxx_builtin_include_directories, - "Failed to generate system module map.", - ).stdout.strip(), - executable = False, - ) - cc_wrapper_src = ( - repository_ctx.attr._osx_cc_wrapper if darwin else repository_ctx.attr._linux_cc_wrapper - ) - repository_ctx.template( - "cc_wrapper.sh", - repository_ctx.path(cc_wrapper_src), - { - "%{cc}": info.tool_paths["gcc"], - "%{env}": "", - }, - ) - if darwin: - info.tool_paths["gcc"] = "cc_wrapper.sh" - info.tool_paths["ar"] = "/usr/bin/libtool" - write_builtin_include_directory_paths( - repository_ctx, - info.tool_paths["gcc"], - info.cxx_builtin_include_directories, - ) - repository_ctx.template( - "BUILD.bazel", - repository_ctx.path(repository_ctx.attr._build), - { - "%{cc_toolchain_identifier}": "local", - "%{name}": cpu_value, - "%{modulemap}": ("\":module.modulemap\"" if needs_module_map else "None"), - "%{supports_param_files}": "0" if darwin else "1", - "%{cc_compiler_deps}": get_starlark_list( - [":builtin_include_directory_paths"] + ( - [":cc_wrapper"] if darwin else [] - ), - ), - "%{compiler}": "compiler", - "%{abi_version}": "local", - "%{abi_libc_version}": "local", - "%{host_system_name}": "local", - "%{target_libc}": "macosx" if darwin else "local", - "%{target_cpu}": cpu_value, - "%{target_system_name}": "local", - "%{tool_paths}": ",\n ".join( - ['"%s": "%s"' % (k, v) for (k, v) in info.tool_paths.items()], - ), - "%{cxx_builtin_include_directories}": get_starlark_list(info.cxx_builtin_include_directories), - "%{compile_flags}": get_starlark_list(info.compile_flags), - "%{cxx_flags}": get_starlark_list(info.cxx_flags), - "%{link_flags}": get_starlark_list(info.link_flags), - "%{link_libs}": get_starlark_list(info.link_libs), - "%{opt_compile_flags}": get_starlark_list(info.opt_compile_flags), - "%{opt_link_flags}": get_starlark_list(info.opt_link_flags), - "%{unfiltered_compile_flags}": get_starlark_list(info.unfiltered_compile_flags), - "%{dbg_compile_flags}": get_starlark_list(info.dbg_compile_flags), - "%{coverage_compile_flags}": get_starlark_list(info.coverage_compile_flags), - "%{coverage_link_flags}": get_starlark_list(info.coverage_link_flags), - "%{supports_start_end_lib}": repr(info.supports_start_end_lib), - }, - ) - -_nixpkgs_cc_toolchain_config = repository_rule( - _nixpkgs_cc_toolchain_config_impl, - attrs = { - "cc_toolchain_info": attr.label(), - "fail_not_supported": attr.bool(), - "_unix_cc_toolchain_config": attr.label( - default = Label("@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl"), - ), - "_armeabi_cc_toolchain_config": attr.label( - default = Label("@bazel_tools//tools/cpp:armeabi_cc_toolchain_config.bzl"), - ), - "_generate_system_module_map": attr.label( - default = Label("@bazel_tools//tools/cpp:generate_system_module_map.sh"), - ), - "_osx_cc_wrapper": attr.label( - default = Label("@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl"), - ), - "_linux_cc_wrapper": attr.label( - default = Label("@bazel_tools//tools/cpp:linux_cc_wrapper.sh.tpl"), - ), - "_build": attr.label( - default = Label("@bazel_tools//tools/cpp:BUILD.tpl"), - ), - }, -) - -def _nixpkgs_cc_toolchain_impl(repository_ctx): - cpu = get_cpu_value(repository_ctx) - exec_constraints, target_constraints = ensure_constraints(repository_ctx) - - repository_ctx.file( - "BUILD.bazel", - executable = False, - content = """\ -package(default_visibility = ["//visibility:public"]) - -toolchain( - name = "cc-toolchain-{cpu}", - toolchain = "@{cc_toolchain_config}//:cc-compiler-{cpu}", - toolchain_type = "@rules_cc//cc:toolchain_type", - exec_compatible_with = {exec_constraints}, - target_compatible_with = {target_constraints}, -) - -toolchain( - name = "cc-toolchain-armeabi-v7a", - toolchain = "@{cc_toolchain_config}//:cc-compiler-armeabi-v7a", - toolchain_type = "@rules_cc//cc:toolchain_type", - exec_compatible_with = {exec_constraints}, - target_compatible_with = [ - "@platforms//cpu:arm", - "@platforms//os:android", - ], -) -""".format( - cc_toolchain_config = repository_ctx.attr.cc_toolchain_config, - cpu = cpu, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ), - ) - -_nixpkgs_cc_toolchain = repository_rule( - _nixpkgs_cc_toolchain_impl, - attrs = { - "cc_toolchain_config": attr.string(), - "exec_constraints": attr.string_list(), - "target_constraints": attr.string_list(), - }, -) - -def nixpkgs_cc_configure( - name = "local_config_cc", - attribute_path = "", - nix_file = None, - nix_file_content = "", - nix_file_deps = [], - repositories = {}, - repository = None, - nixopts = [], - quiet = False, - fail_not_supported = True, - exec_constraints = None, - target_constraints = None, - register = True): - """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. - - By default, Bazel auto-configures a CC toolchain from commands (e.g. - `gcc`) available in the environment. To make builds more hermetic, use - this rule to specify explicitly which commands the toolchain should use. - - Specifically, it builds a Nix derivation that provides the CC toolchain - tools in the `bin/` path and constructs a CC toolchain that uses those - tools. Tools that aren't found are replaced by `${coreutils}/bin/false`. - You can inspect the resulting `@_info//:CC_TOOLCHAIN_INFO` to see - which tools were discovered. - - This rule depends on [`rules_cc`](https://github.com/bazelbuild/rules_cc). - - **Note:** - You need to configure `--crosstool_top=@//:toolchain` to activate - this toolchain. - - Args: - attribute_path: optional, string, Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`. - nix_file: optional, Label, Obtain the toolchain from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. - nix_file_content: optional, string, Obtain the toolchain from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. - nix_file_deps: optional, list of Label, Additional files that the Nix expression depends on. - repositories: dict of Label to string, Provides `` and other repositories. Specify one of `repositories` or `repository`. - repository: Label, Provides ``. Specify one of `repositories` or `repository`. - nixopts: optional, list of string, Extra flags to pass when calling Nix. Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file ferenced by `LABEL` relative to the workspace root. - quiet: bool, Whether to hide `nix-build` output. - fail_not_supported: bool, Whether to fail if `nix-build` is not available. - exec_constraints: Constraints for the execution platform. - target_constraints: Constraints for the target platform. - register: bool, enabled by default, Whether to register (with `register_toolchains`) the generated toolchain and install it as the default cc_toolchain. - """ - - nixopts = list(nixopts) - nix_file_deps = list(nix_file_deps) - - nix_expr = None - if nix_file and nix_file_content: - fail("Cannot specify both 'nix_file' and 'nix_file_content'.") - elif nix_file: - nix_expr = "import $(location {})".format(nix_file) - nix_file_deps.append(nix_file) - elif nix_file_content: - nix_expr = nix_file_content - - if attribute_path and nix_expr == None: - fail("'attribute_path' requires one of 'nix_file' or 'nix_file_content'", "attribute_path") - elif attribute_path: - nixopts.extend([ - "--argstr", - "ccType", - "ccTypeAttribute", - "--argstr", - "ccAttrPath", - attribute_path, - "--arg", - "ccAttrSet", - nix_expr, - ]) - elif nix_expr: - nixopts.extend([ - "--argstr", - "ccType", - "ccTypeExpression", - "--arg", - "ccExpr", - nix_expr, - ]) - else: - nixopts.extend([ - "--argstr", - "ccType", - "ccTypeDefault", - ]) - - # Invoke `toolchains/cc.nix` which generates `CC_TOOLCHAIN_INFO`. - nixpkgs_package( - name = "{}_info".format(name), - nix_file = "@io_tweag_rules_nixpkgs//nixpkgs:toolchains/cc.nix", - nix_file_deps = nix_file_deps, - build_file_content = "exports_files(['CC_TOOLCHAIN_INFO'])", - repositories = repositories, - repository = repository, - nixopts = nixopts, - quiet = quiet, - fail_not_supported = fail_not_supported, - ) - - # Generate the `cc_toolchain_config` workspace. - _nixpkgs_cc_toolchain_config( - name = "{}".format(name), - cc_toolchain_info = "@{}_info//:CC_TOOLCHAIN_INFO".format(name), - fail_not_supported = fail_not_supported, - ) - - # Generate the `cc_toolchain` workspace. - if (exec_constraints == None) != (target_constraints == None): - fail("Both exec_constraints and target_constraints need to be provided or none of them.") - _nixpkgs_cc_toolchain( - name = "{}_toolchains".format(name), - cc_toolchain_config = name, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ) - - if register: - maybe( - native.bind, - name = "cc_toolchain", - actual = "@{}//:toolchain".format(name), - ) - native.register_toolchains("@{}_toolchains//:all".format(name)) - -def _readlink(repository_ctx, path): - return repository_ctx.path(path).realpath +nixpkgs_cc_configure = _nixpkgs_cc_configure def nixpkgs_cc_autoconf_impl(repository_ctx): cpu_value = get_cpu_value(repository_ctx) @@ -445,7 +61,7 @@ def nixpkgs_cc_autoconf_impl(repository_ctx): # the Bazel autoconfiguration with the tools we found. bin_contents = find_children(repository_ctx, workspace_root + "/bin") overriden_tools = { - tool: _readlink(repository_ctx, entry) + tool: repository_ctx.path(entry).realpath for entry in bin_contents for tool in [entry.rpartition("/")[-1]] # Compute basename } diff --git a/nixpkgs/repositories.bzl b/nixpkgs/repositories.bzl index 6df21c54d..177c021cb 100644 --- a/nixpkgs/repositories.bzl +++ b/nixpkgs/repositories.bzl @@ -33,17 +33,28 @@ def rules_nixpkgs_dependencies(local = None): url = "https://github.com/bazelbuild/rules_java/releases/download/4.0.0/rules_java-4.0.0.tar.gz", sha256 = "34b41ec683e67253043ab1a3d1e8b7c61e4e8edefbcad485381328c934d072fe", ) + url = "https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.8.1.tar.gz" if not local: + # XXX: no way to use `sha256` here, but if this surrounding repo comes + # from that URL, Bazel should hit the cache for the sub-workspaces maybe( http_archive, "rules_nixpkgs_core", - # XXX: no way to use `sha256` here, but if this surrounding repo comes - # from that URL, Bazel should hit the cache for the sub-workspace - url = "https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.8.1.tar.gz", + url = url, strip_prefix = "core", ) + maybe( + http_archive, + "rules_nixpkgs_cc", + url = url, + strip_prefix = "toolchains/cc", + ) else: native.local_repository( name = "rules_nixpkgs_core", path = local + "/core", ) + native.local_repository( + name = "rules_nixpkgs_cc", + path = local + "/toolchains/cc", + ) diff --git a/nixpkgs/toolchains/foreign_cc.bzl b/nixpkgs/toolchains/foreign_cc.bzl index fd262bfb1..dcc3cc230 100644 --- a/nixpkgs/toolchains/foreign_cc.bzl +++ b/nixpkgs/toolchains/foreign_cc.bzl @@ -1,114 +1,4 @@ -load("//nixpkgs:nixpkgs.bzl", "nixpkgs_package") -load("//core:util.bzl", "ensure_constraints") +# alias to Bazel module `toolchains/cc` +load("@rules_nixpkgs_cc//:foreign_cc.bzl", _nixpkgs_foreign_cc_configure = "nixpkgs_foreign_cc_configure") -_foreign_cc_nix_build = """ -load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain") - -filegroup( - name = "data", - srcs = glob(["bin/**"]), -) -native_tool_toolchain( - name = "cmake_nix_impl", - path = "bin/cmake", - target = ":data", -) -native_tool_toolchain( - name = "make_nix_impl", - path = "bin/make", - target = ":data", -) -native_tool_toolchain( - name = "ninja_nix_impl", - path = "bin/ninja", - target = ":data", -) -""" - -_foreign_cc_nix_toolchain = """ -toolchain( - name = "cmake_nix_toolchain", - toolchain = "@{toolchain_repo}//:cmake_nix_impl", - toolchain_type = "@rules_foreign_cc//toolchains:cmake_toolchain", - exec_compatible_with = {exec_constraints}, - target_compatible_with = {target_constraints}, -) -toolchain( - name = "make_nix_toolchain", - toolchain = "@{toolchain_repo}//:make_nix_impl", - toolchain_type = "@rules_foreign_cc//toolchains:make_toolchain", - exec_compatible_with = {exec_constraints}, - target_compatible_with = {target_constraints}, -) -toolchain( - name = "ninja_nix_toolchain", - toolchain = "@{toolchain_repo}//:ninja_nix_impl", - toolchain_type = "@rules_foreign_cc//toolchains:ninja_toolchain", - exec_compatible_with = {exec_constraints}, - target_compatible_with = {target_constraints}, -) -""" - -def _nixpkgs_foreign_cc_toolchain_impl(repository_ctx): - exec_constraints, target_constraints = ensure_constraints(repository_ctx) - repository_ctx.file( - "BUILD.bazel", - executable = False, - content = _foreign_cc_nix_toolchain.format( - toolchain_repo = repository_ctx.attr.toolchain_repo, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ), - ) - -_nixpkgs_foreign_cc_toolchain = repository_rule( - _nixpkgs_foreign_cc_toolchain_impl, - attrs = { - "toolchain_repo": attr.string(), - "exec_constraints": attr.string_list(), - "target_constraints": attr.string_list(), - }, -) - -def nixpkgs_foreign_cc_configure( - name = "nixpkgs_foreign_cc", - repository = None, - repositories = {}, - nix_file = None, - nix_file_deps = None, - nix_file_content = None, - nixopts = [], - fail_not_supported = True, - quiet = False, - exec_constraints = None, - target_constraints = None): - if not nix_file and not nix_file_content: - nix_file_content = """ - with import { config = {}; overlays = []; }; buildEnv { - name = "bazel-foreign-cc-toolchain"; - paths = [ cmake gnumake ninja glibc ]; - } - """ - nixpkgs_package( - name = name, - repository = repository, - repositories = repositories, - nix_file = nix_file, - nix_file_deps = nix_file_deps, - nix_file_content = nix_file_content, - build_file_content = _foreign_cc_nix_build, - nixopts = nixopts, - fail_not_supported = fail_not_supported, - quiet = quiet, - ) - _nixpkgs_foreign_cc_toolchain( - name = name + "_toolchain", - toolchain_repo = name, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ) - native.register_toolchains( - str(Label("@{}_toolchain//:cmake_nix_toolchain".format(name))), - str(Label("@{}_toolchain//:make_nix_toolchain".format(name))), - str(Label("@{}_toolchain//:ninja_nix_toolchain".format(name))), - ) +nixpkgs_foreign_cc_configure = _nixpkgs_foreign_cc_configure diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 089d41a85..837a93051 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -142,6 +142,7 @@ sh_test( data = [ "//nixpkgs:srcs", "//tests/invalid_nixpkgs_package:srcs", + "@rules_nixpkgs_cc//:cc.bzl", "@nix-unstable//:bin", ] + select({ "@platforms//os:linux": ["@busybox_static//:bin"], diff --git a/toolchains/cc/BUILD.bazel b/toolchains/cc/BUILD.bazel new file mode 100644 index 000000000..e7abf3300 --- /dev/null +++ b/toolchains/cc/BUILD.bazel @@ -0,0 +1,7 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "cc.bzl", + "foreign_cc.bzl", + "cc.nix", +]) diff --git a/toolchains/cc/MODULE.bazel b/toolchains/cc/MODULE.bazel new file mode 100644 index 000000000..a6dc3e21c --- /dev/null +++ b/toolchains/cc/MODULE.bazel @@ -0,0 +1,6 @@ +module( + name = "rules_nixpkgs_python", + version = "0.8.1", +) + +bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") diff --git a/toolchains/cc/WORKSPACE b/toolchains/cc/WORKSPACE new file mode 100644 index 000000000..2bda73373 --- /dev/null +++ b/toolchains/cc/WORKSPACE @@ -0,0 +1,2 @@ +# only temporary for compatibility with `WORKSPACE` setup +# TODO: remove when migration to `bzlmod` is complete diff --git a/toolchains/cc/cc.bzl b/toolchains/cc/cc.bzl new file mode 100644 index 000000000..1250cb03b --- /dev/null +++ b/toolchains/cc/cc.bzl @@ -0,0 +1,398 @@ +"""Rules for importing a C++ toolchain from Nixpkgs. +""" + +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "get_cpu_value", + "get_starlark_list", + "write_builtin_include_directory_paths", +) +load("@bazel_skylib//lib:sets.bzl", "sets") +load("@bazel_skylib//lib:versions.bzl", "versions") +load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") +load( + "@rules_nixpkgs_core//:util.bzl", + "ensure_constraints", + "execute_or_fail", +) + +def _parse_cc_toolchain_info(content, filename): + """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. + + Attrs: + content: string, The content of the `CC_TOOLCHAIN_INFO` file. + filename: string, The path to the `CC_TOOLCHAIN_INFO` file, used for error reporting. + + Returns: + struct, The substitutions for `@bazel_tools//tools/cpp:BUILD.tpl`. + """ + + # Parse the content of CC_TOOLCHAIN_INFO. + # + # Each line has the form + # + # :::... + info = {} + for line in content.splitlines(): + fields = line.split(":") + if len(fields) == 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Empty line encountered.".format(filename), + "cc_toolchain_info", + ) + info[fields[0]] = fields[1:] + + # Validate the keys in CC_TOOLCHAIN_INFO. + expected_keys = sets.make([ + "TOOL_NAMES", + "TOOL_PATHS", + "CXX_BUILTIN_INCLUDE_DIRECTORIES", + "COMPILE_FLAGS", + "CXX_FLAGS", + "LINK_FLAGS", + "LINK_LIBS", + "OPT_COMPILE_FLAGS", + "OPT_LINK_FLAGS", + "UNFILTERED_COMPILE_FLAGS", + "DBG_COMPILE_FLAGS", + "COVERAGE_COMPILE_FLAGS", + "COVERAGE_LINK_FLAGS", + "SUPPORTS_START_END_LIB", + "IS_CLANG", + ]) + actual_keys = sets.make(info.keys()) + missing_keys = sets.difference(expected_keys, actual_keys) + unexpected_keys = sets.difference(actual_keys, expected_keys) + if sets.length(missing_keys) > 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Missing entries '{}'.".format( + filename, + "', '".join(sets.to_list(missing_keys)), + ), + "cc_toolchain_info", + ) + if sets.length(unexpected_keys) > 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Unexpected entries '{}'.".format( + filename, + "', '".join(sets.to_list(unexpected_keys)), + ), + "cc_toolchain_info", + ) + + return struct( + tool_paths = { + tool: path + for (tool, path) in zip(info["TOOL_NAMES"], info["TOOL_PATHS"]) + }, + cxx_builtin_include_directories = info["CXX_BUILTIN_INCLUDE_DIRECTORIES"], + compile_flags = info["COMPILE_FLAGS"], + cxx_flags = info["CXX_FLAGS"], + link_flags = info["LINK_FLAGS"], + link_libs = info["LINK_LIBS"], + opt_compile_flags = info["OPT_COMPILE_FLAGS"], + opt_link_flags = info["OPT_LINK_FLAGS"], + unfiltered_compile_flags = info["UNFILTERED_COMPILE_FLAGS"], + dbg_compile_flags = info["DBG_COMPILE_FLAGS"], + coverage_compile_flags = info["COVERAGE_COMPILE_FLAGS"], + coverage_link_flags = info["COVERAGE_LINK_FLAGS"], + supports_start_end_lib = info["SUPPORTS_START_END_LIB"] == ["True"], + is_clang = info["IS_CLANG"] == ["True"], + ) + +def _nixpkgs_cc_toolchain_config_impl(repository_ctx): + cpu_value = get_cpu_value(repository_ctx) + darwin = cpu_value == "darwin" + + cc_toolchain_info_file = repository_ctx.path(repository_ctx.attr.cc_toolchain_info) + if not cc_toolchain_info_file.exists and not repository_ctx.attr.fail_not_supported: + return + info = _parse_cc_toolchain_info( + repository_ctx.read(cc_toolchain_info_file), + cc_toolchain_info_file, + ) + + # Generate the cc_toolchain workspace following the example from + # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. + # Uses the corresponding templates from `@bazel_tools` as well, see the + # private attributes of the `_nixpkgs_cc_toolchain_config` rule. + repository_ctx.symlink( + repository_ctx.path(repository_ctx.attr._unix_cc_toolchain_config), + "cc_toolchain_config.bzl", + ) + repository_ctx.symlink( + repository_ctx.path(repository_ctx.attr._armeabi_cc_toolchain_config), + "armeabi_cc_toolchain_config.bzl", + ) + + # A module map is required for clang starting from Bazel version 3.3.0. + # https://github.com/bazelbuild/bazel/commit/8b9f74649512ee17ac52815468bf3d7e5e71c9fa + needs_module_map = info.is_clang and versions.is_at_least("3.3.0", versions.get()) + if needs_module_map: + generate_system_module_map = [ + repository_ctx.path(repository_ctx.attr._generate_system_module_map), + ] + repository_ctx.file( + "module.modulemap", + execute_or_fail( + repository_ctx, + generate_system_module_map + info.cxx_builtin_include_directories, + "Failed to generate system module map.", + ).stdout.strip(), + executable = False, + ) + cc_wrapper_src = ( + repository_ctx.attr._osx_cc_wrapper if darwin else repository_ctx.attr._linux_cc_wrapper + ) + repository_ctx.template( + "cc_wrapper.sh", + repository_ctx.path(cc_wrapper_src), + { + "%{cc}": info.tool_paths["gcc"], + "%{env}": "", + }, + ) + if darwin: + info.tool_paths["gcc"] = "cc_wrapper.sh" + info.tool_paths["ar"] = "/usr/bin/libtool" + write_builtin_include_directory_paths( + repository_ctx, + info.tool_paths["gcc"], + info.cxx_builtin_include_directories, + ) + repository_ctx.template( + "BUILD.bazel", + repository_ctx.path(repository_ctx.attr._build), + { + "%{cc_toolchain_identifier}": "local", + "%{name}": cpu_value, + "%{modulemap}": ("\":module.modulemap\"" if needs_module_map else "None"), + "%{supports_param_files}": "0" if darwin else "1", + "%{cc_compiler_deps}": get_starlark_list( + [":builtin_include_directory_paths"] + ( + [":cc_wrapper"] if darwin else [] + ), + ), + "%{compiler}": "compiler", + "%{abi_version}": "local", + "%{abi_libc_version}": "local", + "%{host_system_name}": "local", + "%{target_libc}": "macosx" if darwin else "local", + "%{target_cpu}": cpu_value, + "%{target_system_name}": "local", + "%{tool_paths}": ",\n ".join( + ['"%s": "%s"' % (k, v) for (k, v) in info.tool_paths.items()], + ), + "%{cxx_builtin_include_directories}": get_starlark_list(info.cxx_builtin_include_directories), + "%{compile_flags}": get_starlark_list(info.compile_flags), + "%{cxx_flags}": get_starlark_list(info.cxx_flags), + "%{link_flags}": get_starlark_list(info.link_flags), + "%{link_libs}": get_starlark_list(info.link_libs), + "%{opt_compile_flags}": get_starlark_list(info.opt_compile_flags), + "%{opt_link_flags}": get_starlark_list(info.opt_link_flags), + "%{unfiltered_compile_flags}": get_starlark_list(info.unfiltered_compile_flags), + "%{dbg_compile_flags}": get_starlark_list(info.dbg_compile_flags), + "%{coverage_compile_flags}": get_starlark_list(info.coverage_compile_flags), + "%{coverage_link_flags}": get_starlark_list(info.coverage_link_flags), + "%{supports_start_end_lib}": repr(info.supports_start_end_lib), + }, + ) + +_nixpkgs_cc_toolchain_config = repository_rule( + _nixpkgs_cc_toolchain_config_impl, + attrs = { + "cc_toolchain_info": attr.label(), + "fail_not_supported": attr.bool(), + "_unix_cc_toolchain_config": attr.label( + default = Label("@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl"), + ), + "_armeabi_cc_toolchain_config": attr.label( + default = Label("@bazel_tools//tools/cpp:armeabi_cc_toolchain_config.bzl"), + ), + "_generate_system_module_map": attr.label( + default = Label("@bazel_tools//tools/cpp:generate_system_module_map.sh"), + ), + "_osx_cc_wrapper": attr.label( + default = Label("@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl"), + ), + "_linux_cc_wrapper": attr.label( + default = Label("@bazel_tools//tools/cpp:linux_cc_wrapper.sh.tpl"), + ), + "_build": attr.label( + default = Label("@bazel_tools//tools/cpp:BUILD.tpl"), + ), + }, +) + +def _nixpkgs_cc_toolchain_impl(repository_ctx): + cpu = get_cpu_value(repository_ctx) + exec_constraints, target_constraints = ensure_constraints(repository_ctx) + + repository_ctx.file( + "BUILD.bazel", + executable = False, + content = """\ +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "cc-toolchain-{cpu}", + toolchain = "@{cc_toolchain_config}//:cc-compiler-{cpu}", + toolchain_type = "@rules_cc//cc:toolchain_type", + exec_compatible_with = {exec_constraints}, + target_compatible_with = {target_constraints}, +) + +toolchain( + name = "cc-toolchain-armeabi-v7a", + toolchain = "@{cc_toolchain_config}//:cc-compiler-armeabi-v7a", + toolchain_type = "@rules_cc//cc:toolchain_type", + exec_compatible_with = {exec_constraints}, + target_compatible_with = [ + "@platforms//cpu:arm", + "@platforms//os:android", + ], +) +""".format( + cc_toolchain_config = repository_ctx.attr.cc_toolchain_config, + cpu = cpu, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ), + ) + +_nixpkgs_cc_toolchain = repository_rule( + _nixpkgs_cc_toolchain_impl, + attrs = { + "cc_toolchain_config": attr.string(), + "exec_constraints": attr.string_list(), + "target_constraints": attr.string_list(), + }, +) + +def nixpkgs_cc_configure( + name = "local_config_cc", + attribute_path = "", + nix_file = None, + nix_file_content = "", + nix_file_deps = [], + repositories = {}, + repository = None, + nixopts = [], + quiet = False, + fail_not_supported = True, + exec_constraints = None, + target_constraints = None, + register = True): + """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. + + By default, Bazel auto-configures a CC toolchain from commands (e.g. + `gcc`) available in the environment. To make builds more hermetic, use + this rule to specify explicitly which commands the toolchain should use. + + Specifically, it builds a Nix derivation that provides the CC toolchain + tools in the `bin/` path and constructs a CC toolchain that uses those + tools. Tools that aren't found are replaced by `${coreutils}/bin/false`. + You can inspect the resulting `@_info//:CC_TOOLCHAIN_INFO` to see + which tools were discovered. + + This rule depends on [`rules_cc`](https://github.com/bazelbuild/rules_cc). + + **Note:** + You need to configure `--crosstool_top=@//:toolchain` to activate + this toolchain. + + Args: + attribute_path: optional, string, Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`. + nix_file: optional, Label, Obtain the toolchain from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. + nix_file_content: optional, string, Obtain the toolchain from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. + nix_file_deps: optional, list of Label, Additional files that the Nix expression depends on. + repositories: dict of Label to string, Provides `` and other repositories. Specify one of `repositories` or `repository`. + repository: Label, Provides ``. Specify one of `repositories` or `repository`. + nixopts: optional, list of string, Extra flags to pass when calling Nix. Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file ferenced by `LABEL` relative to the workspace root. + quiet: bool, Whether to hide `nix-build` output. + fail_not_supported: bool, Whether to fail if `nix-build` is not available. + exec_constraints: Constraints for the execution platform. + target_constraints: Constraints for the target platform. + register: bool, enabled by default, Whether to register (with `register_toolchains`) the generated toolchain and install it as the default cc_toolchain. + """ + + nixopts = list(nixopts) + nix_file_deps = list(nix_file_deps) + + nix_expr = None + if nix_file and nix_file_content: + fail("Cannot specify both 'nix_file' and 'nix_file_content'.") + elif nix_file: + nix_expr = "import $(location {})".format(nix_file) + nix_file_deps.append(nix_file) + elif nix_file_content: + nix_expr = nix_file_content + + if attribute_path and nix_expr == None: + fail("'attribute_path' requires one of 'nix_file' or 'nix_file_content'", "attribute_path") + elif attribute_path: + nixopts.extend([ + "--argstr", + "ccType", + "ccTypeAttribute", + "--argstr", + "ccAttrPath", + attribute_path, + "--arg", + "ccAttrSet", + nix_expr, + ]) + elif nix_expr: + nixopts.extend([ + "--argstr", + "ccType", + "ccTypeExpression", + "--arg", + "ccExpr", + nix_expr, + ]) + else: + nixopts.extend([ + "--argstr", + "ccType", + "ccTypeDefault", + ]) + + # Invoke `cc.nix` which generates `CC_TOOLCHAIN_INFO`. + nixpkgs_package( + name = "{}_info".format(name), + nix_file = "@rules_nixpkgs_cc//:cc.nix", + nix_file_deps = nix_file_deps, + build_file_content = "exports_files(['CC_TOOLCHAIN_INFO'])", + repositories = repositories, + repository = repository, + nixopts = nixopts, + quiet = quiet, + fail_not_supported = fail_not_supported, + ) + + # Generate the `cc_toolchain_config` workspace. + _nixpkgs_cc_toolchain_config( + name = "{}".format(name), + cc_toolchain_info = "@{}_info//:CC_TOOLCHAIN_INFO".format(name), + fail_not_supported = fail_not_supported, + ) + + # Generate the `cc_toolchain` workspace. + if (exec_constraints == None) != (target_constraints == None): + fail("Both exec_constraints and target_constraints need to be provided or none of them.") + _nixpkgs_cc_toolchain( + name = "{}_toolchains".format(name), + cc_toolchain_config = name, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ) + + if register: + maybe( + native.bind, + name = "cc_toolchain", + actual = "@{}//:toolchain".format(name), + ) + native.register_toolchains("@{}_toolchains//:all".format(name)) diff --git a/nixpkgs/toolchains/cc.nix b/toolchains/cc/cc.nix similarity index 100% rename from nixpkgs/toolchains/cc.nix rename to toolchains/cc/cc.nix diff --git a/toolchains/cc/foreign_cc.bzl b/toolchains/cc/foreign_cc.bzl new file mode 100644 index 000000000..bdec71ced --- /dev/null +++ b/toolchains/cc/foreign_cc.bzl @@ -0,0 +1,114 @@ +load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") +load("@rules_nixpkgs_core//:util.bzl", "ensure_constraints") + +_foreign_cc_nix_build = """ +load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain") + +filegroup( + name = "data", + srcs = glob(["bin/**"]), +) +native_tool_toolchain( + name = "cmake_nix_impl", + path = "bin/cmake", + target = ":data", +) +native_tool_toolchain( + name = "make_nix_impl", + path = "bin/make", + target = ":data", +) +native_tool_toolchain( + name = "ninja_nix_impl", + path = "bin/ninja", + target = ":data", +) +""" + +_foreign_cc_nix_toolchain = """ +toolchain( + name = "cmake_nix_toolchain", + toolchain = "@{toolchain_repo}//:cmake_nix_impl", + toolchain_type = "@rules_foreign_cc//toolchains:cmake_toolchain", + exec_compatible_with = {exec_constraints}, + target_compatible_with = {target_constraints}, +) +toolchain( + name = "make_nix_toolchain", + toolchain = "@{toolchain_repo}//:make_nix_impl", + toolchain_type = "@rules_foreign_cc//toolchains:make_toolchain", + exec_compatible_with = {exec_constraints}, + target_compatible_with = {target_constraints}, +) +toolchain( + name = "ninja_nix_toolchain", + toolchain = "@{toolchain_repo}//:ninja_nix_impl", + toolchain_type = "@rules_foreign_cc//toolchains:ninja_toolchain", + exec_compatible_with = {exec_constraints}, + target_compatible_with = {target_constraints}, +) +""" + +def _nixpkgs_foreign_cc_toolchain_impl(repository_ctx): + exec_constraints, target_constraints = ensure_constraints(repository_ctx) + repository_ctx.file( + "BUILD.bazel", + executable = False, + content = _foreign_cc_nix_toolchain.format( + toolchain_repo = repository_ctx.attr.toolchain_repo, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ), + ) + +_nixpkgs_foreign_cc_toolchain = repository_rule( + _nixpkgs_foreign_cc_toolchain_impl, + attrs = { + "toolchain_repo": attr.string(), + "exec_constraints": attr.string_list(), + "target_constraints": attr.string_list(), + }, +) + +def nixpkgs_foreign_cc_configure( + name = "nixpkgs_foreign_cc", + repository = None, + repositories = {}, + nix_file = None, + nix_file_deps = None, + nix_file_content = None, + nixopts = [], + fail_not_supported = True, + quiet = False, + exec_constraints = None, + target_constraints = None): + if not nix_file and not nix_file_content: + nix_file_content = """ + with import { config = {}; overlays = []; }; buildEnv { + name = "bazel-foreign-cc-toolchain"; + paths = [ cmake gnumake ninja glibc ]; + } + """ + nixpkgs_package( + name = name, + repository = repository, + repositories = repositories, + nix_file = nix_file, + nix_file_deps = nix_file_deps, + nix_file_content = nix_file_content, + build_file_content = _foreign_cc_nix_build, + nixopts = nixopts, + fail_not_supported = fail_not_supported, + quiet = quiet, + ) + _nixpkgs_foreign_cc_toolchain( + name = name + "_toolchain", + toolchain_repo = name, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ) + native.register_toolchains( + str(Label("@{}_toolchain//:cmake_nix_toolchain".format(name))), + str(Label("@{}_toolchain//:make_nix_toolchain".format(name))), + str(Label("@{}_toolchain//:ninja_nix_toolchain".format(name))), + ) From f525a130b9411d52b1ee39ca929878afec988386 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Wed, 9 Mar 2022 11:37:52 +0100 Subject: [PATCH 11/25] split long sentence --- README.md | 4 ++-- toolchains/go/go.bzl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 32081ac24..7f70f6da1 100644 --- a/README.md +++ b/README.md @@ -1417,8 +1417,8 @@ nixpkgs_go_configure(sdk_name, Date: Wed, 9 Mar 2022 11:45:09 +0100 Subject: [PATCH 12/25] add notes on aliases to aid migration --- nixpkgs/toolchains/go.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixpkgs/toolchains/go.bzl b/nixpkgs/toolchains/go.bzl index 2fb1519f2..1ce9428fd 100644 --- a/nixpkgs/toolchains/go.bzl +++ b/nixpkgs/toolchains/go.bzl @@ -7,6 +7,8 @@ dependencies on rules_go for those who don't need go toolchain. `@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl`.** """ +# alias to Bazel module `toolchains/go` + load( "//toolchains/go:go.bzl", _nixpkgs_go_configure = "nixpkgs_go_configure", From 8297cec2ccf429be3fbff4e769c155d380f76a87 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Wed, 9 Mar 2022 12:45:59 +0100 Subject: [PATCH 13/25] extract Rust toolchain into module --- nixpkgs/toolchains/rust.bzl | 175 +---------------------------------- toolchains/rust/BUILD.bazel | 5 + toolchains/rust/MODULE.bazel | 6 ++ toolchains/rust/rust.bzl | 174 ++++++++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 172 deletions(-) create mode 100644 toolchains/rust/BUILD.bazel create mode 100644 toolchains/rust/MODULE.bazel create mode 100644 toolchains/rust/rust.bzl diff --git a/nixpkgs/toolchains/rust.bzl b/nixpkgs/toolchains/rust.bzl index b1f6ea4e8..803f00106 100644 --- a/nixpkgs/toolchains/rust.bzl +++ b/nixpkgs/toolchains/rust.bzl @@ -1,174 +1,5 @@ -load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") -load("@rules_nixpkgs_core//:util.bzl", "ensure_constraints") +# alias to Bazel module `toolchains/go` -# Adapted from rules_rust toolchain BUILD: -# https://github.com/bazelbuild/rules_rust/blob/fd436df9e2d4ac1b234ca5e969e34a4cb5891910/rust/private/repository_utils.bzl#L17-L46 -# Nix generation is used to dynamically compute both Linux and Darwin environments -_rust_nix_contents = """\ -let - pkgs = import {{ config = {{}}; overrides = []; }}; - rust = pkgs.rust; - os = rust.toTargetOs pkgs.stdenv.targetPlatform; - build-triple = rust.toRustTargetSpec pkgs.stdenv.buildPlatform; - target-triple = rust.toRustTargetSpec pkgs.stdenv.targetPlatform; -in -pkgs.buildEnv {{ - extraOutputsToInstall = ["out" "bin" "lib"]; - name = "bazel-rust-toolchain"; - paths = [ pkgs.rustc pkgs.rustfmt pkgs.cargo pkgs.clippy ]; - postBuild = '' - cat < $out/BUILD - filegroup( - name = "rustc", - srcs = ["bin/rustc"], - visibility = ["//visibility:public"], - ) +load("//toolchains/rust:rust.bzl", _nixpkgs_rust_configure = "nixpkgs_rust_configure") - filegroup( - name = "rustfmt", - srcs = ["bin/rustfmt"], - visibility = ["//visibility:public"], - ) - - filegroup( - name = "clippy_driver", - srcs = ["bin/clippy-driver"], - visibility = ["//visibility:public"], - ) - - filegroup( - name = "rustc_lib", - srcs = glob( - [ - "bin/*.so", - "lib/*.so", - "lib/rustlib/*/codegen-backends/*.so", - "lib/rustlib/*/codegen-backends/*.dylib", - "lib/rustlib/*/bin/rust-lld", - "lib/rustlib/*/lib/*.so", - "lib/rustlib/*/lib/*.dylib", - ], - allow_empty = True, - ), - visibility = ["//visibility:public"], - ) - - load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") - rust_stdlib_filegroup( - name = "rust_lib", - srcs = glob( - [ - "lib/rustlib/*/lib/*.rlib", - "lib/rustlib/*/lib/*.so", - "lib/rustlib/*/lib/*.dylib", - "lib/rustlib/*/lib/*.a", - "lib/rustlib/*/lib/self-contained/**", - ], - # Some patterns (e.g. `lib/*.a`) don't match anything, see https://github.com/bazelbuild/rules_rust/pull/245 - allow_empty = True, - ), - visibility = ["//visibility:public"], - ) - - filegroup( - name = "rust_doc", - srcs = ["bin/rustdoc"], - visibility = ["//visibility:public"], - ) - - load('@rules_rust//rust:toolchain.bzl', 'rust_toolchain') - rust_toolchain( - name = "rust_nix_impl", - rust_doc = ":rust_doc", - rust_lib = ":rust_lib", - rustc = ":rustc", - rustfmt = ":rustfmt", - cargo = ":cargo", - clippy_driver = ":clippy_driver", - rustc_lib = ":rustc_lib", - binary_ext = "{binary_ext}", - staticlib_ext = "{staticlib_ext}", - dylib_ext = "{dylib_ext}", - os = "${{os}}", - exec_triple = "${{build-triple}}", - target_triple = "${{target-triple}}", - default_edition = "{default_edition}", - stdlib_linkflags = {stdlib_linkflags}, - visibility = ["//visibility:public"], - ) - EOF - ''; -}} -""" - -_rust_nix_toolchain = """ -toolchain( - name = "rust_nix", - toolchain = "@{toolchain_repo}//:rust_nix_impl", - toolchain_type = "@rules_rust//rust:toolchain", - exec_compatible_with = {exec_constraints}, - target_compatible_with = {target_constraints}, -) -""" - -def _nixpkgs_rust_toolchain_impl(repository_ctx): - exec_constraints, target_constraints = ensure_constraints(repository_ctx) - repository_ctx.file( - "BUILD.bazel", - executable = False, - content = _rust_nix_toolchain.format( - toolchain_repo = repository_ctx.attr.toolchain_repo, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ), - ) - -_nixpkgs_rust_toolchain = repository_rule( - _nixpkgs_rust_toolchain_impl, - attrs = { - "toolchain_repo": attr.string(), - "exec_constraints": attr.string_list(), - "target_constraints": attr.string_list(), - }, -) - -def nixpkgs_rust_configure( - name = "nixpkgs_rust", - default_edition = "2018", - repository = None, - repositories = {}, - nix_file = None, - nix_file_deps = None, - nix_file_content = None, - nixopts = [], - fail_not_supported = True, - quiet = False, - exec_constraints = None, - target_constraints = None): - if not nix_file and not nix_file_content: - nix_file_content = _rust_nix_contents.format( - binary_ext = "", - dylib_ext = ".so", - staticlib_ext = ".a", - default_edition = default_edition, - stdlib_linkflags = '["-lpthread", "-ldl"]', - ) - - nixpkgs_package( - name = name, - repository = repository, - repositories = repositories, - nix_file = nix_file, - nix_file_deps = nix_file_deps, - nix_file_content = nix_file_content, - nixopts = nixopts, - fail_not_supported = fail_not_supported, - quiet = quiet, - ) - _nixpkgs_rust_toolchain( - name = name + "_toolchain", - toolchain_repo = name, - exec_constraints = exec_constraints, - target_constraints = target_constraints, - ) - native.register_toolchains("@{}_toolchain//:rust_nix".format(name)) +nixpkgs_rust_configure = _nixpkgs_rust_configure diff --git a/toolchains/rust/BUILD.bazel b/toolchains/rust/BUILD.bazel new file mode 100644 index 000000000..54381548a --- /dev/null +++ b/toolchains/rust/BUILD.bazel @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "rust.bzl", +]) \ No newline at end of file diff --git a/toolchains/rust/MODULE.bazel b/toolchains/rust/MODULE.bazel new file mode 100644 index 000000000..582fd610c --- /dev/null +++ b/toolchains/rust/MODULE.bazel @@ -0,0 +1,6 @@ +module( + name = "rules_nixpkgs_rust", + version = "0.8.1", +) + +bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") diff --git a/toolchains/rust/rust.bzl b/toolchains/rust/rust.bzl new file mode 100644 index 000000000..b1f6ea4e8 --- /dev/null +++ b/toolchains/rust/rust.bzl @@ -0,0 +1,174 @@ +load("@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_package") +load("@rules_nixpkgs_core//:util.bzl", "ensure_constraints") + +# Adapted from rules_rust toolchain BUILD: +# https://github.com/bazelbuild/rules_rust/blob/fd436df9e2d4ac1b234ca5e969e34a4cb5891910/rust/private/repository_utils.bzl#L17-L46 +# Nix generation is used to dynamically compute both Linux and Darwin environments +_rust_nix_contents = """\ +let + pkgs = import {{ config = {{}}; overrides = []; }}; + rust = pkgs.rust; + os = rust.toTargetOs pkgs.stdenv.targetPlatform; + build-triple = rust.toRustTargetSpec pkgs.stdenv.buildPlatform; + target-triple = rust.toRustTargetSpec pkgs.stdenv.targetPlatform; +in +pkgs.buildEnv {{ + extraOutputsToInstall = ["out" "bin" "lib"]; + name = "bazel-rust-toolchain"; + paths = [ pkgs.rustc pkgs.rustfmt pkgs.cargo pkgs.clippy ]; + postBuild = '' + cat < $out/BUILD + filegroup( + name = "rustc", + srcs = ["bin/rustc"], + visibility = ["//visibility:public"], + ) + + filegroup( + name = "rustfmt", + srcs = ["bin/rustfmt"], + visibility = ["//visibility:public"], + ) + + filegroup( + name = "clippy_driver", + srcs = ["bin/clippy-driver"], + visibility = ["//visibility:public"], + ) + + filegroup( + name = "rustc_lib", + srcs = glob( + [ + "bin/*.so", + "lib/*.so", + "lib/rustlib/*/codegen-backends/*.so", + "lib/rustlib/*/codegen-backends/*.dylib", + "lib/rustlib/*/bin/rust-lld", + "lib/rustlib/*/lib/*.so", + "lib/rustlib/*/lib/*.dylib", + ], + allow_empty = True, + ), + visibility = ["//visibility:public"], + ) + + load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + rust_stdlib_filegroup( + name = "rust_lib", + srcs = glob( + [ + "lib/rustlib/*/lib/*.rlib", + "lib/rustlib/*/lib/*.so", + "lib/rustlib/*/lib/*.dylib", + "lib/rustlib/*/lib/*.a", + "lib/rustlib/*/lib/self-contained/**", + ], + # Some patterns (e.g. `lib/*.a`) don't match anything, see https://github.com/bazelbuild/rules_rust/pull/245 + allow_empty = True, + ), + visibility = ["//visibility:public"], + ) + + filegroup( + name = "rust_doc", + srcs = ["bin/rustdoc"], + visibility = ["//visibility:public"], + ) + + load('@rules_rust//rust:toolchain.bzl', 'rust_toolchain') + rust_toolchain( + name = "rust_nix_impl", + rust_doc = ":rust_doc", + rust_lib = ":rust_lib", + rustc = ":rustc", + rustfmt = ":rustfmt", + cargo = ":cargo", + clippy_driver = ":clippy_driver", + rustc_lib = ":rustc_lib", + binary_ext = "{binary_ext}", + staticlib_ext = "{staticlib_ext}", + dylib_ext = "{dylib_ext}", + os = "${{os}}", + exec_triple = "${{build-triple}}", + target_triple = "${{target-triple}}", + default_edition = "{default_edition}", + stdlib_linkflags = {stdlib_linkflags}, + visibility = ["//visibility:public"], + ) + EOF + ''; +}} +""" + +_rust_nix_toolchain = """ +toolchain( + name = "rust_nix", + toolchain = "@{toolchain_repo}//:rust_nix_impl", + toolchain_type = "@rules_rust//rust:toolchain", + exec_compatible_with = {exec_constraints}, + target_compatible_with = {target_constraints}, +) +""" + +def _nixpkgs_rust_toolchain_impl(repository_ctx): + exec_constraints, target_constraints = ensure_constraints(repository_ctx) + repository_ctx.file( + "BUILD.bazel", + executable = False, + content = _rust_nix_toolchain.format( + toolchain_repo = repository_ctx.attr.toolchain_repo, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ), + ) + +_nixpkgs_rust_toolchain = repository_rule( + _nixpkgs_rust_toolchain_impl, + attrs = { + "toolchain_repo": attr.string(), + "exec_constraints": attr.string_list(), + "target_constraints": attr.string_list(), + }, +) + +def nixpkgs_rust_configure( + name = "nixpkgs_rust", + default_edition = "2018", + repository = None, + repositories = {}, + nix_file = None, + nix_file_deps = None, + nix_file_content = None, + nixopts = [], + fail_not_supported = True, + quiet = False, + exec_constraints = None, + target_constraints = None): + if not nix_file and not nix_file_content: + nix_file_content = _rust_nix_contents.format( + binary_ext = "", + dylib_ext = ".so", + staticlib_ext = ".a", + default_edition = default_edition, + stdlib_linkflags = '["-lpthread", "-ldl"]', + ) + + nixpkgs_package( + name = name, + repository = repository, + repositories = repositories, + nix_file = nix_file, + nix_file_deps = nix_file_deps, + nix_file_content = nix_file_content, + nixopts = nixopts, + fail_not_supported = fail_not_supported, + quiet = quiet, + ) + _nixpkgs_rust_toolchain( + name = name + "_toolchain", + toolchain_repo = name, + exec_constraints = exec_constraints, + target_constraints = target_constraints, + ) + native.register_toolchains("@{}_toolchain//:rust_nix".format(name)) From 6aa5eb65c5be84914f84bc4e84b59c37d491e58b Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Wed, 9 Mar 2022 13:06:30 +0100 Subject: [PATCH 14/25] wrap Java toolchain in workspace otherwise internal references will break --- nixpkgs/BUILD.bazel | 2 +- nixpkgs/nixpkgs.bzl | 2 +- nixpkgs/repositories.bzl | 11 +++++++++++ toolchains/java/WORKSPACE | 2 ++ toolchains/java/java.bzl | 2 +- toolchains/java/local_java_repository.bzl | 2 +- 6 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 toolchains/java/WORKSPACE diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 220058288..7b0ddcb25 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -42,7 +42,7 @@ bzl_library( deps = [ "@rules_nixpkgs_core//:core", "@rules_nixpkgs_cc//:cc.bzl", - "//toolchains/java:java.bzl", + "@rules_nixpkgs_java//:java.bzl", "//toolchains/python:python.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index ad87b7b06..d26d830b9 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -22,7 +22,7 @@ load( _nixpkgs_python_configure = "nixpkgs_python_configure", ) load( - "//toolchains/java:java.bzl", + "@rules_nixpkgs_java//:java.bzl", _nixpkgs_java_configure = "nixpkgs_java_configure", ) load( diff --git a/nixpkgs/repositories.bzl b/nixpkgs/repositories.bzl index 177c021cb..fababecb4 100644 --- a/nixpkgs/repositories.bzl +++ b/nixpkgs/repositories.bzl @@ -49,6 +49,12 @@ def rules_nixpkgs_dependencies(local = None): url = url, strip_prefix = "toolchains/cc", ) + maybe( + http_archive, + "rules_nixpkgs_java", + url = url, + strip_prefix = "toolchains/java", + ) else: native.local_repository( name = "rules_nixpkgs_core", @@ -58,3 +64,8 @@ def rules_nixpkgs_dependencies(local = None): name = "rules_nixpkgs_cc", path = local + "/toolchains/cc", ) + native.local_repository( + name = "rules_nixpkgs_java", + path = local + "/toolchains/java", + ) + diff --git a/toolchains/java/WORKSPACE b/toolchains/java/WORKSPACE new file mode 100644 index 000000000..a69f72f2c --- /dev/null +++ b/toolchains/java/WORKSPACE @@ -0,0 +1,2 @@ +# only temporary for compatibility with `WORKSPACE` setup +# TODO: remove when migration to `bzlmod` is complete \ No newline at end of file diff --git a/toolchains/java/java.bzl b/toolchains/java/java.bzl index 731a08cc8..43d754b2f 100644 --- a/toolchains/java/java.bzl +++ b/toolchains/java/java.bzl @@ -66,7 +66,7 @@ def _nixpkgs_java_toolchain_impl(repository_ctx): "BUILD.bazel", executable = False, content = """\ -load("@io_tweag_rules_nixpkgs//toolchains/java:local_java_repository.bzl", "local_java_runtime") +load("@rules_nixpkgs_java//:local_java_repository.bzl", "local_java_runtime") local_java_runtime( name = "{name}", version = "{version}", diff --git a/toolchains/java/local_java_repository.bzl b/toolchains/java/local_java_repository.bzl index 53c19775c..cf19104d5 100644 --- a/toolchains/java/local_java_repository.bzl +++ b/toolchains/java/local_java_repository.bzl @@ -14,7 +14,7 @@ """Rules for importing and registering a local JDK.""" -load("//toolchains/java:default_java_toolchain.bzl", "JVM8_TOOLCHAIN_CONFIGURATION", "default_java_toolchain") +load("@rules_nixpkgs_java//:default_java_toolchain.bzl", "JVM8_TOOLCHAIN_CONFIGURATION", "default_java_toolchain") def _detect_java_version(repository_ctx, java_bin): properties_out = repository_ctx.execute([java_bin, "-XshowSettings:properties"]).stderr From 95e898641816a40f21860b6f6813bc092126f267 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Thu, 10 Mar 2022 10:32:59 +0100 Subject: [PATCH 15/25] wrap Python toolchain in workspace necessary to facilitate sandboxed integration testing, where the original source directory of the main workspace is not accessible without more jumping through hoops. --- nixpkgs/BUILD.bazel | 2 +- nixpkgs/nixpkgs.bzl | 2 +- toolchains/python/BUILD.bazel | 8 +++++++- toolchains/python/WORKSPACE | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 toolchains/python/WORKSPACE diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 7b0ddcb25..8aedcbe72 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -43,7 +43,7 @@ bzl_library( "@rules_nixpkgs_core//:core", "@rules_nixpkgs_cc//:cc.bzl", "@rules_nixpkgs_java//:java.bzl", - "//toolchains/python:python.bzl", + "@rules_nixpkgs_python//:python.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index d26d830b9..6fadcf584 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -18,7 +18,7 @@ load( "is_supported_platform", ) load( - "//toolchains/python:python.bzl", + "@rules_nixpkgs_python//:python.bzl", _nixpkgs_python_configure = "nixpkgs_python_configure", ) load( diff --git a/toolchains/python/BUILD.bazel b/toolchains/python/BUILD.bazel index 6c9dfd46f..f66f4c83b 100644 --- a/toolchains/python/BUILD.bazel +++ b/toolchains/python/BUILD.bazel @@ -2,4 +2,10 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "python.bzl", -]) \ No newline at end of file +]) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) diff --git a/toolchains/python/WORKSPACE b/toolchains/python/WORKSPACE new file mode 100644 index 000000000..2bda73373 --- /dev/null +++ b/toolchains/python/WORKSPACE @@ -0,0 +1,2 @@ +# only temporary for compatibility with `WORKSPACE` setup +# TODO: remove when migration to `bzlmod` is complete From 22dbd3c808121297cd318c548e40a5174051efb6 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Wed, 9 Mar 2022 13:51:02 +0100 Subject: [PATCH 16/25] fix sandboxed integration test requires exposing `:srcs` for all the toolchain modules, so they can be added as `runfiles` to the sandbox. --- core/BUILD.bazel | 6 +++ nixpkgs/repositories.bzl | 48 +++++-------------- tests/BUILD.bazel | 13 +++-- tests/invalid_nixpkgs_package/workspace.bazel | 24 +++++++++- toolchains/cc/BUILD.bazel | 6 +++ toolchains/java/BUILD.bazel | 8 +++- 6 files changed, 62 insertions(+), 43 deletions(-) diff --git a/core/BUILD.bazel b/core/BUILD.bazel index 7158bd251..eac735fd9 100644 --- a/core/BUILD.bazel +++ b/core/BUILD.bazel @@ -6,6 +6,12 @@ exports_files([ "nixpkgs.bzl", ]) +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) + bzl_library( name = "bazel_tools", srcs = [ diff --git a/nixpkgs/repositories.bzl b/nixpkgs/repositories.bzl index fababecb4..d00819a17 100644 --- a/nixpkgs/repositories.bzl +++ b/nixpkgs/repositories.bzl @@ -33,39 +33,17 @@ def rules_nixpkgs_dependencies(local = None): url = "https://github.com/bazelbuild/rules_java/releases/download/4.0.0/rules_java-4.0.0.tar.gz", sha256 = "34b41ec683e67253043ab1a3d1e8b7c61e4e8edefbcad485381328c934d072fe", ) - url = "https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.8.1.tar.gz" - if not local: - # XXX: no way to use `sha256` here, but if this surrounding repo comes - # from that URL, Bazel should hit the cache for the sub-workspaces - maybe( - http_archive, - "rules_nixpkgs_core", - url = url, - strip_prefix = "core", - ) - maybe( - http_archive, - "rules_nixpkgs_cc", - url = url, - strip_prefix = "toolchains/cc", - ) - maybe( - http_archive, - "rules_nixpkgs_java", - url = url, - strip_prefix = "toolchains/java", - ) - else: - native.local_repository( - name = "rules_nixpkgs_core", - path = local + "/core", - ) - native.local_repository( - name = "rules_nixpkgs_cc", - path = local + "/toolchains/cc", - ) - native.local_repository( - name = "rules_nixpkgs_java", - path = local + "/toolchains/java", - ) + url = "https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.8.1.tar.gz" + for repo, prefix in [ + ("rules_nixpkgs_core", "core"), + ("rules_nixpkgs_cc", "toolchains/cc"), + ("rules_nixpkgs_java", "toolchains/java"), + ("rules_nixpkgs_python", "toolchains/python"), + ]: + if not local: + # XXX: no way to use `sha256` here, but if this surrounding repo comes + # from that URL, Bazel should hit the cache for the sub-workspaces + maybe(http_archive, repo, url = url, strip_prefix = prefix) + else: + maybe(native.local_repository, repo, path = "{}/{}".format(local, prefix)) diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 837a93051..beda66f4d 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,11 +1,11 @@ -package(default_testonly = 1) - load("@io_bazel_rules_go//go:def.bzl", "go_binary") -load("@rules_java//java:defs.bzl", "java_binary", "java_test") +load("@rules_java//java:defs.bzl", "java_test") load(":cc-test.bzl", "cc_toolchain_test") load(":java-test.bzl", "java_runtime_test") load(":location_expansion_unit_test.bzl", "expand_location_unit_test_suite") +package(default_testonly = 1) + expand_location_unit_test_suite() [ @@ -97,8 +97,8 @@ cc_toolchain_test( # Test nixpkgs_java_configure() by building some Java code. java_test( name = "java-test", - test_class = "JavaTest", srcs = ["JavaTest.java"], + test_class = "JavaTest", ) # Test that nixpkgs_java_runtime is selected. @@ -142,7 +142,10 @@ sh_test( data = [ "//nixpkgs:srcs", "//tests/invalid_nixpkgs_package:srcs", - "@rules_nixpkgs_cc//:cc.bzl", + "@rules_nixpkgs_core//:srcs", + "@rules_nixpkgs_cc//:srcs", + "@rules_nixpkgs_java//:srcs", + "@rules_nixpkgs_python//:srcs", "@nix-unstable//:bin", ] + select({ "@platforms//os:linux": ["@busybox_static//:bin"], diff --git a/tests/invalid_nixpkgs_package/workspace.bazel b/tests/invalid_nixpkgs_package/workspace.bazel index 2113a7733..a6aa7aa5e 100644 --- a/tests/invalid_nixpkgs_package/workspace.bazel +++ b/tests/invalid_nixpkgs_package/workspace.bazel @@ -2,6 +2,19 @@ workspace(name = "io_tweag_rules_nixpkgs") load("//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") +# fetch external repositories from runfiles directory. this does not work +# otherwise, as the test runs in a sandbox and is not aware of the actual +# directory where the original code lives +[local_repository( + name = repo, + path = "external/{}".format(repo), +) for repo in [ + "rules_nixpkgs_core", + "rules_nixpkgs_cc", + "rules_nixpkgs_java", + "rules_nixpkgs_python", +]] + rules_nixpkgs_dependencies() load( @@ -13,12 +26,19 @@ load( nixpkgs_local_repository( name = "nixpkgs", nix_file = "//:default.nix", - nix_file_deps = [ "//:message.nix" ], + nix_file_deps = ["//:message.nix"], ) nixpkgs_package( name = "hello", attribute_path = "hello", + nixopts = [ + "--option", + "sandbox", + "false", + "--option", + "binary-caches", + "", + ], repository = "@nixpkgs", - nixopts = [ "--option", "sandbox", "false", "--option", "binary-caches", "" ] ) diff --git a/toolchains/cc/BUILD.bazel b/toolchains/cc/BUILD.bazel index e7abf3300..04e6915d0 100644 --- a/toolchains/cc/BUILD.bazel +++ b/toolchains/cc/BUILD.bazel @@ -5,3 +5,9 @@ exports_files([ "foreign_cc.bzl", "cc.nix", ]) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) diff --git a/toolchains/java/BUILD.bazel b/toolchains/java/BUILD.bazel index d7e2b0b13..ea96c23c8 100644 --- a/toolchains/java/BUILD.bazel +++ b/toolchains/java/BUILD.bazel @@ -2,4 +2,10 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "java.bzl", -]) \ No newline at end of file +]) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) From add02d179d9b3452d1500b20e44f848615d55553 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Thu, 10 Mar 2022 11:10:51 +0100 Subject: [PATCH 17/25] move constraints definitions into `core` keep aliases for backwards compatibility --- core/constraints/BUILD.bazel | 8 ++++++++ core/platforms/BUILD.bazel | 6 ++++++ core/util.bzl | 2 +- nixpkgs/constraints/BUILD.bazel | 9 ++++++--- nixpkgs/platforms/BUILD.bazel | 5 ++--- 5 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 core/constraints/BUILD.bazel create mode 100644 core/platforms/BUILD.bazel diff --git a/core/constraints/BUILD.bazel b/core/constraints/BUILD.bazel new file mode 100644 index 000000000..c08fbb44b --- /dev/null +++ b/core/constraints/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//visibility:public"]) + +constraint_setting(name = "nix") + +constraint_value( + name = "support_nix", + constraint_setting = ":nix", +) diff --git a/core/platforms/BUILD.bazel b/core/platforms/BUILD.bazel new file mode 100644 index 000000000..c4d0fab84 --- /dev/null +++ b/core/platforms/BUILD.bazel @@ -0,0 +1,6 @@ +platform( + name = "host", + constraint_values = ["@rules_nixpkgs_core//constraints:support_nix"], + parents = ["@local_config_platform//:host"], + visibility = ["//visibility:public"], +) diff --git a/core/util.bzl b/core/util.bzl index 9f1df6e1c..a40aa6082 100644 --- a/core/util.bzl +++ b/core/util.bzl @@ -118,7 +118,7 @@ def ensure_constraints(repository_ctx): else: target_constraints = list(repository_ctx.attr.target_constraints) exec_constraints = list(repository_ctx.attr.exec_constraints) - exec_constraints.append("@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix") + exec_constraints.append("@rules_nixpkgs_core//constraints:support_nix") return exec_constraints, target_constraints def parse_expand_location(string): diff --git a/nixpkgs/constraints/BUILD.bazel b/nixpkgs/constraints/BUILD.bazel index 94678bda9..05faae341 100644 --- a/nixpkgs/constraints/BUILD.bazel +++ b/nixpkgs/constraints/BUILD.bazel @@ -1,10 +1,13 @@ package(default_visibility = ["//visibility:public"]) -constraint_setting(name = "nix") +alias( + name = "nix", + actual = "@rules_nixpkgs_core//constraints:nix", +) -constraint_value( +alias( name = "support_nix", - constraint_setting = ":nix", + actual = "@rules_nixpkgs_core//constraints:support_nix", ) constraint_value( diff --git a/nixpkgs/platforms/BUILD.bazel b/nixpkgs/platforms/BUILD.bazel index d574b24c6..92b46e33c 100644 --- a/nixpkgs/platforms/BUILD.bazel +++ b/nixpkgs/platforms/BUILD.bazel @@ -1,6 +1,5 @@ -platform( +alias( name = "host", - constraint_values = ["@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix"], - parents = ["@local_config_platform//:host"], + actual = "@rules_nixpkgs_core//platforms:host", visibility = ["//visibility:public"], ) From 5819e1df3764d24e3ccf654783eae455cae8699e Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Thu, 10 Mar 2022 11:56:20 +0100 Subject: [PATCH 18/25] add module dependencies where possible --- toolchains/cc/MODULE.bazel | 4 +++- toolchains/go/MODULE.bazel | 6 +++++- toolchains/java/MODULE.bazel | 1 + toolchains/python/MODULE.bazel | 1 + toolchains/rust/MODULE.bazel | 5 +++++ 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/toolchains/cc/MODULE.bazel b/toolchains/cc/MODULE.bazel index a6dc3e21c..3908059ee 100644 --- a/toolchains/cc/MODULE.bazel +++ b/toolchains/cc/MODULE.bazel @@ -1,6 +1,8 @@ module( - name = "rules_nixpkgs_python", + name = "rules_nixpkgs_cc", version = "0.8.1", ) bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") +bazel_dep(name = "bazel_skylib", version = "1.0.3") +bazel_dep(name = "rules_cc", version = "0.0.1") diff --git a/toolchains/go/MODULE.bazel b/toolchains/go/MODULE.bazel index d03156951..eb3862b6e 100644 --- a/toolchains/go/MODULE.bazel +++ b/toolchains/go/MODULE.bazel @@ -4,4 +4,8 @@ module( ) bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") -bazel_dep(name = "io_bazel_rules_go", repo_name = "rules_go", version = "0.26.0") \ No newline at end of file +# TODO: there is no BCR entry for `rules_go` yet, and you will have to add a +# local registry entry to map a commit to a module "version". the caller will +# also have to know this and point `--registry` at the file from right revision +# on GitHub! +# bazel_dep(name = "rules_go", repo_name = "rules_go", version = "0.26.0") \ No newline at end of file diff --git a/toolchains/java/MODULE.bazel b/toolchains/java/MODULE.bazel index 784c3eb40..e31306551 100644 --- a/toolchains/java/MODULE.bazel +++ b/toolchains/java/MODULE.bazel @@ -4,3 +4,4 @@ module( ) bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") +bazel_dep(name = "rules_java", version = "4.0.0") diff --git a/toolchains/python/MODULE.bazel b/toolchains/python/MODULE.bazel index a6dc3e21c..3c8345f5e 100644 --- a/toolchains/python/MODULE.bazel +++ b/toolchains/python/MODULE.bazel @@ -4,3 +4,4 @@ module( ) bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") +bazel_dep(name = "bazel_skylib", version = "1.0.3") diff --git a/toolchains/rust/MODULE.bazel b/toolchains/rust/MODULE.bazel index 582fd610c..fa4901e9c 100644 --- a/toolchains/rust/MODULE.bazel +++ b/toolchains/rust/MODULE.bazel @@ -4,3 +4,8 @@ module( ) bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") +# TODO: there is no BCR entry for `rules_go` yet, and you will have to add a +# local registry entry to map a commit to a module "version". the caller will +# also have to know this and point `--registry` at the file from right revision +# on GitHub! +# bazel_dep(name = "rules_rust", version = "9dadbcd1136f7d4f3f2e7c0790531be0fdcccc535dec42a4c5a6f2df7380e3e3") From a792c4391090adf986cf6d785719bd974c74c0ae Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Thu, 10 Mar 2022 12:59:01 +0100 Subject: [PATCH 19/25] extract POSIX toolchain into module --- nixpkgs/BUILD.bazel | 1 + nixpkgs/nixpkgs.bzl | 157 +----------------- nixpkgs/repositories.bzl | 1 + tests/BUILD.bazel | 1 + tests/invalid_nixpkgs_package/workspace.bazel | 1 + toolchains/posix/BUILD.bazel | 11 ++ toolchains/posix/MODULE.bazel | 7 + toolchains/posix/WORKSPACE | 2 + toolchains/posix/posix.bzl | 157 ++++++++++++++++++ 9 files changed, 186 insertions(+), 152 deletions(-) create mode 100644 toolchains/posix/BUILD.bazel create mode 100644 toolchains/posix/MODULE.bazel create mode 100644 toolchains/posix/WORKSPACE create mode 100644 toolchains/posix/posix.bzl diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 8aedcbe72..ab0bf9bca 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -44,6 +44,7 @@ bzl_library( "@rules_nixpkgs_cc//:cc.bzl", "@rules_nixpkgs_java//:java.bzl", "@rules_nixpkgs_python//:python.bzl", + "@rules_nixpkgs_posix//:posix.bzl", ":bazel_tools", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 6fadcf584..e66c60e8c 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -29,6 +29,10 @@ load( "@rules_nixpkgs_cc//:cc.bzl", _nixpkgs_cc_configure = "nixpkgs_cc_configure", ) +load( + "@rules_nixpkgs_posix//:posix.bzl", + _nixpkgs_sh_posix_configure = "nixpkgs_sh_posix_configure", +) # aliases for backwards compatibility prior to `bzlmod` nixpkgs_git_repository = _nixpkgs_git_repository @@ -37,6 +41,7 @@ nixpkgs_package = _nixpkgs_package nixpkgs_python_configure = _nixpkgs_python_configure nixpkgs_java_configure = _nixpkgs_java_configure nixpkgs_cc_configure = _nixpkgs_cc_configure +nixpkgs_sh_posix_configure = _nixpkgs_sh_posix_configure def nixpkgs_cc_autoconf_impl(repository_ctx): cpu_value = get_cpu_value(repository_ctx) @@ -178,155 +183,3 @@ def nixpkgs_cc_configure_deprecated( nixpkgs_cc_autoconf(name = "local_config_cc") native.bind(name = "cc_toolchain", actual = "@local_config_cc//:toolchain") native.register_toolchains("@local_config_cc//:all") - -def nixpkgs_sh_posix_config(name, packages, **kwargs): - nixpkgs_package( - name = name, - nix_file_content = """ -with import {{ config = {{}}; overlays = []; }}; - -let - # `packages` might include lists, e.g. `stdenv.initialPath` is a list itself, - # so we need to flatten `packages`. - flatten = builtins.concatMap (x: if builtins.isList x then x else [x]); - env = buildEnv {{ - name = "posix-toolchain"; - paths = flatten [ {} ]; - }}; - cmd_glob = "${{env}}/bin/*"; - os = if stdenv.isDarwin then "osx" else "linux"; -in - -runCommand "bazel-nixpkgs-posix-toolchain" - {{ executable = false; - # Pointless to do this on a remote machine. - preferLocalBuild = true; - allowSubstitutes = false; - }} - '' - n=$out/nixpkgs_sh_posix.bzl - mkdir -p "$(dirname "$n")" - - cat >>$n <>$n - fi - done - cat >>$n < {{ config = {{}}; overlays = []; }}; + +let + # `packages` might include lists, e.g. `stdenv.initialPath` is a list itself, + # so we need to flatten `packages`. + flatten = builtins.concatMap (x: if builtins.isList x then x else [x]); + env = buildEnv {{ + name = "posix-toolchain"; + paths = flatten [ {} ]; + }}; + cmd_glob = "${{env}}/bin/*"; + os = if stdenv.isDarwin then "osx" else "linux"; +in + +runCommand "bazel-nixpkgs-posix-toolchain" + {{ executable = false; + # Pointless to do this on a remote machine. + preferLocalBuild = true; + allowSubstitutes = false; + }} + '' + n=$out/nixpkgs_sh_posix.bzl + mkdir -p "$(dirname "$n")" + + cat >>$n <>$n + fi + done + cat >>$n < Date: Thu, 10 Mar 2022 13:31:25 +0100 Subject: [PATCH 20/25] move `BUILD.pkg` template to `core` it belongs there and is used only there --- nixpkgs/BUILD.pkg => core/BUILD.bazel.tpl | 0 core/nixpkgs.bzl | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename nixpkgs/BUILD.pkg => core/BUILD.bazel.tpl (100%) diff --git a/nixpkgs/BUILD.pkg b/core/BUILD.bazel.tpl similarity index 100% rename from nixpkgs/BUILD.pkg rename to core/BUILD.bazel.tpl diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl index 847542afb..c9825695b 100644 --- a/core/nixpkgs.bzl +++ b/core/nixpkgs.bzl @@ -239,7 +239,7 @@ def _nixpkgs_package_impl(repository_ctx): if create_build_file_if_needed: p = repository_ctx.path("BUILD") if not p.exists: - repository_ctx.template("BUILD", Label("@io_tweag_rules_nixpkgs//nixpkgs:BUILD.pkg")) + repository_ctx.template("BUILD", Label("@rules_nixpkgs_core//:BUILD.bazel.tpl")) _nixpkgs_package = repository_rule( implementation = _nixpkgs_package_impl, From d06eafc42b5c365dccef13b7e04416d09d3491c0 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 14 Mar 2022 09:09:14 +0100 Subject: [PATCH 21/25] fix comment typo --- toolchains/rust/MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchains/rust/MODULE.bazel b/toolchains/rust/MODULE.bazel index fa4901e9c..299663fdf 100644 --- a/toolchains/rust/MODULE.bazel +++ b/toolchains/rust/MODULE.bazel @@ -4,7 +4,7 @@ module( ) bazel_dep(name = "rules_nixpkgs_core", version = "0.8.1") -# TODO: there is no BCR entry for `rules_go` yet, and you will have to add a +# TODO: there is no BCR entry for `rules_rust` yet, and you will have to add a # local registry entry to map a commit to a module "version". the caller will # also have to know this and point `--registry` at the file from right revision # on GitHub! From 885708afd4c4724b093dc13cd2eead4dd7c9eeec Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Mon, 14 Mar 2022 12:41:31 +0100 Subject: [PATCH 22/25] generalize scripts to generate documentation the boilerplate for accessing runfiles turned out to be not needed. from now on one can just add targets or filenames for generating documentation and checking that the generated files are up to date under version control. --- docs/BUILD.bazel | 25 +++++++++++++++---------- docs/check-readme.sh | 25 ------------------------- docs/compare-files.sh | 12 ++++++++++++ docs/copy-files.sh | 7 +++++++ docs/stardoc.bzl | 26 ++++++++++++++++++++++++++ docs/update-readme.sh | 24 ------------------------ 6 files changed, 60 insertions(+), 59 deletions(-) delete mode 100755 docs/check-readme.sh create mode 100755 docs/compare-files.sh create mode 100755 docs/copy-files.sh delete mode 100755 docs/update-readme.sh diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 2de42ef43..e471d69d9 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -1,4 +1,4 @@ -load("stardoc.bzl", "stardoc") +load("stardoc.bzl", "compare_files", "copy_files", "stardoc") stardoc( name = "nixpkgs", @@ -93,19 +93,24 @@ genrule( toolchains = ["@rules_sh//sh/posix:make_variables"], ) -sh_test( +compare_files( name = "check-readme", - srcs = ["check-readme.sh"], data = [ - "README.md", - "//:README.md", + ("README.md", "//:README.md"), + ("core/README.md", "@rules_nixpkgs_core//:README.md"), ], - deps = ["@bazel_tools//tools/bash/runfiles"], + error_message = """ + The project README is not up-to-date. + Please update it using the following command. + + bazel run //docs:update-readme + """, ) -sh_binary( +copy_files( name = "update-readme", - srcs = ["update-readme.sh"], - data = ["README.md", "core/README.md"], - deps = ["@bazel_tools//tools/bash/runfiles"], + data = [ + "README.md", + "core/README.md", + ], ) diff --git a/docs/check-readme.sh b/docs/check-readme.sh deleted file mode 100755 index 76d56d539..000000000 --- a/docs/check-readme.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Copy-pasted from the Bazel Bash runfiles library v2. -set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash -source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ - source "$0.runfiles/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e -# --- end runfiles.bash initialization v2 --- -set -euo pipefail - -old="$(rlocation io_tweag_rules_nixpkgs/README.md)" -new="$(rlocation io_tweag_rules_nixpkgs/docs/README.md)" - -if ! cmp -s "$old" "$new"; then - cat >&2 </dev/null || \ - source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ - source "$0.runfiles/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e -# --- end runfiles.bash initialization v2 --- -set -euo pipefail - -new="$(rlocation io_tweag_rules_nixpkgs/docs/README.md)" -new2="$(rlocation io_tweag_rules_nixpkgs/docs/core/README.md)" - - -# this variable is set by `bazel run` -if [ -z "$BUILD_WORKSPACE_DIRECTORY" ]; then - echo "This script must be executed using bazel run." >&2 - exit 1 -fi - -cp "$new" "$BUILD_WORKSPACE_DIRECTORY/README.md" -cp "$new2" "$BUILD_WORKSPACE_DIRECTORY/core/README.md" From 29bae9ac39aa83b11acf69627ccfc5b019fd1e1b Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Tue, 15 Mar 2022 10:15:29 +0100 Subject: [PATCH 23/25] import sub-repositories based on original import type while incomplete, this should cover most existing use cases seamlessly, such that existing users would not notice the refactoring. --- WORKSPACE | 2 +- examples/toolchains/cc/WORKSPACE | 2 +- examples/toolchains/cc_with_deps/WORKSPACE | 3 +- examples/toolchains/go/WORKSPACE | 3 +- examples/toolchains/java/WORKSPACE | 3 +- examples/toolchains/python/WORKSPACE | 3 +- examples/toolchains/rust/WORKSPACE | 3 +- nixpkgs/repositories.bzl | 61 ++++++++++++++++++---- 8 files changed, 57 insertions(+), 23 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index c5b9c387a..da8457af3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,7 +3,7 @@ workspace(name = "io_tweag_rules_nixpkgs") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies(local=".") +rules_nixpkgs_dependencies() load( "//nixpkgs:nixpkgs.bzl", diff --git a/examples/toolchains/cc/WORKSPACE b/examples/toolchains/cc/WORKSPACE index dd67de6d7..2e3d750f3 100644 --- a/examples/toolchains/cc/WORKSPACE +++ b/examples/toolchains/cc/WORKSPACE @@ -16,7 +16,7 @@ http_archive( load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -rules_nixpkgs_dependencies(local="../../..") +rules_nixpkgs_dependencies() load( "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", diff --git a/examples/toolchains/cc_with_deps/WORKSPACE b/examples/toolchains/cc_with_deps/WORKSPACE index d5ec27954..7f9588dea 100644 --- a/examples/toolchains/cc_with_deps/WORKSPACE +++ b/examples/toolchains/cc_with_deps/WORKSPACE @@ -6,8 +6,7 @@ local_repository( load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -# Remove `local=...` -rules_nixpkgs_dependencies(local="../../..") +rules_nixpkgs_dependencies() load( "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", diff --git a/examples/toolchains/go/WORKSPACE b/examples/toolchains/go/WORKSPACE index ebeb7ef77..4a7b625b7 100644 --- a/examples/toolchains/go/WORKSPACE +++ b/examples/toolchains/go/WORKSPACE @@ -21,8 +21,7 @@ go_rules_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -# Remove `local=...` -rules_nixpkgs_dependencies(local="../../..") +rules_nixpkgs_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/examples/toolchains/java/WORKSPACE b/examples/toolchains/java/WORKSPACE index e30a9720d..7c0d30f10 100644 --- a/examples/toolchains/java/WORKSPACE +++ b/examples/toolchains/java/WORKSPACE @@ -19,8 +19,7 @@ rules_java_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -# Remove `local=...` -rules_nixpkgs_dependencies(local="../../..") +rules_nixpkgs_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/examples/toolchains/python/WORKSPACE b/examples/toolchains/python/WORKSPACE index 03731b7b8..10e6b4df5 100644 --- a/examples/toolchains/python/WORKSPACE +++ b/examples/toolchains/python/WORKSPACE @@ -7,8 +7,7 @@ local_repository( load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -# Remove `local=...` -rules_nixpkgs_dependencies(local="../../..") +rules_nixpkgs_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/examples/toolchains/rust/WORKSPACE b/examples/toolchains/rust/WORKSPACE index c42f95617..1f54ebb61 100644 --- a/examples/toolchains/rust/WORKSPACE +++ b/examples/toolchains/rust/WORKSPACE @@ -20,8 +20,7 @@ http_archive( load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies") -# Remove `local=...` -rules_nixpkgs_dependencies(local="../../..") +rules_nixpkgs_dependencies() load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository") diff --git a/nixpkgs/repositories.bzl b/nixpkgs/repositories.bzl index 319f5f8ca..8b045ab4a 100644 --- a/nixpkgs/repositories.bzl +++ b/nixpkgs/repositories.bzl @@ -1,13 +1,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -def rules_nixpkgs_dependencies(local = None): +def rules_nixpkgs_dependencies(rules_nixpkgs_name = "io_tweag_rules_nixpkgs"): """Load repositories required by rules_nixpkgs. Args: - local: path to local `rules_nixpkgs` repository. - use for testing and CI. - TODO: remove when migration to `bzlmod` is complete. + rules_nixpkgs_name: name under which this repository is known in your workspace """ maybe( http_archive, @@ -34,17 +33,57 @@ def rules_nixpkgs_dependencies(local = None): sha256 = "34b41ec683e67253043ab1a3d1e8b7c61e4e8edefbcad485381328c934d072fe", ) - url = "https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.8.1.tar.gz" - for repo, prefix in [ + # the following complication is due to migrating to `bzlmod`. + # fetch extracted submodules as external repositories from an existing source tree, based on the import type. + rules_nixpkgs = native.existing_rule(rules_nixpkgs_name) + if not rules_nixpkgs: + errormsg = [ + "External repository `rules_nixpkgs` not found as `{}`.".format(rules_nixpkgs_name), + "Specify `rules_nixpkgs_dependencies(rules_nixpkgs_name=)`", + "with `` as used for importing `rules_nixpkgs`.", + ] + fail("\n".join(errormsg)) + kind = rules_nixpkgs.get("kind") + for name, prefix in [ ("rules_nixpkgs_core", "core"), ("rules_nixpkgs_cc", "toolchains/cc"), ("rules_nixpkgs_java", "toolchains/java"), ("rules_nixpkgs_python", "toolchains/python"), ("rules_nixpkgs_posix", "toolchains/posix"), ]: - if not local: - # XXX: no way to use `sha256` here, but if this surrounding repo comes - # from that URL, Bazel should hit the cache for the sub-workspaces - maybe(http_archive, repo, url = url, strip_prefix = prefix) + # case analysis in inner loop to reduce code duplication + if kind == "local_repository": + path = rules_nixpkgs.get("path") + maybe(native.local_repository, name, path = "{}/{}".format(path, prefix)) + elif kind == "http_archive": + maybe( + http_archive, + name, + strip_prefix = prefix, + # there may be more attributes needed. please submit a pull request to support your use case. + url = rules_nixpkgs.get("url"), + urls = rules_nixpkgs.get("urls"), + sha256 = rules_nixpkgs.get("sha256"), + ) + elif kind == "git_repository": + maybe( + git_repository, + name, + strip_prefix = prefix, + # there may be more attributes needed. please submit a pull request to support your use case. + remote = rules_nixpkgs.get("remote"), + commit = rules_nixpkgs.get("commit"), + branch = rules_nixpkgs.get("branch"), + tag = rules_nixpkgs.get("tag"), + shallow_since = rules_nixpkgs.get("shallow_since"), + ) else: - maybe(native.local_repository, repo, path = "{}/{}".format(local, prefix)) + errormsg = [ + "Could not find any import type for `rules_nixpkgs`.", + "This should not happen. If you encounter this using the latest release", + "of `rules_nixpkgs`, please file an issue describing your use case:", + "https://github.com/tweag/rules_nixpkgs/issues", + "or submit a pull request with corrections:", + "https://github.com/tweag/rules_nixpkgs/pulls", + ] + fail("\n".join(errormsg)) From 610d6ebf1e335462bd68a50f7f7c6abbcc54a3e1 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Tue, 15 Mar 2022 12:11:13 +0100 Subject: [PATCH 24/25] prepare reference docs generation for each submodule this requires some more cleanup and refactoring before it is readable again, see TODOs. --- docs/BUILD.bazel | 58 +++++++++++++++++++++++++++++++++++-- docs/README_cc.md.tpl | 5 ++++ docs/README_go.md.tpl | 5 ++++ docs/README_java.md.tpl | 5 ++++ docs/README_posix.md.tpl | 5 ++++ docs/README_python.tpl | 5 ++++ docs/README_rust.md.tpl | 5 ++++ toolchains/java/BUILD.bazel | 19 ++++++++++++ toolchains/java/README.md | 0 9 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 docs/README_cc.md.tpl create mode 100644 docs/README_go.md.tpl create mode 100644 docs/README_java.md.tpl create mode 100644 docs/README_posix.md.tpl create mode 100644 docs/README_python.tpl create mode 100644 docs/README_rust.md.tpl create mode 100644 toolchains/java/README.md diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index e471d69d9..56997289b 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -29,6 +29,16 @@ stardoc( deps = ["@rules_nixpkgs_core//:core"], ) +stardoc( + name = "java", + out = "toolchains/java.md", + input = "@rules_nixpkgs_java//:java.bzl", + symbol_names = [ + "nixpkgs_java_configure", + ], + deps = ["@rules_nixpkgs_java//:java"], +) + stardoc( name = "go", out = "toolchains/go.md", @@ -69,6 +79,41 @@ genrule( toolchains = ["@rules_sh//sh/posix:make_variables"], ) +[ + genrule( + name = "readme_{}".format(rule), + srcs = [ + "README_{}.md.tpl".format(rule), + "toolchains/{}.md".format(rule), + ], + outs = ["toolchains/{}/README.md".format(rule)], + # sorry for this mess. Bazel `str.format()` does not allow nested `{}` + # pairs and `awk` is sensitive to line breaks. + # TODO: consider alternative: leave reference for the end and just + # append the generated string with `cat`. + cmd = """$(POSIX_AWK) \\ + <$(execpath README_{}.md.tpl) \\ + >$(OUTS) \\""".format(rule) + """ + '{ + if (/{{""" + "{}".format(rule) + """}}/) { + RS="\\0"; + getline content <"$(execpath """ + "toolchains/{}.md)".format(rule) + """"; + print content; + RS="\\n"; + } else { + print + } + }' + """, + toolchains = ["@rules_sh//sh/posix:make_variables"], + ) + # TODO: add other toolchains once the sub-repositories are appropriately + # prepared xor extract a macro for rules to generate and check documentation + # into `rules_nixpkgs_core`, then call it here for all sub-repositories + # NOTE: don't forget to add to `copy_files` and `compare_files` (good reason to extract a macro) + for rule in ["java"] +] + genrule( name = "readme_core", srcs = [ @@ -98,7 +143,13 @@ compare_files( data = [ ("README.md", "//:README.md"), ("core/README.md", "@rules_nixpkgs_core//:README.md"), - ], + ] + [( + "toolchains/{}/README.md".format(rule), + "@rules_nixpkgs_{}//:README.md".format(rule), + # TODO: add other toolchains once the sub-repositories are appropriately + # prepared xor extract a macro for rules to generate and check documentation + # into `rules_nixpkgs_core`, then call it here for all sub-repositories + ) for rule in ["java"]], error_message = """ The project README is not up-to-date. Please update it using the following command. @@ -112,5 +163,8 @@ copy_files( data = [ "README.md", "core/README.md", - ], + # TODO: add other toolchains once the sub-repositories are appropriately + # prepared xor extract a macro for rules to generate and check documentation + # into `rules_nixpkgs_core`, then call it here for all sub-repositories + ] + ["toolchains/{}/README.md".format(rule) for rule in ["java"]], ) diff --git a/docs/README_cc.md.tpl b/docs/README_cc.md.tpl new file mode 100644 index 000000000..46168e743 --- /dev/null +++ b/docs/README_cc.md.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for C/C++ with Bazel + +## Rules + +{{cc}} \ No newline at end of file diff --git a/docs/README_go.md.tpl b/docs/README_go.md.tpl new file mode 100644 index 000000000..c3ea9e5fa --- /dev/null +++ b/docs/README_go.md.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for Go with Bazel + +## Rules + +{{go}} \ No newline at end of file diff --git a/docs/README_java.md.tpl b/docs/README_java.md.tpl new file mode 100644 index 000000000..e56d2cbd4 --- /dev/null +++ b/docs/README_java.md.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for Java with Bazel + +## Rules + +{{java}} \ No newline at end of file diff --git a/docs/README_posix.md.tpl b/docs/README_posix.md.tpl new file mode 100644 index 000000000..5978eca03 --- /dev/null +++ b/docs/README_posix.md.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for POSIX shell Bazel + +## Rules + +{{posix}} \ No newline at end of file diff --git a/docs/README_python.tpl b/docs/README_python.tpl new file mode 100644 index 000000000..d72e5b5eb --- /dev/null +++ b/docs/README_python.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for Python with Bazel + +## Rules + +{{python}} \ No newline at end of file diff --git a/docs/README_rust.md.tpl b/docs/README_rust.md.tpl new file mode 100644 index 000000000..1894e08df --- /dev/null +++ b/docs/README_rust.md.tpl @@ -0,0 +1,5 @@ +# Nixpkgs for Rust with Bazel + +## Rules + +{{rust}} \ No newline at end of file diff --git a/toolchains/java/BUILD.bazel b/toolchains/java/BUILD.bazel index ea96c23c8..b8de8bba8 100644 --- a/toolchains/java/BUILD.bazel +++ b/toolchains/java/BUILD.bazel @@ -1,3 +1,5 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + package(default_visibility = ["//visibility:public"]) exports_files([ @@ -9,3 +11,20 @@ filegroup( srcs = glob(["**"]), visibility = ["//visibility:public"], ) + +bzl_library( + name = "bazel_tools", + srcs = [ + "@bazel_tools//tools:bzl_srcs", + ], +) + +bzl_library( + name = "java", + srcs = [":srcs"], + visibility = ["//visibility:public"], + deps = [ + ":bazel_tools", + "@rules_nixpkgs_core//:core", + ], +) diff --git a/toolchains/java/README.md b/toolchains/java/README.md new file mode 100644 index 000000000..e69de29bb From b4967da9f979bb9802d1a70e80ba3e8afae1d20b Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Tue, 15 Mar 2022 12:12:45 +0100 Subject: [PATCH 25/25] add rendered documentation for Java --- toolchains/java/README.md | 308 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) diff --git a/toolchains/java/README.md b/toolchains/java/README.md index e69de29bb..5c3861e9c 100644 --- a/toolchains/java/README.md +++ b/toolchains/java/README.md @@ -0,0 +1,308 @@ +# Nixpkgs for Java with Bazel + +## Rules + + + +Rules for importing a Java toolchain from Nixpkgs. + + + + +### nixpkgs_java_configure + +
+nixpkgs_java_configure(name, attribute_path, java_home_path, repository, repositories, nix_file,
+                       nix_file_content, nix_file_deps, nixopts, fail_not_supported, quiet, toolchain,
+                       toolchain_name, toolchain_version, exec_constraints, target_constraints)
+
+ +Define a Java runtime provided by nixpkgs. + +Creates a `nixpkgs_package` for a `java_runtime` instance. Optionally, +you can also create & register a Java toolchain. This only works with Bazel >= 5.0 +Bazel can use this instance to run JVM binaries and tests, refer to the +[Bazel documentation](https://docs.bazel.build/versions/4.0.0/bazel-and-java.html#configuring-the-jdk) for details. + +#### Example + +##### Bazel 4 + +Add the following to your `WORKSPACE` file to import a JDK from nixpkgs: +```bzl +load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_java_configure") +nixpkgs_java_configure( + attribute_path = "jdk11.home", + repository = "@nixpkgs", +) +``` + +Add the following configuration to `.bazelrc` to enable this Java runtime: +``` +build --javabase=@nixpkgs_java_runtime//:runtime +build --host_javabase=@nixpkgs_java_runtime//:runtime +# Adjust this to match the Java version provided by this runtime. +# See `bazel query 'kind(java_toolchain, @bazel_tools//tools/jdk:all)'` for available options. +build --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 +build --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 +``` + +##### Bazel 5 + +Add the following to your `WORKSPACE` file to import a JDK from nixpkgs: +```bzl +load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_java_configure") +nixpkgs_java_configure( + attribute_path = "jdk11.home", + repository = "@nixpkgs", + toolchain = True, + toolchain_name = "nixpkgs_java", + toolchain_version = "11", +) +``` + +Add the following configuration to `.bazelrc` to enable this Java runtime: +``` +build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host +build --java_runtime_version=nixpkgs_java +build --tool_java_runtime_version=nixpkgs_java +``` + + +#### Parameters + + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
name + +optional. +default is "nixpkgs_java_runtime" + +

+ +The name-prefix for the created external repositories. + +

+
attribute_path + +optional. +default is None + +

+ +string, The nixpkgs attribute path for `jdk.home`. + +

+
java_home_path + +optional. +default is "" + +

+ +optional, string, The path to `JAVA_HOME` within the package. + +

+
repository + +optional. +default is None + +

+ +See [`nixpkgs_package`](#nixpkgs_package-repository). + +

+
repositories + +optional. +default is {} + +

+ +See [`nixpkgs_package`](#nixpkgs_package-repositories). + +

+
nix_file + +optional. +default is None + +

+ +optional, Label, Obtain the runtime from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. + +

+
nix_file_content + +optional. +default is "" + +

+ +optional, string, Obtain the runtime from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. + +

+
nix_file_deps + +optional. +default is None + +

+ +See [`nixpkgs_package`](#nixpkgs_package-nix_file_deps). + +

+
nixopts + +optional. +default is [] + +

+ +See [`nixpkgs_package`](#nixpkgs_package-nixopts). + +

+
fail_not_supported + +optional. +default is True + +

+ +See [`nixpkgs_package`](#nixpkgs_package-fail_not_supported). + +

+
quiet + +optional. +default is False + +

+ +See [`nixpkgs_package`](#nixpkgs_package-quiet). + +

+
toolchain + +optional. +default is False + +

+ +Create & register a Bazel toolchain based on the Java runtime. + +

+
toolchain_name + +optional. +default is None + +

+ +The name of the toolchain that can be used in --java_runtime_version. + +

+
toolchain_version + +optional. +default is None + +

+ +The version of the toolchain that can be used in --java_runtime_version. + +

+
exec_constraints + +optional. +default is None + +

+ +Constraints for the execution platform. + +

+
target_constraints + +optional. +default is None + +

+ +Constraints for the target platform. + +

+
+ + +