From d2b4632af666bec06130df0debf0b9e950249780 Mon Sep 17 00:00:00 2001 From: globegitter Date: Mon, 18 Feb 2019 08:49:54 +0100 Subject: [PATCH 01/19] First progress. --- internal/common/os_name.bzl | 12 +++++-- internal/node/node.bzl | 56 +++++++++++++++++++++++------ internal/node/node_labels.bzl | 38 ++++++++++---------- internal/node/node_repositories.bzl | 32 +++++++++-------- 4 files changed, 91 insertions(+), 47 deletions(-) diff --git a/internal/common/os_name.bzl b/internal/common/os_name.bzl index 7b4229fb63..46cc02a14a 100644 --- a/internal/common/os_name.bzl +++ b/internal/common/os_name.bzl @@ -15,6 +15,12 @@ """Helper function for repository rules """ +OS_ARCH_NAMES = [ + "darwin_amd64", + "windows_amd64", + "linux_amd64", +] + def os_name(repository_ctx): """Get the os name for a repository rule @@ -26,10 +32,10 @@ def os_name(repository_ctx): """ os_name = repository_ctx.os.name.lower() if os_name.startswith("mac os"): - return "darwin_amd64" + return OS_ARCH_NAMES[0] elif os_name.find("windows") != -1: - return "windows_amd64" + return OS_ARCH_NAMES[1] elif os_name.startswith("linux"): - return "linux_amd64" + return OS_ARCH_NAMES[2] else: fail("Unsupported operating system: " + os_name) diff --git a/internal/node/node.bzl b/internal/node/node.bzl index ce946334f5..1414f62272 100644 --- a/internal/node/node.bzl +++ b/internal/node/node.bzl @@ -100,7 +100,7 @@ def _short_path_to_manifest_path(ctx, short_path): return ctx.workspace_name + "/" + short_path def _nodejs_binary_impl(ctx): - node = ctx.file.node + # node = ctx.file.node node_modules = ctx.files.node_modules # Using a depset will allow us to avoid flattening files and sources @@ -133,6 +133,38 @@ def _nodejs_binary_impl(ctx): if hasattr(ctx.attr, "expected_exit_code"): expected_exit_code = ctx.attr.expected_exit_code + node_tool_info = ctx.toolchains["@build_bazel_rules_nodejs//toolchains/node:toolchain_type"].nodeinfo + node_tool_files = [] + if node_tool_info.tool_path == "" and not node_tool_info.tool_target: + # If tool_path is empty and tool_target is None then there is no local + # node tool, we will just print a nice error message if the user + # attempts to do bazel run + ctx.actions.write( + content = ("echo node toolchain was not properly configured so %s cannot be executed." % ctx.attr.name), + output = ctx.outputs.script, + ) + else: + node_tool = node_tool_info.tool_path + print(node_tool) + if node_tool_info.tool_target: + node_tool = _short_path_to_manifest_path(ctx, node_tool_info.tool_target.files.to_list()[0].short_path) + # print(node.short_path) + # print(_short_path_to_manifest_path(ctx, node.short_path)) + # print(node_tool.path) + # print(node_tool.short_path) + # print(_short_path_to_manifest_path(ctx, node_tool.short_path)) + # node_tool = _runfiles(ctx, node_tool_info.tool_target.files.to_list()[0]) + node_tool_files += node_tool_info.tool_target.files.to_list() + + print("------------") + # print(node_tool) + # print(node_tool.path) + # print(_short_path_to_manifest_path(ctx, node_tool)) + # print(node.short_path) + # print(_short_path_to_manifest_path(ctx, node.short_path)) + # print("------------") + print("node_tool_files", node_tool_files) + substitutions = { "TEMPLATED_args": " ".join([ expand_location_into_runfiles(ctx, a) @@ -140,7 +172,8 @@ def _nodejs_binary_impl(ctx): ]), "TEMPLATED_env_vars": env_vars, "TEMPLATED_expected_exit_code": str(expected_exit_code), - "TEMPLATED_node": _short_path_to_manifest_path(ctx, node.short_path), + # "TEMPLATED_node": _short_path_to_manifest_path(ctx, node.short_path), + "TEMPLATED_node": node_tool, "TEMPLATED_repository_args": _short_path_to_manifest_path(ctx, ctx.file._repository_args.short_path), "TEMPLATED_script_path": script_path, } @@ -151,14 +184,13 @@ def _nodejs_binary_impl(ctx): is_executable = True, ) - runfiles = depset([node, ctx.outputs.loader, ctx.file._repository_args] + node_modules + ctx.files._node_runfiles, transitive = [sources]) + runfiles = depset(node_tool_files + [ctx.outputs.loader, ctx.file._repository_args] + node_modules + ctx.files._node_runfiles, transitive = [sources]) return [DefaultInfo( executable = ctx.outputs.script, runfiles = ctx.runfiles( transitive_files = runfiles, - files = [ - node, + files = node_tool_files + [ ctx.outputs.loader, ] + ctx.files._source_map_support_files + node_modules + @@ -204,11 +236,11 @@ _NODEJS_EXECUTABLE_ATTRS = { in TypeScript.""", default = True, ), - "node": attr.label( - doc = """The node entry point target.""", - default = Label("@nodejs//:node"), - allow_single_file = True, - ), + # "node": attr.label( + # doc = """The node entry point target.""", + # default = Label("@nodejs//:node"), + # allow_single_file = True, + # ), "node_modules": attr.label( doc = """The npm packages which should be available to `require()` during execution. @@ -290,10 +322,12 @@ _NODEJS_EXECUTABLE_ATTRS = { default = Label("//internal/node:node_loader.js"), allow_single_file = True, ), + # TODO(markus): Remove and get from toolchain "_node_runfiles": attr.label( default = Label("@nodejs//:node_runfiles"), allow_files = True, ), + # TODO(markus): Remove and get from toolchain "_repository_args": attr.label( default = Label("@nodejs//:bin/node_args.sh"), allow_single_file = True, @@ -322,6 +356,7 @@ nodejs_binary = rule( attrs = _NODEJS_EXECUTABLE_ATTRS, executable = True, outputs = _NODEJS_EXECUTABLE_OUTPUTS, + toolchains = ["@build_bazel_rules_nodejs//toolchains/node:toolchain_type"], ) """ Runs some JavaScript code in NodeJS. @@ -337,6 +372,7 @@ nodejs_test = rule( }), test = True, outputs = _NODEJS_EXECUTABLE_OUTPUTS, + toolchains = ["@build_bazel_rules_nodejs//toolchains/node:toolchain_type"], ) """ Identical to `nodejs_binary`, except this can be used with `bazel test` as well. diff --git a/internal/node/node_labels.bzl b/internal/node/node_labels.bzl index 8a8d9e9cf6..cd12de737c 100644 --- a/internal/node/node_labels.bzl +++ b/internal/node/node_labels.bzl @@ -17,37 +17,37 @@ Labels are different on windows and linux/OSX. """ -def get_node_label(repository_ctx): - if repository_ctx.os.name.lower().find("windows") != -1: - label = Label("@nodejs//:bin/node.cmd") +def get_node_label(os_name): + if os_name.find("windows") != -1: + label = Label("@nodejs_%s//:bin/node.cmd" % os_name) else: - label = Label("@nodejs//:bin/node") + label = Label("@nodejs_%s//:bin/node" % os_name) return label def get_npm_label(repository_ctx): - if repository_ctx.os.name.lower().find("windows") != -1: - label = Label("@nodejs//:bin/npm.cmd") + if os_name.find("windows") != -1: + label = Label("@nodejs_%s//:bin/npm.cmd" % os_name) else: - label = Label("@nodejs//:bin/npm") + label = Label("@nodejs_%s//:bin/npm" % os_name) return label -def get_npm_node_repositories_label(repository_ctx): - if repository_ctx.os.name.lower().find("windows") != -1: - label = Label("@nodejs//:bin/npm_node_repositories.cmd") +def get_npm_node_repositories_label(os_name): + if os_name.find("windows") != -1: + label = Label("@nodejs_%s//:bin/npm_node_repositories.cmd" % os_name) else: - label = Label("@nodejs//:bin/npm_node_repositories") + label = Label("@nodejs_%s//:bin/npm_node_repositories" % os_name) return label -def get_yarn_label(repository_ctx): - if repository_ctx.os.name.lower().find("windows") != -1: - label = Label("@nodejs//:bin/yarn.cmd") +def get_yarn_label(os_name): + if os_name.find("windows") != -1: + label = Label("@nodejs_%s//:bin/yarn.cmd" % os_name) else: - label = Label("@nodejs//:bin/yarn") + label = Label("@nodejs_%s//:bin/yarn" % os_name) return label -def get_yarn_node_repositories_label(repository_ctx): - if repository_ctx.os.name.lower().find("windows") != -1: - label = Label("@nodejs//:bin/yarn_node_repositories.cmd") +def get_yarn_node_repositories_label(os_name): + if os_name.find("windows") != -1: + label = Label("@nodejs%s//:bin/yarn_node_repositories.cmd" % os_name) else: - label = Label("@nodejs//:bin/yarn_node_repositories") + label = Label("@nodejs_%s//:bin/yarn_node_repositories" % os_name) return label diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 0c4487f24a..f7a3f955bb 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -19,7 +19,7 @@ See https://docs.bazel.build/versions/master/skylark/repository_rules.html """ load("//internal/common:check_bazel_version.bzl", "check_bazel_version") -load("//internal/common:os_name.bzl", "os_name") +load("//internal/common:os_name.bzl", "os_name", "OS_ARCH_NAMES") load("//internal/npm_install:npm_install.bzl", "yarn_install") load("//third_party/github.com/bazelbuild/bazel-skylib:lib/paths.bzl", "paths") load(":node_labels.bzl", "get_yarn_node_repositories_label") @@ -129,7 +129,8 @@ def _download_node(repository_ctx): if repository_ctx.attr.vendored_node: return - host = os_name(repository_ctx) + # host = os_name(repository_ctx) + host = "darwin_amd64" if "darwin" in repository_ctx.attr.name else "linux_amd64" node_version = repository_ctx.attr.node_version node_repositories = repository_ctx.attr.node_repositories node_urls = repository_ctx.attr.node_urls @@ -543,19 +544,20 @@ def node_repositories( # 0.21.0: repository_ctx.report_progress API check_bazel_version("0.21.0") - _nodejs_repo( - name = "nodejs", - package_json = package_json, - node_version = node_version, - yarn_version = yarn_version, - vendored_node = vendored_node, - vendored_yarn = vendored_yarn, - node_repositories = node_repositories, - yarn_repositories = yarn_repositories, - node_urls = node_urls, - yarn_urls = yarn_urls, - preserve_symlinks = preserve_symlinks, - ) + for os_arch_name in OS_ARCH_NAMES: + _nodejs_repo( + name = "nodejs_%s" % os_arch_name, + package_json = package_json, + node_version = node_version, + yarn_version = yarn_version, + vendored_node = vendored_node, + vendored_yarn = vendored_yarn, + node_repositories = node_repositories, + yarn_repositories = yarn_repositories, + node_urls = node_urls, + yarn_urls = yarn_urls, + preserve_symlinks = preserve_symlinks, + ) _yarn_repo( name = "yarn", From 6a2d932c56400d2766f4524f5bf4b6f5981a76f9 Mon Sep 17 00:00:00 2001 From: globegitter Date: Mon, 18 Feb 2019 08:50:07 +0100 Subject: [PATCH 02/19] Add toolchain files. --- toolchains/node/BUILD | 60 +++++++++ toolchains/node/BUILD.path.tpl | 25 ++++ toolchains/node/BUILD.target.tpl | 25 ++++ toolchains/node/defaults.bzl | 50 ++++++++ toolchains/node/node_configure.bzl | 193 +++++++++++++++++++++++++++++ toolchains/node/node_toolchain.bzl | 67 ++++++++++ 6 files changed, 420 insertions(+) create mode 100644 toolchains/node/BUILD create mode 100644 toolchains/node/BUILD.path.tpl create mode 100644 toolchains/node/BUILD.target.tpl create mode 100644 toolchains/node/defaults.bzl create mode 100644 toolchains/node/node_configure.bzl create mode 100644 toolchains/node/node_toolchain.bzl diff --git a/toolchains/node/BUILD b/toolchains/node/BUILD new file mode 100644 index 0000000000..5d0c270755 --- /dev/null +++ b/toolchains/node/BUILD @@ -0,0 +1,60 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":node_toolchain.bzl", "node_toolchain") + +package(default_visibility = ["//visibility:private"]) + +licenses(["notice"]) # Apache 2.0 + +# node toolchain type +toolchain_type(name = "toolchain_type") + +# Default node toolchain that expects the 'node' executable +# to be in the PATH +# node_toolchain( +# name = "default_node", +# tool_path = "node", +# visibility = ["//visibility:public"], +# ) + +toolchain( + name = "node_linux_toolchain", + target_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "@node_config_linux//:toolchain", + toolchain_type = ":toolchain_type", +) + +toolchain( + name = "node_osx_toolchain", + target_compatible_with = [ + "@bazel_tools//platforms:osx", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "@node_config_osx//:toolchain", + toolchain_type = ":toolchain_type", +) + +toolchain( + name = "node_windows_toolchain", + target_compatible_with = [ + "@bazel_tools//platforms:windows", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "@node_config_windows//:toolchain", + toolchain_type = ":toolchain_type", +) \ No newline at end of file diff --git a/toolchains/node/BUILD.path.tpl b/toolchains/node/BUILD.path.tpl new file mode 100644 index 0000000000..391fdc46bb --- /dev/null +++ b/toolchains/node/BUILD.path.tpl @@ -0,0 +1,25 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This BUILD file is auto-generated from toolchains/kubectl/BUILD.tpl +""" + +package(default_visibility = ["//visibility:public"]) + +load("@build_bazel_rules_nodejs//toolchains/node:node_toolchain.bzl", "node_toolchain") + +node_toolchain( + name = "toolchain", + tool_path = "%{NODE_PATH}", +) \ No newline at end of file diff --git a/toolchains/node/BUILD.target.tpl b/toolchains/node/BUILD.target.tpl new file mode 100644 index 0000000000..919b4d3999 --- /dev/null +++ b/toolchains/node/BUILD.target.tpl @@ -0,0 +1,25 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This BUILD file is auto-generated from toolchains/kubectl/BUILD.tpl +""" + +package(default_visibility = ["//visibility:public"]) + +load("@build_bazel_rules_nodejs//toolchains/node:node_toolchain.bzl", "node_toolchain") + +node_toolchain( + name = "toolchain", + tool_target = "%{NODE_TARGET}", +) \ No newline at end of file diff --git a/toolchains/node/defaults.bzl b/toolchains/node/defaults.bzl new file mode 100644 index 0000000000..3a686293bd --- /dev/null +++ b/toolchains/node/defaults.bzl @@ -0,0 +1,50 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Defines defaults for the kubectl toolchain. +""" + +# The following variables define the kubectl source repository that is pulled +# when the kubectl toolchain is configured to build from source. +# The repository is at https://github.com/kubernetes/kubernetes +# The kubernetes github organization. +k8s_org = "kubernetes" + +# The kubernetes repository. +k8s_repo = "kubernetes" + +# The release commit/tag to for the kubernetes repo. +k8s_commit = "v1.13.0-beta.1" + +# The archive prefix. This is the name of the top level directory in the +# downloaded repository archive tarball. +k8s_prefix = "kubernetes-1.13.0-beta.1" + +# The SHA256 of the k8s repo. +k8s_sha256 = "dfb39ce36284c1ce228954ca12bf016c09be61e40a875e8af4fff84e116bd3a7" + +# The kubernetes repository infrastructure tools repository. +# https://github.com/kubernetes/repo-infra +k8s_repo_tools_repo = "repo-infra" + +# The commit pin to use for the kuebernetes repository infrastructure tools +# repository. +k8s_repo_tools_commit = "b4bc4f1552c7fc1d4654753ca9b0e5e13883429f" + +# The archive prefix. This is the name of the top level directory in the +# downloaded repository archive tarball. +k8s_repo_tools_prefix = "{}-{}".format(k8s_repo_tools_repo, k8s_repo_tools_commit) + +# The SHA256 of the kubernetes repository infrastructure tools repository. +k8s_repo_tools_sha = "21160531ea8a9a4001610223ad815622bf60671d308988c7057168a495a7e2e8" diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl new file mode 100644 index 0000000000..7fc2497b0b --- /dev/null +++ b/toolchains/node/node_configure.bzl @@ -0,0 +1,193 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Defines a repository rule for configuring the node binary. +""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@build_bazel_rules_nodejs//:defs.bzl", "node_repositories") +load( + ":defaults.bzl", + _k8s_commit = "k8s_commit", + _k8s_org = "k8s_org", + _k8s_prefix = "k8s_prefix", + _k8s_repo = "k8s_repo", + _k8s_repo_tools_commit = "k8s_repo_tools_commit", + _k8s_repo_tools_prefix = "k8s_repo_tools_prefix", + _k8s_repo_tools_repo = "k8s_repo_tools_repo", + _k8s_repo_tools_sha = "k8s_repo_tools_sha", + _k8s_sha256 = "k8s_sha256", +) + +def _impl(repository_ctx): + substitutions = None + if repository_ctx.attr.target: + node_target = repository_ctx.attr.target + substitutions = {"%{NODE_TARGET}": "%s" % node_target} + template = Label("@build_bazel_rules_nodejs//toolchains/node:BUILD.target.tpl") + else: + node_tool_path = repository_ctx.attr.local_path or repository_ctx.which("node") or "" + substitutions = {"%{NODE_PATH}": "%s" % node_tool_path} + template = Label("@build_bazel_rules_nodejs//toolchains/node:BUILD.path.tpl") + + repository_ctx.template( + "BUILD", + template, + substitutions, + False, + ) + +_node_configure = repository_rule( + implementation = _impl, + attrs = { + "os": attr.string( + mandatory = True, + doc = "Default target OS", + ), + "arch": attr.string( + mandatory = True, + doc = "Default target architecture", + ), + "target_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the target os.", + mandatory = False, + ), + "target_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the target os.", + mandatory = False, + allow_single_file = True, + ), + "host_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the host os.", + mandatory = False, + ), + "host_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the host os.", + mandatory = False, + allow_single_file = True, + ), + }, +) + +def _ensure_all_provided(func_name, attrs, kwargs): + """ + For function func_name, ensure either all attributes in 'attrs' were + specified in kwargs or none were specified. + """ + any_specified = False + for key in kwargs.keys(): + if key in attrs: + any_specified = True + break + if not any_specified: + return + provided = [] + missing = [] + for attr in attrs: + if attr in kwargs: + provided.append(attr) + else: + missing.append(attr) + if len(missing) != 0: + fail("Attribute(s) {} are required for function {} because attribute(s) {} were specified.".format( + ", ".join(missing), + func_name, + ", ".join(provided), + )) + +def node_configure(name, **kwargs): + """ + Creates an external repository with a node_toolchain target + properly configured. + + Args: + **kwargs: + Required Args + name: A unique name for this rule. + Default Args: + local_node: Optional. + local_path: Optional. Set to true to build kubectl from sources. Default: False. + Can't be specified if kubectl_path is specified. + k8s_commit: Optional. Commit / release tag at which to build kubectl + from. Default is defined as k8s_tag in :defaults.bzl. + k8s_sha256: Optional. sha256 of commit at which to build kubectl from. + Default is defined as k8s_sha256 in :defaults.bzl. + kubectl_path: Optional. Use the kubectl binary at the given path or label. + This can't be used with 'build_srcs'. + Note: Not all versions/commits of kubernetes project can be used to compile + kubectl from an external repo. Notably, we have only tested with v1.13.0-beta.1 + or above. Note this rule has a hardcoded pointer to io_kubernetes_build repo + if your commit (above v1.13.0-beta.1) does not work due to problems, + related to @io_kubernetes_build repo, please send a PR to update these values. + """ + + native.register_toolchains( + "@build_bazel_rules_nodejs//toolchains/node:node_linux_toolchain", + "@build_bazel_rules_nodejs//toolchains/node:node_osx_toolchain", + "@build_bazel_rules_nodejs//toolchains/node:node_windows_toolchain", + ) + + node_repositories(**kwargs) + + _node_configure(name = name + "_linux", os="linux", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_linux//:node") + _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") + + # build_srcs = False + # if "build_srcs" in kwargs and "kubectl_path" in kwargs: + # fail("Attributes 'build_srcs' and 'kubectl_path' can't be specified at" + + # " the same time") + # if "build_srcs" in kwargs and kwargs["build_srcs"]: + # build_srcs = True + # _ensure_all_provided( + # "kubectl_configure", + # ["k8s_commit", "k8s_sha256", "k8s_prefix"], + # kwargs, + # ) + # k8s_commit = kwargs["k8s_commit"] if "k8s_commit" in kwargs else _k8s_commit + # k8s_sha256 = kwargs["k8s_sha256"] if "k8s_sha256" in kwargs else _k8s_sha256 + # k8s_prefix = kwargs["k8s_prefix"] if "k8s_prefix" in kwargs else _k8s_prefix + + # _ensure_all_provided( + # "kubectl_configure", + # ["k8s_repo_tools_sha", "k8s_repo_tools_commit", "k8s_repo_tools_prefix"], + # kwargs, + # ) + # k8s_repo_tools_sha = kwargs["k8s_repo_tools_sha"] if "k8s_repo_tools_sha" in kwargs else _k8s_repo_tools_sha + # k8s_repo_tools_commit = kwargs["k8s_repo_tools_commit"] if "k8s_repo_tools_commit" in kwargs else _k8s_repo_tools_commit + # k8s_repo_tools_prefix = kwargs["k8s_repo_tools_prefix"] if "k8s_repo_tools_prefix" in kwargs else _k8s_repo_tools_prefix + + # http_archive( + # name = "io_kubernetes", + # sha256 = k8s_sha256, + # strip_prefix = k8s_prefix, + # urls = [("https://github.com/{}/{}/archive/{}.tar.gz".format( + # _k8s_org, + # _k8s_repo, + # k8s_commit, + # ))], + # ) + # http_archive( + # name = "io_kubernetes_build", + # sha256 = k8s_repo_tools_sha, + # strip_prefix = k8s_repo_tools_prefix, + # urls = ["https://github.com/{}/{}/archive/{}.tar.gz".format( + # _k8s_org, + # _k8s_repo_tools_repo, + # k8s_repo_tools_commit, + # )], + # ) + # if "kubectl_path" in kwargs: + # _kubectl_configure(name = name, kubectl_path = kwargs["kubectl_path"]) + # else: + # _kubectl_configure(name = name, build_srcs = build_srcs) diff --git a/toolchains/node/node_toolchain.bzl b/toolchains/node/node_toolchain.bzl new file mode 100644 index 0000000000..c4caafdf99 --- /dev/null +++ b/toolchains/node/node_toolchain.bzl @@ -0,0 +1,67 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module implements the node toolchain rule. +""" + +NodeInfo = provider( + doc = "Information about how to invoke the node binary.", + fields = { + "tool_path": "Path to an existing nodejs executable", + "tool_target": "A hermetically downloaded nodejs executable target.", + }, +) + +def _node_toolchain_impl(ctx): + if not ctx.attr.tool_path and not ctx.attr.tool_target: + print("No nodejs binary was found or built, executing run for rules_nodejs targets might not work.") + toolchain_info = platform_common.ToolchainInfo( + nodeinfo = NodeInfo( + tool_path = ctx.attr.tool_path, + tool_target = ctx.attr.tool_target, + ), + ) + return [toolchain_info] + +node_toolchain = rule( + implementation = _node_toolchain_impl, + attrs = { + "os": attr.string( + mandatory = True, + doc = "Default target OS", + ), + "arch": attr.string( + mandatory = True, + doc = "Default target architecture", + ), + "target_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the target os.", + mandatory = False, + ), + "target_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the target os.", + mandatory = False, + allow_single_file = True, + ), + "host_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the host os.", + mandatory = False, + ), + "host_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the host os.", + mandatory = False, + allow_single_file = True, + ), + }, +) From 3f73056b4ed48d055e3af464ff102de8a158afac Mon Sep 17 00:00:00 2001 From: globegitter Date: Fri, 22 Feb 2019 21:55:37 +0100 Subject: [PATCH 03/19] Working toolchain setup for normal build. --- internal/common/os_name.bzl | 14 +++++---- internal/node/node.bzl | 36 ++++++++++++----------- internal/node/node_labels.bzl | 2 +- internal/node/node_repositories.bzl | 13 +++++++-- internal/npm_install/npm_install.bzl | 12 ++++---- toolchains/node/BUILD | 6 ++-- toolchains/node/BUILD.target.tpl | 7 ++++- toolchains/node/node_configure.bzl | 43 ++++++++++++++++++++++------ toolchains/node/node_toolchain.bzl | 30 +++++++++++++++---- 9 files changed, 113 insertions(+), 50 deletions(-) diff --git a/internal/common/os_name.bzl b/internal/common/os_name.bzl index 46cc02a14a..315c033451 100644 --- a/internal/common/os_name.bzl +++ b/internal/common/os_name.bzl @@ -16,11 +16,13 @@ """ OS_ARCH_NAMES = [ - "darwin_amd64", - "windows_amd64", - "linux_amd64", + ("darwin", "amd64"), + ("windows", "amd64"), + ("linux", "amd64"), ] +OS_NAMES = ["_".join(os_arch_name) for os_arch_name in OS_ARCH_NAMES] + def os_name(repository_ctx): """Get the os name for a repository rule @@ -32,10 +34,10 @@ def os_name(repository_ctx): """ os_name = repository_ctx.os.name.lower() if os_name.startswith("mac os"): - return OS_ARCH_NAMES[0] + return OS_NAMES[0] elif os_name.find("windows") != -1: - return OS_ARCH_NAMES[1] + return OS_NAMES[1] elif os_name.startswith("linux"): - return OS_ARCH_NAMES[2] + return OS_NAMES[2] else: fail("Unsupported operating system: " + os_name) diff --git a/internal/node/node.bzl b/internal/node/node.bzl index 1414f62272..07c1910a6f 100644 --- a/internal/node/node.bzl +++ b/internal/node/node.bzl @@ -135,7 +135,7 @@ def _nodejs_binary_impl(ctx): node_tool_info = ctx.toolchains["@build_bazel_rules_nodejs//toolchains/node:toolchain_type"].nodeinfo node_tool_files = [] - if node_tool_info.tool_path == "" and not node_tool_info.tool_target: + if node_tool_info.target_tool_path == "" and not node_tool_info.target_tool: # If tool_path is empty and tool_target is None then there is no local # node tool, we will just print a nice error message if the user # attempts to do bazel run @@ -144,17 +144,17 @@ def _nodejs_binary_impl(ctx): output = ctx.outputs.script, ) else: - node_tool = node_tool_info.tool_path + node_tool = node_tool_info.target_tool_path print(node_tool) - if node_tool_info.tool_target: - node_tool = _short_path_to_manifest_path(ctx, node_tool_info.tool_target.files.to_list()[0].short_path) + if node_tool_info.target_tool: + node_tool = _short_path_to_manifest_path(ctx, node_tool_info.target_tool.files.to_list()[0].short_path) # print(node.short_path) # print(_short_path_to_manifest_path(ctx, node.short_path)) # print(node_tool.path) # print(node_tool.short_path) # print(_short_path_to_manifest_path(ctx, node_tool.short_path)) # node_tool = _runfiles(ctx, node_tool_info.tool_target.files.to_list()[0]) - node_tool_files += node_tool_info.tool_target.files.to_list() + node_tool_files += node_tool_info.target_tool.files.to_list() print("------------") # print(node_tool) @@ -165,6 +165,8 @@ def _nodejs_binary_impl(ctx): # print("------------") print("node_tool_files", node_tool_files) + target_tool_args = node_tool_info.target_tool_args.files.to_list()[0] + substitutions = { "TEMPLATED_args": " ".join([ expand_location_into_runfiles(ctx, a) @@ -174,7 +176,8 @@ def _nodejs_binary_impl(ctx): "TEMPLATED_expected_exit_code": str(expected_exit_code), # "TEMPLATED_node": _short_path_to_manifest_path(ctx, node.short_path), "TEMPLATED_node": node_tool, - "TEMPLATED_repository_args": _short_path_to_manifest_path(ctx, ctx.file._repository_args.short_path), + "TEMPLATED_repository_args": _short_path_to_manifest_path(ctx, target_tool_args.short_path), + # "TEMPLATED_repository_args": "", "TEMPLATED_script_path": script_path, } ctx.actions.expand_template( @@ -184,7 +187,8 @@ def _nodejs_binary_impl(ctx): is_executable = True, ) - runfiles = depset(node_tool_files + [ctx.outputs.loader, ctx.file._repository_args] + node_modules + ctx.files._node_runfiles, transitive = [sources]) + runfiles = depset(node_tool_files + [ctx.outputs.loader, target_tool_args] + node_modules, transitive = [node_tool_info.target_tool_runfiles.files, sources]) + # runfiles = depset(node_tool_files + [ctx.outputs.loader] + node_modules, transitive = [sources]) return [DefaultInfo( executable = ctx.outputs.script, @@ -323,15 +327,15 @@ _NODEJS_EXECUTABLE_ATTRS = { allow_single_file = True, ), # TODO(markus): Remove and get from toolchain - "_node_runfiles": attr.label( - default = Label("@nodejs//:node_runfiles"), - allow_files = True, - ), - # TODO(markus): Remove and get from toolchain - "_repository_args": attr.label( - default = Label("@nodejs//:bin/node_args.sh"), - allow_single_file = True, - ), + # "_node_runfiles": attr.label( + # default = Label("@nodejs//:node_runfiles"), + # allow_files = True, + # ), + # # TODO(markus): Remove and get from toolchain + # "_repository_args": attr.label( + # default = Label("@nodejs//:bin/node_args.sh"), + # allow_single_file = True, + # ), "_source_map_support_files": attr.label_list( default = [ Label("//third_party/github.com/buffer-from:contents"), diff --git a/internal/node/node_labels.bzl b/internal/node/node_labels.bzl index cd12de737c..92b99d7cf0 100644 --- a/internal/node/node_labels.bzl +++ b/internal/node/node_labels.bzl @@ -24,7 +24,7 @@ def get_node_label(os_name): label = Label("@nodejs_%s//:bin/node" % os_name) return label -def get_npm_label(repository_ctx): +def get_npm_label(os_name): if os_name.find("windows") != -1: label = Label("@nodejs_%s//:bin/npm.cmd" % os_name) else: diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index f7a3f955bb..7f4d8de2ff 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -19,10 +19,11 @@ See https://docs.bazel.build/versions/master/skylark/repository_rules.html """ load("//internal/common:check_bazel_version.bzl", "check_bazel_version") -load("//internal/common:os_name.bzl", "os_name", "OS_ARCH_NAMES") +load("//internal/common:os_name.bzl", "os_name", "OS_ARCH_NAMES", "OS_NAMES") load("//internal/npm_install:npm_install.bzl", "yarn_install") load("//third_party/github.com/bazelbuild/bazel-skylib:lib/paths.bzl", "paths") load(":node_labels.bzl", "get_yarn_node_repositories_label") +load("//toolchains/node:node_configure.bzl", node_toolchain_configure = "node_configure") # Callers that don't specify a particular version will get these. DEFAULT_NODE_VERSION = "10.13.0" @@ -544,9 +545,12 @@ def node_repositories( # 0.21.0: repository_ctx.report_progress API check_bazel_version("0.21.0") - for os_arch_name in OS_ARCH_NAMES: + # This needs to be setup so toolchains can access nodejs for all different versions + nodejs_repositorie_names = [] + for os_name in OS_NAMES: + nodejs_repository = "nodejs_%s" % os_name _nodejs_repo( - name = "nodejs_%s" % os_arch_name, + name = nodejs_repository, package_json = package_json, node_version = node_version, yarn_version = yarn_version, @@ -558,6 +562,9 @@ def node_repositories( yarn_urls = yarn_urls, preserve_symlinks = preserve_symlinks, ) + nodejs_repositorie_names.append(nodejs_repository) + + node_toolchain_configure(nodejs_repositorie_names) _yarn_repo( name = "yarn", diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 95a7fd78da..1e72869246 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -119,9 +119,10 @@ def _add_data_dependencies(repository_ctx): def _npm_install_impl(repository_ctx): """Core implementation of npm_install.""" - is_windows = os_name(repository_ctx).find("windows") != -1 - node = repository_ctx.path(get_node_label(repository_ctx)) - npm = get_npm_label(repository_ctx) + os = os_name(repository_ctx) + is_windows = os.find("windows") != -1 + node = repository_ctx.path(get_node_label(os)) + npm = get_npm_label(os) npm_args = ["install"] if repository_ctx.attr.prod_only: @@ -224,8 +225,9 @@ npm_install = repository_rule( def _yarn_install_impl(repository_ctx): """Core implementation of yarn_install.""" - node = repository_ctx.path(get_node_label(repository_ctx)) - yarn = get_yarn_label(repository_ctx) + os = os_name(repository_ctx) + node = repository_ctx.path(get_node_label(os)) + yarn = get_yarn_label(os) if repository_ctx.attr.yarn_lock: if repository_ctx.attr.exclude_packages: diff --git a/toolchains/node/BUILD b/toolchains/node/BUILD index 5d0c270755..2085e0c279 100644 --- a/toolchains/node/BUILD +++ b/toolchains/node/BUILD @@ -35,7 +35,7 @@ toolchain( "@bazel_tools//platforms:linux", "@bazel_tools//platforms:x86_64", ], - toolchain = "@node_config_linux//:toolchain", + toolchain = "@nodejs_config_linux_amd64//:toolchain", toolchain_type = ":toolchain_type", ) @@ -45,7 +45,7 @@ toolchain( "@bazel_tools//platforms:osx", "@bazel_tools//platforms:x86_64", ], - toolchain = "@node_config_osx//:toolchain", + toolchain = "@nodejs_config_osx_amd64//:toolchain", toolchain_type = ":toolchain_type", ) @@ -55,6 +55,6 @@ toolchain( "@bazel_tools//platforms:windows", "@bazel_tools//platforms:x86_64", ], - toolchain = "@node_config_windows//:toolchain", + toolchain = "@nodejs_config_windows_amd64//:toolchain", toolchain_type = ":toolchain_type", ) \ No newline at end of file diff --git a/toolchains/node/BUILD.target.tpl b/toolchains/node/BUILD.target.tpl index 919b4d3999..54519da045 100644 --- a/toolchains/node/BUILD.target.tpl +++ b/toolchains/node/BUILD.target.tpl @@ -21,5 +21,10 @@ load("@build_bazel_rules_nodejs//toolchains/node:node_toolchain.bzl", "node_tool node_toolchain( name = "toolchain", - tool_target = "%{NODE_TARGET}", + target_tool = "%{NODE_TARGET_TOOL}", + target_tool_runfiles = "%{NODE_TARGET_RUNFILES}", + target_tool_args = "%{NODE_TARGET_ARGS}", + host_tool = "%{NODE_HOST_TOOL}", + os = "%{OS}", + arch = "%{ARCH}", ) \ No newline at end of file diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index 7fc2497b0b..96b82ddf5c 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -16,7 +16,6 @@ Defines a repository rule for configuring the node binary. """ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@build_bazel_rules_nodejs//:defs.bzl", "node_repositories") load( ":defaults.bzl", _k8s_commit = "k8s_commit", @@ -29,13 +28,35 @@ load( _k8s_repo_tools_sha = "k8s_repo_tools_sha", _k8s_sha256 = "k8s_sha256", ) +load("//internal/common:os_name.bzl", "OS_ARCH_NAMES") def _impl(repository_ctx): substitutions = None - if repository_ctx.attr.target: - node_target = repository_ctx.attr.target - substitutions = {"%{NODE_TARGET}": "%s" % node_target} + host_os = repository_ctx.os.name.lower() + if repository_ctx.attr.nodejs_repository_names: + target_tool = "" + host_tool = "" + target_repo_name = "" + for repo_name in repository_ctx.attr.nodejs_repository_names: + if repository_ctx.attr.os in repo_name: + target_tool = "@%s//:node" % repo_name + target_repo_name = repo_name + if host_os in repo_name: + host_tool = "@%s//:node" % repo_name + + if not target_tool or not host_tool: + fail("No host_tool nor target_tool found") + + substitutions = { + "%{NODE_TARGET_TOOL}": "%s" % target_tool, + "%{NODE_TARGET_RUNFILES}": "@%s//:node_runfiles" % target_repo_name, + "%{NODE_TARGET_ARGS}": "@%s//:bin/node_args.sh" % target_repo_name, + "%{NODE_HOST_TOOL}": "%s" % host_tool, + "%{OS}": "%s" % repository_ctx.attr.os, + "%{ARCH}": "%s" % repository_ctx.attr.arch, + } template = Label("@build_bazel_rules_nodejs//toolchains/node:BUILD.target.tpl") + else: node_tool_path = repository_ctx.attr.local_path or repository_ctx.which("node") or "" substitutions = {"%{NODE_PATH}": "%s" % node_tool_path} @@ -77,6 +98,9 @@ _node_configure = repository_rule( mandatory = False, allow_single_file = True, ), + "nodejs_repository_names": attr.string_list( + mandatory = False, + ), }, ) @@ -106,7 +130,7 @@ def _ensure_all_provided(func_name, attrs, kwargs): ", ".join(provided), )) -def node_configure(name, **kwargs): +def node_configure(node_repositories): """ Creates an external repository with a node_toolchain target properly configured. @@ -138,10 +162,11 @@ def node_configure(name, **kwargs): "@build_bazel_rules_nodejs//toolchains/node:node_windows_toolchain", ) - node_repositories(**kwargs) - - _node_configure(name = name + "_linux", os="linux", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_linux//:node") - _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") + if node_repositories: + print("!!!!!!!!!InHERE!!!!!!!!!!!!") + for os, arch in OS_ARCH_NAMES: + _node_configure(name = "nodejs_config_%s_%s" % (os, arch), os=os, arch=arch, nodejs_repository_names=node_repositories) + # _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") # build_srcs = False # if "build_srcs" in kwargs and "kubectl_path" in kwargs: diff --git a/toolchains/node/node_toolchain.bzl b/toolchains/node/node_toolchain.bzl index c4caafdf99..bed34496f1 100644 --- a/toolchains/node/node_toolchain.bzl +++ b/toolchains/node/node_toolchain.bzl @@ -18,18 +18,26 @@ This module implements the node toolchain rule. NodeInfo = provider( doc = "Information about how to invoke the node binary.", fields = { - "tool_path": "Path to an existing nodejs executable", - "tool_target": "A hermetically downloaded nodejs executable target.", + "target_tool_path": "Path to an existing nodejs executable", + "target_tool": "A hermetically downloaded nodejs executable target.", + "target_tool_runfiles": "A hermetically downloaded nodejs executable target.", + "target_tool_args": "A hermetically downloaded nodejs executable target.", + "host_tool_path": "Path to an existing nodejs executable", + "host_tool": "A hermetically downloaded nodejs executable target.", }, ) def _node_toolchain_impl(ctx): - if not ctx.attr.tool_path and not ctx.attr.tool_target: - print("No nodejs binary was found or built, executing run for rules_nodejs targets might not work.") + if not ctx.attr.host_tool and not ctx.attr.host_tool_path: + print("No nodejs binary was not found or built, executing run for rules_nodejs targets might not work.") toolchain_info = platform_common.ToolchainInfo( nodeinfo = NodeInfo( - tool_path = ctx.attr.tool_path, - tool_target = ctx.attr.tool_target, + target_tool_path = ctx.attr.target_tool_path, + target_tool = ctx.attr.target_tool, + target_tool_runfiles = ctx.attr.target_tool_runfiles, + target_tool_args = ctx.attr.target_tool_args, + host_tool_path = ctx.attr.host_tool_path, + host_tool = ctx.attr.host_tool, ), ) return [toolchain_info] @@ -54,6 +62,16 @@ node_toolchain = rule( mandatory = False, allow_single_file = True, ), + "target_tool_runfiles": attr.label( + doc = "Target for a downloaded nodejs binary for the target os.", + mandatory = False, + allow_files = False, + ), + "target_tool_args": attr.label( + doc = "Target for a downloaded nodejs binary for the target os.", + mandatory = False, + allow_single_file = True, + ), "host_tool_path": attr.string( doc = "Absolute path to a pre-installed nodejs binary for the host os.", mandatory = False, From 8d5e4198c899d0818b31c9057ce2d898bc5241ce Mon Sep 17 00:00:00 2001 From: globegitter Date: Fri, 22 Feb 2019 22:43:46 +0100 Subject: [PATCH 04/19] Early progress on multi-os npm packages. --- internal/node/node_repositories.bzl | 9 +++++++- internal/npm_install/generate_build_file.js | 25 ++++++++++++++++----- internal/npm_install/npm_install.bzl | 17 ++++++++++++-- toolchains/node/BUILD | 2 +- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 7f4d8de2ff..7a73cef4f0 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -419,7 +419,14 @@ if %errorlevel% neq 0 exit /b %errorlevel% "TEMPLATED_yarn_dir": YARN_DIR, }, ) - result = repository_ctx.execute([node_entry, "generate_build_file.js"]) + host_os = repository_ctx.os.name.lower() + if host_os in repository_ctx.attr.name: + # We have to use the relative path here otherwise bazel report a cycle + result = repository_ctx.execute([node_entry, "generate_build_file.js"]) + else: + # Note: It seems we can only get a string identifying the os but not the architecture + result = repository_ctx.execute([repository_ctx.path(Label("@nodejs_%s_amd64//:bin/node" % host_os)), "generate_build_file.js"]) + if result.return_code: fail("node failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)) diff --git a/internal/npm_install/generate_build_file.js b/internal/npm_install/generate_build_file.js index 8b7af9bbcd..b8e6f80905 100644 --- a/internal/npm_install/generate_build_file.js +++ b/internal/npm_install/generate_build_file.js @@ -676,13 +676,26 @@ ${printJson(pkg)} filegroup( name = "${pkg._name}__pkg", - srcs = [ - # ${pkg._dir} package contents (and contents of nested node_modules) - ":${pkg._name}__files", - # direct or transitive dependencies hoisted to root by the package manager - ${ + srcs = select({ + "//conditions:default": [ + # ${pkg._dir} package contents (and contents of nested node_modules) + ":${pkg._name}__files", + # direct or transitive dependencies hoisted to root by the package manager + ${ pkgDeps.map(dep => `"//node_modules/${dep._dir}:${dep._name}__files",`).join('\n ')} - ], + ], + "@io_bazel_rules_go//go/platform:darwin_amd64": [ + # ${pkg._dir} package contents (and contents of nested node_modules) + "@npm_darwin_amd64//${pkg._dir}:${pkg._name}__files", + # direct or transitive dependencies hoisted to root by the package manager + ${ + pkgDeps.map(dep => `"@npm_darwin_amd64//node_modules/${dep._dir}:${dep._name}__files",`) + .join('\n ')} + ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "cgo_windows.go", + ], + }), tags = ["NODE_MODULE_MARKER"], ) diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 1e72869246..1a2c020087 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -21,7 +21,7 @@ as the package manager. See discussion in the README. """ -load("//internal/common:os_name.bzl", "os_name") +load("//internal/common:os_name.bzl", "os_name", "OS_ARCH_NAMES", "OS_NAMES") load("//internal/node:node_labels.bzl", "get_node_label", "get_npm_label", "get_yarn_label") COMMON_ATTRIBUTES = dict(dict(), **{ @@ -285,7 +285,7 @@ def _yarn_install_impl(repository_ctx): _create_build_file(repository_ctx, node) -yarn_install = repository_rule( +_yarn_install = repository_rule( attrs = dict(COMMON_ATTRIBUTES, **{ "timeout": attr.int( default = 3600, @@ -315,3 +315,16 @@ yarn_install = repository_rule( ) """Runs yarn install during workspace setup. """ + +def yarn_install(**kwargs): + name = kwargs.pop(name) + _yarn_install( + name = name, + **kwargs, + ) + + for os in OS_NAMES: + _yarn_install( + name = "%s_%s" % (name, os), + **kwargs, + ) \ No newline at end of file diff --git a/toolchains/node/BUILD b/toolchains/node/BUILD index 2085e0c279..1d9e476a59 100644 --- a/toolchains/node/BUILD +++ b/toolchains/node/BUILD @@ -45,7 +45,7 @@ toolchain( "@bazel_tools//platforms:osx", "@bazel_tools//platforms:x86_64", ], - toolchain = "@nodejs_config_osx_amd64//:toolchain", + toolchain = "@nodejs_config_darwin_amd64//:toolchain", toolchain_type = ":toolchain_type", ) From 8a45f99678fa19830ed04d2a06d0b8048130a07d Mon Sep 17 00:00:00 2001 From: globegitter Date: Tue, 2 Apr 2019 20:26:48 +0200 Subject: [PATCH 05/19] Run buildifier. --- internal/node/node.bzl | 2 ++ internal/node/node_repositories.bzl | 4 +-- internal/npm_install/npm_install.bzl | 6 ++-- internal/rollup/BUILD.bazel | 12 +++---- internal/web_package/BUILD.bazel | 8 +---- toolchains/node/BUILD | 4 +-- toolchains/node/node_configure.bzl | 52 +++++++++++----------------- toolchains/node/node_toolchain.bzl | 40 ++++++++++----------- 8 files changed, 55 insertions(+), 73 deletions(-) diff --git a/internal/node/node.bzl b/internal/node/node.bzl index 533f0c0a85..af4beb94e6 100644 --- a/internal/node/node.bzl +++ b/internal/node/node.bzl @@ -148,6 +148,7 @@ def _nodejs_binary_impl(ctx): print(node_tool) if node_tool_info.target_tool: node_tool = _short_path_to_manifest_path(ctx, node_tool_info.target_tool.files.to_list()[0].short_path) + # print(node.short_path) # print(_short_path_to_manifest_path(ctx, node.short_path)) # print(node_tool.path) @@ -157,6 +158,7 @@ def _nodejs_binary_impl(ctx): node_tool_files += node_tool_info.target_tool.files.to_list() print("------------") + # print(node_tool) # print(node_tool.path) # print(_short_path_to_manifest_path(ctx, node_tool)) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 71613befd4..9510bcf5be 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -19,11 +19,11 @@ See https://docs.bazel.build/versions/master/skylark/repository_rules.html """ load("//internal/common:check_bazel_version.bzl", "check_bazel_version") -load("//internal/common:os_name.bzl", "os_name", "OS_ARCH_NAMES", "OS_NAMES") +load("//internal/common:os_name.bzl", "OS_NAMES", "os_name") load("//internal/npm_install:npm_install.bzl", "yarn_install") load("//third_party/github.com/bazelbuild/bazel-skylib:lib/paths.bzl", "paths") -load(":node_labels.bzl", "get_yarn_node_repositories_label") load("//toolchains/node:node_configure.bzl", node_toolchain_configure = "node_configure") +load(":node_labels.bzl", "get_yarn_node_repositories_label") # Callers that don't specify a particular version will get these. DEFAULT_NODE_VERSION = "10.13.0" diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 8d223ece25..359f38e8d4 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -21,7 +21,7 @@ as the package manager. See discussion in the README. """ -load("//internal/common:os_name.bzl", "os_name", "OS_ARCH_NAMES", "OS_NAMES") +load("//internal/common:os_name.bzl", "OS_NAMES", "os_name") load("//internal/node:node_labels.bzl", "get_node_label", "get_npm_label", "get_yarn_label") COMMON_ATTRIBUTES = dict(dict(), **{ @@ -323,11 +323,11 @@ def yarn_install(**kwargs): name = kwargs.pop(name) _yarn_install( name = name, - **kwargs, + **kwargs ) for os in OS_NAMES: _yarn_install( name = "%s_%s" % (name, os), - **kwargs, + **kwargs ) diff --git a/internal/rollup/BUILD.bazel b/internal/rollup/BUILD.bazel index 430f916fe1..a6a9f0a485 100644 --- a/internal/rollup/BUILD.bazel +++ b/internal/rollup/BUILD.bazel @@ -13,6 +13,12 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +# BEGIN-INTERNAL +# TODO: switch to npm_bazel_jasmine +# buildozer: disable=load-on-top +# buildozer: disable=out-of-order-load +load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test") load("//:defs.bzl", "nodejs_binary") licenses(["notice"]) # Apache 2.0 @@ -86,12 +92,6 @@ nodejs_binary( visibility = ["//visibility:public"], ) -# BEGIN-INTERNAL -# TODO: switch to npm_bazel_jasmine -# buildozer: disable=load-on-top -# buildozer: disable=out-of-order-load -load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test") - jasmine_node_test( name = "test", srcs = glob(["*.spec.js"]), diff --git a/internal/web_package/BUILD.bazel b/internal/web_package/BUILD.bazel index 8565200636..bbeb54064f 100644 --- a/internal/web_package/BUILD.bazel +++ b/internal/web_package/BUILD.bazel @@ -1,5 +1,5 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary") +load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test", "nodejs_binary") package(default_visibility = ["//visibility:public"]) @@ -36,12 +36,6 @@ nodejs_binary( install_source_map_support = False, ) -# BEGIN-INTERNAL -# TODO: switch to npm_bazel_jasmine -# buildozer: disable=load-on-top -# buildozer: disable=same-origin-load -load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test") - jasmine_node_test( name = "injector_test", srcs = ["injector_spec.js"], diff --git a/toolchains/node/BUILD b/toolchains/node/BUILD index 1d9e476a59..02447c2c53 100644 --- a/toolchains/node/BUILD +++ b/toolchains/node/BUILD @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":node_toolchain.bzl", "node_toolchain") - package(default_visibility = ["//visibility:private"]) licenses(["notice"]) # Apache 2.0 @@ -57,4 +55,4 @@ toolchain( ], toolchain = "@nodejs_config_windows_amd64//:toolchain", toolchain_type = ":toolchain_type", -) \ No newline at end of file +) diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index 96b82ddf5c..9cc194e127 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -15,19 +15,6 @@ Defines a repository rule for configuring the node binary. """ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load( - ":defaults.bzl", - _k8s_commit = "k8s_commit", - _k8s_org = "k8s_org", - _k8s_prefix = "k8s_prefix", - _k8s_repo = "k8s_repo", - _k8s_repo_tools_commit = "k8s_repo_tools_commit", - _k8s_repo_tools_prefix = "k8s_repo_tools_prefix", - _k8s_repo_tools_repo = "k8s_repo_tools_repo", - _k8s_repo_tools_sha = "k8s_repo_tools_sha", - _k8s_sha256 = "k8s_sha256", -) load("//internal/common:os_name.bzl", "OS_ARCH_NAMES") def _impl(repository_ctx): @@ -43,17 +30,17 @@ def _impl(repository_ctx): target_repo_name = repo_name if host_os in repo_name: host_tool = "@%s//:node" % repo_name - + if not target_tool or not host_tool: fail("No host_tool nor target_tool found") substitutions = { - "%{NODE_TARGET_TOOL}": "%s" % target_tool, - "%{NODE_TARGET_RUNFILES}": "@%s//:node_runfiles" % target_repo_name, - "%{NODE_TARGET_ARGS}": "@%s//:bin/node_args.sh" % target_repo_name, + "%{ARCH}": "%s" % repository_ctx.attr.arch, "%{NODE_HOST_TOOL}": "%s" % host_tool, + "%{NODE_TARGET_ARGS}": "@%s//:bin/node_args.sh" % target_repo_name, + "%{NODE_TARGET_RUNFILES}": "@%s//:node_runfiles" % target_repo_name, + "%{NODE_TARGET_TOOL}": "%s" % target_tool, "%{OS}": "%s" % repository_ctx.attr.os, - "%{ARCH}": "%s" % repository_ctx.attr.arch, } template = Label("@build_bazel_rules_nodejs//toolchains/node:BUILD.target.tpl") @@ -72,20 +59,12 @@ def _impl(repository_ctx): _node_configure = repository_rule( implementation = _impl, attrs = { - "os": attr.string( - mandatory = True, - doc = "Default target OS", - ), "arch": attr.string( mandatory = True, doc = "Default target architecture", ), - "target_tool_path": attr.string( - doc = "Absolute path to a pre-installed nodejs binary for the target os.", - mandatory = False, - ), - "target_tool": attr.label( - doc = "Target for a downloaded nodejs binary for the target os.", + "host_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the host os.", mandatory = False, allow_single_file = True, ), @@ -93,12 +72,20 @@ _node_configure = repository_rule( doc = "Absolute path to a pre-installed nodejs binary for the host os.", mandatory = False, ), - "host_tool": attr.label( - doc = "Target for a downloaded nodejs binary for the host os.", + "nodejs_repository_names": attr.string_list( + mandatory = False, + ), + "os": attr.string( + mandatory = True, + doc = "Default target OS", + ), + "target_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the target os.", mandatory = False, allow_single_file = True, ), - "nodejs_repository_names": attr.string_list( + "target_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the target os.", mandatory = False, ), }, @@ -165,7 +152,8 @@ def node_configure(node_repositories): if node_repositories: print("!!!!!!!!!InHERE!!!!!!!!!!!!") for os, arch in OS_ARCH_NAMES: - _node_configure(name = "nodejs_config_%s_%s" % (os, arch), os=os, arch=arch, nodejs_repository_names=node_repositories) + _node_configure(name = "nodejs_config_%s_%s" % (os, arch), os = os, arch = arch, nodejs_repository_names = node_repositories) + # _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") # build_srcs = False diff --git a/toolchains/node/node_toolchain.bzl b/toolchains/node/node_toolchain.bzl index bed34496f1..b6a54a3bf8 100644 --- a/toolchains/node/node_toolchain.bzl +++ b/toolchains/node/node_toolchain.bzl @@ -18,12 +18,12 @@ This module implements the node toolchain rule. NodeInfo = provider( doc = "Information about how to invoke the node binary.", fields = { - "target_tool_path": "Path to an existing nodejs executable", + "host_tool": "A hermetically downloaded nodejs executable target.", + "host_tool_path": "Path to an existing nodejs executable", "target_tool": "A hermetically downloaded nodejs executable target.", - "target_tool_runfiles": "A hermetically downloaded nodejs executable target.", "target_tool_args": "A hermetically downloaded nodejs executable target.", - "host_tool_path": "Path to an existing nodejs executable", - "host_tool": "A hermetically downloaded nodejs executable target.", + "target_tool_path": "Path to an existing nodejs executable", + "target_tool_runfiles": "A hermetically downloaded nodejs executable target.", }, ) @@ -45,41 +45,41 @@ def _node_toolchain_impl(ctx): node_toolchain = rule( implementation = _node_toolchain_impl, attrs = { - "os": attr.string( - mandatory = True, - doc = "Default target OS", - ), "arch": attr.string( mandatory = True, doc = "Default target architecture", ), - "target_tool_path": attr.string( - doc = "Absolute path to a pre-installed nodejs binary for the target os.", + "host_tool": attr.label( + doc = "Target for a downloaded nodejs binary for the host os.", mandatory = False, + allow_single_file = True, ), - "target_tool": attr.label( - doc = "Target for a downloaded nodejs binary for the target os.", + "host_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the host os.", mandatory = False, - allow_single_file = True, ), - "target_tool_runfiles": attr.label( + "os": attr.string( + mandatory = True, + doc = "Default target OS", + ), + "target_tool": attr.label( doc = "Target for a downloaded nodejs binary for the target os.", mandatory = False, - allow_files = False, + allow_single_file = True, ), "target_tool_args": attr.label( doc = "Target for a downloaded nodejs binary for the target os.", mandatory = False, allow_single_file = True, ), - "host_tool_path": attr.string( - doc = "Absolute path to a pre-installed nodejs binary for the host os.", + "target_tool_path": attr.string( + doc = "Absolute path to a pre-installed nodejs binary for the target os.", mandatory = False, ), - "host_tool": attr.label( - doc = "Target for a downloaded nodejs binary for the host os.", + "target_tool_runfiles": attr.label( + doc = "Target for a downloaded nodejs binary for the target os.", mandatory = False, - allow_single_file = True, + allow_files = False, ), }, ) From ac80ba4d0c2befcc3d6a6abc8efbf507fb345677 Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 22:47:11 +0200 Subject: [PATCH 06/19] Delete per platform deps as well as some other unnecessary code. --- internal/node/node_repositories.bzl | 3 +- internal/npm_install/generate_build_file.js | 25 +++-------- internal/npm_install/npm_install.bzl | 14 +----- internal/rollup/BUILD.bazel | 12 ++--- internal/web_package/BUILD.bazel | 8 +++- toolchains/node/defaults.bzl | 50 --------------------- 6 files changed, 21 insertions(+), 91 deletions(-) delete mode 100644 toolchains/node/defaults.bzl diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 9510bcf5be..f5fdf29dcc 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -131,8 +131,7 @@ def _download_node(repository_ctx): if repository_ctx.attr.vendored_node: return - # host = os_name(repository_ctx) - host = "darwin_amd64" if "darwin" in repository_ctx.attr.name else "linux_amd64" + host = os_name(repository_ctx) node_version = repository_ctx.attr.node_version node_repositories = repository_ctx.attr.node_repositories node_urls = repository_ctx.attr.node_urls diff --git a/internal/npm_install/generate_build_file.js b/internal/npm_install/generate_build_file.js index 9386976196..fab6671db3 100644 --- a/internal/npm_install/generate_build_file.js +++ b/internal/npm_install/generate_build_file.js @@ -678,26 +678,13 @@ function printPkgTarget(pkg, pkgDeps) { return ` filegroup( name = "${pkg._name}__pkg", - srcs = select({ - "//conditions:default": [ - # ${pkg._dir} package contents (and contents of nested node_modules) - ":${pkg._name}__files", - # direct or transitive dependencies hoisted to root by the package manager - ${ + srcs = [ + # ${pkg._dir} package contents (and contents of nested node_modules) + ":${pkg._name}__files", + # direct or transitive dependencies hoisted to root by the package manager + ${ pkgDeps.map(dep => `"//node_modules/${dep._dir}:${dep._name}__files",`).join('\n ')} - ], - "@io_bazel_rules_go//go/platform:darwin_amd64": [ - # ${pkg._dir} package contents (and contents of nested node_modules) - "@npm_darwin_amd64//${pkg._dir}:${pkg._name}__files", - # direct or transitive dependencies hoisted to root by the package manager - ${ - pkgDeps.map(dep => `"@npm_darwin_amd64//node_modules/${dep._dir}:${dep._name}__files",`) - .join('\n ')} - ], - "@io_bazel_rules_go//go/platform:windows_amd64": [ - "cgo_windows.go", - ], - }), + ], tags = ["NODE_MODULE_MARKER"], ) `; diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index faf676e657..4749865b08 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -300,7 +300,7 @@ def _yarn_install_impl(repository_ctx): _create_build_file(repository_ctx, node, repository_ctx.attr.yarn_lock) -_yarn_install = repository_rule( +yarn_install = repository_rule( attrs = dict(COMMON_ATTRIBUTES, **{ "timeout": attr.int( default = 3600, @@ -331,15 +331,3 @@ _yarn_install = repository_rule( """Runs yarn install during workspace setup. """ -def yarn_install(**kwargs): - name = kwargs.pop(name) - _yarn_install( - name = name, - **kwargs - ) - - for os in OS_NAMES: - _yarn_install( - name = "%s_%s" % (name, os), - **kwargs - ) diff --git a/internal/rollup/BUILD.bazel b/internal/rollup/BUILD.bazel index a6a9f0a485..430f916fe1 100644 --- a/internal/rollup/BUILD.bazel +++ b/internal/rollup/BUILD.bazel @@ -13,12 +13,6 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") - -# BEGIN-INTERNAL -# TODO: switch to npm_bazel_jasmine -# buildozer: disable=load-on-top -# buildozer: disable=out-of-order-load -load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test") load("//:defs.bzl", "nodejs_binary") licenses(["notice"]) # Apache 2.0 @@ -92,6 +86,12 @@ nodejs_binary( visibility = ["//visibility:public"], ) +# BEGIN-INTERNAL +# TODO: switch to npm_bazel_jasmine +# buildozer: disable=load-on-top +# buildozer: disable=out-of-order-load +load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test") + jasmine_node_test( name = "test", srcs = glob(["*.spec.js"]), diff --git a/internal/web_package/BUILD.bazel b/internal/web_package/BUILD.bazel index bbeb54064f..8565200636 100644 --- a/internal/web_package/BUILD.bazel +++ b/internal/web_package/BUILD.bazel @@ -1,5 +1,5 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test", "nodejs_binary") +load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary") package(default_visibility = ["//visibility:public"]) @@ -36,6 +36,12 @@ nodejs_binary( install_source_map_support = False, ) +# BEGIN-INTERNAL +# TODO: switch to npm_bazel_jasmine +# buildozer: disable=load-on-top +# buildozer: disable=same-origin-load +load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test") + jasmine_node_test( name = "injector_test", srcs = ["injector_spec.js"], diff --git a/toolchains/node/defaults.bzl b/toolchains/node/defaults.bzl deleted file mode 100644 index 3a686293bd..0000000000 --- a/toolchains/node/defaults.bzl +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2018 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Defines defaults for the kubectl toolchain. -""" - -# The following variables define the kubectl source repository that is pulled -# when the kubectl toolchain is configured to build from source. -# The repository is at https://github.com/kubernetes/kubernetes -# The kubernetes github organization. -k8s_org = "kubernetes" - -# The kubernetes repository. -k8s_repo = "kubernetes" - -# The release commit/tag to for the kubernetes repo. -k8s_commit = "v1.13.0-beta.1" - -# The archive prefix. This is the name of the top level directory in the -# downloaded repository archive tarball. -k8s_prefix = "kubernetes-1.13.0-beta.1" - -# The SHA256 of the k8s repo. -k8s_sha256 = "dfb39ce36284c1ce228954ca12bf016c09be61e40a875e8af4fff84e116bd3a7" - -# The kubernetes repository infrastructure tools repository. -# https://github.com/kubernetes/repo-infra -k8s_repo_tools_repo = "repo-infra" - -# The commit pin to use for the kuebernetes repository infrastructure tools -# repository. -k8s_repo_tools_commit = "b4bc4f1552c7fc1d4654753ca9b0e5e13883429f" - -# The archive prefix. This is the name of the top level directory in the -# downloaded repository archive tarball. -k8s_repo_tools_prefix = "{}-{}".format(k8s_repo_tools_repo, k8s_repo_tools_commit) - -# The SHA256 of the kubernetes repository infrastructure tools repository. -k8s_repo_tools_sha = "21160531ea8a9a4001610223ad815622bf60671d308988c7057168a495a7e2e8" From 1858f9c1cfd8a360ffa83b1348706533fb7af0ef Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 22:58:27 +0200 Subject: [PATCH 07/19] Download correct nodejs version and have convinience nodejs. --- internal/node/node_repositories.bzl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index f5fdf29dcc..2b95163405 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -130,8 +130,11 @@ def _download_node(repository_ctx): """ if repository_ctx.attr.vendored_node: return - - host = os_name(repository_ctx) + if repository_ctx.name == "nodejs": + host = os_name(repository_ctx) + else: + host = repository_ctx.name.lstrip("nodejs_") + print(host) node_version = repository_ctx.attr.node_version node_repositories = repository_ctx.attr.node_repositories node_urls = repository_ctx.attr.node_urls @@ -552,6 +555,22 @@ def node_repositories( # 0.21.0: repository_ctx.report_progress API check_bazel_version("0.21.0") + # This "nodejs" repo is just for convinience so one does not have to target @nodejs_//... + _maybe( + _nodejs_repo, + name = "nodejs", + package_json = package_json, + node_version = node_version, + yarn_version = yarn_version, + vendored_node = vendored_node, + vendored_yarn = vendored_yarn, + node_repositories = node_repositories, + yarn_repositories = yarn_repositories, + node_urls = node_urls, + yarn_urls = yarn_urls, + preserve_symlinks = preserve_symlinks, + ) + # This needs to be setup so toolchains can access nodejs for all different versions nodejs_repositorie_names = [] for os_name in OS_NAMES: From 5a20d43e296cfd459029551d4df7d79f303c9b81 Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 23:02:02 +0200 Subject: [PATCH 08/19] Lint fix. --- internal/npm_install/npm_install.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 4749865b08..4610085a80 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -330,4 +330,3 @@ yarn_install = repository_rule( ) """Runs yarn install during workspace setup. """ - From 6fd742fd655f11751c1ac985694ffc468c24c6e9 Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 23:06:27 +0200 Subject: [PATCH 09/19] Rename build file and add toolchain files to the release package. --- BUILD.bazel | 1 + toolchains/node/{BUILD => BUILD.bazel} | 6 ++++++ 2 files changed, 7 insertions(+) rename toolchains/node/{BUILD => BUILD.bazel} (94%) diff --git a/BUILD.bazel b/BUILD.bazel index 31a19fafb3..0e3d1c098c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -59,6 +59,7 @@ npm_package( "//internal/npm_install:package_contents", "//internal/npm_package:package_contents", "//internal/web_package:package_contents", + "//toolchains/node:package_contents", ], ) diff --git a/toolchains/node/BUILD b/toolchains/node/BUILD.bazel similarity index 94% rename from toolchains/node/BUILD rename to toolchains/node/BUILD.bazel index 02447c2c53..4f31ec6760 100644 --- a/toolchains/node/BUILD +++ b/toolchains/node/BUILD.bazel @@ -16,6 +16,12 @@ package(default_visibility = ["//visibility:private"]) licenses(["notice"]) # Apache 2.0 +filegroup( + name = "package_contents", + srcs = glob(["*"]), + visibility = ["//:__pkg__"], +) + # node toolchain type toolchain_type(name = "toolchain_type") From d64d24b899f8dc0f2ecd7e4f62f01e814575874c Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 23:09:45 +0200 Subject: [PATCH 10/19] Fix lint. --- internal/npm_install/npm_install.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 4610085a80..d613f56618 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -21,7 +21,7 @@ as the package manager. See discussion in the README. """ -load("//internal/common:os_name.bzl", "OS_NAMES", "os_name") +load("//internal/common:os_name.bzl", "os_name") load("//internal/node:node_labels.bzl", "get_node_label", "get_npm_label", "get_yarn_label") COMMON_ATTRIBUTES = dict(dict(), **{ From 5a9feaa65ad2f59c7ef1577f3f2316d6dadb43b8 Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 23:24:39 +0200 Subject: [PATCH 11/19] Add toolchain bazel libraries. --- BUILD.bazel | 1 + toolchains/node/BUILD.bazel | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index 0e3d1c098c..4d6e0f461d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -16,6 +16,7 @@ bzl_library( "//internal/jasmine_node_test:bzl", "//internal/npm_package:bzl", "//internal/rollup:bzl", + "//toolchains/node:bzl", ], ) diff --git a/toolchains/node/BUILD.bazel b/toolchains/node/BUILD.bazel index 4f31ec6760..6bfd1aeb69 100644 --- a/toolchains/node/BUILD.bazel +++ b/toolchains/node/BUILD.bazel @@ -11,11 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") package(default_visibility = ["//visibility:private"]) licenses(["notice"]) # Apache 2.0 +bzl_library( + name = "bzl", + srcs = glob(["*.bzl"]), + visibility = ["//visibility:public"], +) + filegroup( name = "package_contents", srcs = glob(["*"]), From 54db48bd242c6ee304f5defcc2e2d0d89a904eb4 Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 23:47:25 +0200 Subject: [PATCH 12/19] Added platforms and fix fetching. --- internal/node/node_repositories.bzl | 3 +-- toolchains/node/BUILD.bazel | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 2b95163405..786cb085a8 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -133,8 +133,7 @@ def _download_node(repository_ctx): if repository_ctx.name == "nodejs": host = os_name(repository_ctx) else: - host = repository_ctx.name.lstrip("nodejs_") - print(host) + host = repository_ctx.name.split("nodejs_", 1)[1] node_version = repository_ctx.attr.node_version node_repositories = repository_ctx.attr.node_repositories node_urls = repository_ctx.attr.node_urls diff --git a/toolchains/node/BUILD.bazel b/toolchains/node/BUILD.bazel index 6bfd1aeb69..4a66935a3a 100644 --- a/toolchains/node/BUILD.bazel +++ b/toolchains/node/BUILD.bazel @@ -17,6 +17,30 @@ package(default_visibility = ["//visibility:private"]) licenses(["notice"]) # Apache 2.0 +platform( + name = "darwin_amd64", + constraint_values = [ + "@bazel_tools//platforms:osx", + "@bazel_tools//platforms:x86_64", + ], +) + +platform( + name = "linux_amd64", + constraint_values = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], +) + +platform( + name = "windows_amd64", + constraint_values = [ + "@bazel_tools//platforms:windows", + "@bazel_tools//platforms:x86_64", + ], +) + bzl_library( name = "bzl", srcs = glob(["*.bzl"]), From f992aa87a000b33859758deedfcba09b077f1e75 Mon Sep 17 00:00:00 2001 From: globegitter Date: Thu, 4 Apr 2019 23:52:27 +0200 Subject: [PATCH 13/19] Fix host os. --- internal/node/node_repositories.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 786cb085a8..2bab69c334 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -421,13 +421,13 @@ if %errorlevel% neq 0 exit /b %errorlevel% "TEMPLATED_yarn_dir": YARN_DIR, }, ) - host_os = repository_ctx.os.name.lower() + host_os = os_name(repository_ctx) if host_os in repository_ctx.attr.name: # We have to use the relative path here otherwise bazel report a cycle result = repository_ctx.execute([node_entry, "generate_build_file.js"]) else: # Note: It seems we can only get a string identifying the os but not the architecture - result = repository_ctx.execute([repository_ctx.path(Label("@nodejs_%s_amd64//:bin/node" % host_os)), "generate_build_file.js"]) + result = repository_ctx.execute([repository_ctx.path(Label("@nodejs_%s//:bin/node" % host_os)), "generate_build_file.js"]) if result.return_code: fail("node failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)) From e0a8b9f27d4fcaa2c47c9b668f4ac5497de575a1 Mon Sep 17 00:00:00 2001 From: globegitter Date: Fri, 5 Apr 2019 00:42:02 +0200 Subject: [PATCH 14/19] Special case nodejs name. --- internal/node/node_repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 2bab69c334..e444fa3ae2 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -422,7 +422,7 @@ if %errorlevel% neq 0 exit /b %errorlevel% }, ) host_os = os_name(repository_ctx) - if host_os in repository_ctx.attr.name: + if host_os in repository_ctx.attr.name or repository_ctx.attr.name == "nodejs": # We have to use the relative path here otherwise bazel report a cycle result = repository_ctx.execute([node_entry, "generate_build_file.js"]) else: From 605fd0d630cc5986691a08ff8082772faf5a8351 Mon Sep 17 00:00:00 2001 From: Markus Padourek Date: Mon, 13 May 2019 00:03:39 +0200 Subject: [PATCH 15/19] Added more debugging logs. --- toolchains/node/node_configure.bzl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index 9cc194e127..c2463865cf 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -25,13 +25,29 @@ def _impl(repository_ctx): host_tool = "" target_repo_name = "" for repo_name in repository_ctx.attr.nodejs_repository_names: + print("-----") + print("name", repository_ctx.attr.name) + print("repo_name", repo_name) + print("repository_ctx.attr.os", repository_ctx.attr.os) + print("host_os", host_os) if repository_ctx.attr.os in repo_name: target_tool = "@%s//:node" % repo_name target_repo_name = repo_name + + if host_os == "mac os x": + print("host_os new", host_os) + host_os = "darwin" + if host_os in repo_name: host_tool = "@%s//:node" % repo_name + print("host_tool", host_tool) + print("target_tool", target_tool) + print("target_repo_name", target_repo_name) + print("-----") + if not target_tool or not host_tool: + print("fail") fail("No host_tool nor target_tool found") substitutions = { @@ -152,6 +168,7 @@ def node_configure(node_repositories): if node_repositories: print("!!!!!!!!!InHERE!!!!!!!!!!!!") for os, arch in OS_ARCH_NAMES: + print("nodejs_config_%s_%s" % (os, arch)) _node_configure(name = "nodejs_config_%s_%s" % (os, arch), os = os, arch = arch, nodejs_repository_names = node_repositories) # _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") From 1ec81fad705853343e771abf4e0f49cfb2137095 Mon Sep 17 00:00:00 2001 From: Markus Padourek Date: Mon, 13 May 2019 00:34:17 +0200 Subject: [PATCH 16/19] Fix windows builds. --- toolchains/node/node_configure.bzl | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index c2463865cf..93dfa50ff3 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -25,30 +25,23 @@ def _impl(repository_ctx): host_tool = "" target_repo_name = "" for repo_name in repository_ctx.attr.nodejs_repository_names: - print("-----") - print("name", repository_ctx.attr.name) - print("repo_name", repo_name) - print("repository_ctx.attr.os", repository_ctx.attr.os) - print("host_os", host_os) if repository_ctx.attr.os in repo_name: target_tool = "@%s//:node" % repo_name target_repo_name = repo_name if host_os == "mac os x": - print("host_os new", host_os) host_os = "darwin" + if "windows" in host_os: + host_os = "windows" + if host_os in repo_name: host_tool = "@%s//:node" % repo_name - print("host_tool", host_tool) - print("target_tool", target_tool) - print("target_repo_name", target_repo_name) - print("-----") if not target_tool or not host_tool: - print("fail") - fail("No host_tool nor target_tool found") + fail(("No host_tool for Host OS '%s' and/or no target_tool for provided OS '%s' found with given nodejs" + + " repository names: %s.") % (host_os, repository_ctx.attr.os, repository_ctx.attr.nodejs_repository_names)) substitutions = { "%{ARCH}": "%s" % repository_ctx.attr.arch, From 265b0f41c690197bb38267c943e4b5925e35b77b Mon Sep 17 00:00:00 2001 From: Markus Padourek Date: Mon, 13 May 2019 00:36:22 +0200 Subject: [PATCH 17/19] Remove debug statements. --- toolchains/node/node_configure.bzl | 2 -- 1 file changed, 2 deletions(-) diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index 93dfa50ff3..175957a012 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -159,9 +159,7 @@ def node_configure(node_repositories): ) if node_repositories: - print("!!!!!!!!!InHERE!!!!!!!!!!!!") for os, arch in OS_ARCH_NAMES: - print("nodejs_config_%s_%s" % (os, arch)) _node_configure(name = "nodejs_config_%s_%s" % (os, arch), os = os, arch = arch, nodejs_repository_names = node_repositories) # _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") From 0befde57a5a4b910e8a4558b79d94589806c58d7 Mon Sep 17 00:00:00 2001 From: Markus Padourek Date: Mon, 13 May 2019 01:13:57 +0200 Subject: [PATCH 18/19] Lint fix. --- toolchains/node/node_configure.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index 175957a012..a9328ea3de 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -38,10 +38,9 @@ def _impl(repository_ctx): if host_os in repo_name: host_tool = "@%s//:node" % repo_name - if not target_tool or not host_tool: fail(("No host_tool for Host OS '%s' and/or no target_tool for provided OS '%s' found with given nodejs" + - " repository names: %s.") % (host_os, repository_ctx.attr.os, repository_ctx.attr.nodejs_repository_names)) + " repository names: %s.") % (host_os, repository_ctx.attr.os, repository_ctx.attr.nodejs_repository_names)) substitutions = { "%{ARCH}": "%s" % repository_ctx.attr.arch, From 1b740a5401a673fe3943ad8ab4b78471036d2b3b Mon Sep 17 00:00:00 2001 From: Markus Padourek Date: Tue, 14 May 2019 21:45:38 +0200 Subject: [PATCH 19/19] Some progress. --- internal/node/node_repositories.bzl | 10 +-- toolchains/node/node_configure.bzl | 122 ++++------------------------ 2 files changed, 21 insertions(+), 111 deletions(-) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index f6e33a8154..ef5e5489e4 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -562,12 +562,12 @@ def node_repositories( ) # This needs to be setup so toolchains can access nodejs for all different versions - nodejs_repositorie_names = [] + node_repository_names = [] for os_name in OS_NAMES: - nodejs_repository = "nodejs_%s" % os_name + node_repository_name = "nodejs_%s" % os_name _maybe( _nodejs_repo, - name = nodejs_repository, + name = node_repository_name, package_json = package_json, node_version = node_version, yarn_version = yarn_version, @@ -579,9 +579,9 @@ def node_repositories( yarn_urls = yarn_urls, preserve_symlinks = preserve_symlinks, ) - nodejs_repositorie_names.append(nodejs_repository) + node_repository_names.append(node_repository_name) - node_toolchain_configure(nodejs_repositorie_names) + node_toolchain_configure(node_repository_names) _maybe( _yarn_repo, diff --git a/toolchains/node/node_configure.bzl b/toolchains/node/node_configure.bzl index a9328ea3de..d4525fbdcf 100644 --- a/toolchains/node/node_configure.bzl +++ b/toolchains/node/node_configure.bzl @@ -20,11 +20,11 @@ load("//internal/common:os_name.bzl", "OS_ARCH_NAMES") def _impl(repository_ctx): substitutions = None host_os = repository_ctx.os.name.lower() - if repository_ctx.attr.nodejs_repository_names: + if repository_ctx.attr.node_repository_names: target_tool = "" host_tool = "" target_repo_name = "" - for repo_name in repository_ctx.attr.nodejs_repository_names: + for repo_name in repository_ctx.attr.node_repository_names: if repository_ctx.attr.os in repo_name: target_tool = "@%s//:node" % repo_name target_repo_name = repo_name @@ -40,7 +40,7 @@ def _impl(repository_ctx): if not target_tool or not host_tool: fail(("No host_tool for Host OS '%s' and/or no target_tool for provided OS '%s' found with given nodejs" + - " repository names: %s.") % (host_os, repository_ctx.attr.os, repository_ctx.attr.nodejs_repository_names)) + " repository names: %s.") % (host_os, repository_ctx.attr.os, repository_ctx.attr.node_repository_names)) substitutions = { "%{ARCH}": "%s" % repository_ctx.attr.arch, @@ -80,9 +80,9 @@ _node_configure = repository_rule( doc = "Absolute path to a pre-installed nodejs binary for the host os.", mandatory = False, ), - "nodejs_repository_names": attr.string_list( - mandatory = False, - ), + # "node_repository_names": attr.string_list( + # mandatory = False, + # ), "os": attr.string( mandatory = True, doc = "Default target OS", @@ -99,56 +99,11 @@ _node_configure = repository_rule( }, ) -def _ensure_all_provided(func_name, attrs, kwargs): - """ - For function func_name, ensure either all attributes in 'attrs' were - specified in kwargs or none were specified. - """ - any_specified = False - for key in kwargs.keys(): - if key in attrs: - any_specified = True - break - if not any_specified: - return - provided = [] - missing = [] - for attr in attrs: - if attr in kwargs: - provided.append(attr) - else: - missing.append(attr) - if len(missing) != 0: - fail("Attribute(s) {} are required for function {} because attribute(s) {} were specified.".format( - ", ".join(missing), - func_name, - ", ".join(provided), - )) - -def node_configure(node_repositories): - """ - Creates an external repository with a node_toolchain target - properly configured. +def node_configure(node_repository_name): + """Creates an external repository with a node_toolchain target properly configured. Args: - **kwargs: - Required Args - name: A unique name for this rule. - Default Args: - local_node: Optional. - local_path: Optional. Set to true to build kubectl from sources. Default: False. - Can't be specified if kubectl_path is specified. - k8s_commit: Optional. Commit / release tag at which to build kubectl - from. Default is defined as k8s_tag in :defaults.bzl. - k8s_sha256: Optional. sha256 of commit at which to build kubectl from. - Default is defined as k8s_sha256 in :defaults.bzl. - kubectl_path: Optional. Use the kubectl binary at the given path or label. - This can't be used with 'build_srcs'. - Note: Not all versions/commits of kubernetes project can be used to compile - kubectl from an external repo. Notably, we have only tested with v1.13.0-beta.1 - or above. Note this rule has a hardcoded pointer to io_kubernetes_build repo - if your commit (above v1.13.0-beta.1) does not work due to problems, - related to @io_kubernetes_build repo, please send a PR to update these values. + node_repository_name: list of strings """ native.register_toolchains( @@ -159,55 +114,10 @@ def node_configure(node_repositories): if node_repositories: for os, arch in OS_ARCH_NAMES: - _node_configure(name = "nodejs_config_%s_%s" % (os, arch), os = os, arch = arch, nodejs_repository_names = node_repositories) - - # _node_configure(name = name + "_osx", os="osx", arch="x86_64", host_tool="@nodejs_linux//:node", target_tool="@nodejs_darwin//:node") - - # build_srcs = False - # if "build_srcs" in kwargs and "kubectl_path" in kwargs: - # fail("Attributes 'build_srcs' and 'kubectl_path' can't be specified at" + - # " the same time") - # if "build_srcs" in kwargs and kwargs["build_srcs"]: - # build_srcs = True - # _ensure_all_provided( - # "kubectl_configure", - # ["k8s_commit", "k8s_sha256", "k8s_prefix"], - # kwargs, - # ) - # k8s_commit = kwargs["k8s_commit"] if "k8s_commit" in kwargs else _k8s_commit - # k8s_sha256 = kwargs["k8s_sha256"] if "k8s_sha256" in kwargs else _k8s_sha256 - # k8s_prefix = kwargs["k8s_prefix"] if "k8s_prefix" in kwargs else _k8s_prefix - - # _ensure_all_provided( - # "kubectl_configure", - # ["k8s_repo_tools_sha", "k8s_repo_tools_commit", "k8s_repo_tools_prefix"], - # kwargs, - # ) - # k8s_repo_tools_sha = kwargs["k8s_repo_tools_sha"] if "k8s_repo_tools_sha" in kwargs else _k8s_repo_tools_sha - # k8s_repo_tools_commit = kwargs["k8s_repo_tools_commit"] if "k8s_repo_tools_commit" in kwargs else _k8s_repo_tools_commit - # k8s_repo_tools_prefix = kwargs["k8s_repo_tools_prefix"] if "k8s_repo_tools_prefix" in kwargs else _k8s_repo_tools_prefix - - # http_archive( - # name = "io_kubernetes", - # sha256 = k8s_sha256, - # strip_prefix = k8s_prefix, - # urls = [("https://github.com/{}/{}/archive/{}.tar.gz".format( - # _k8s_org, - # _k8s_repo, - # k8s_commit, - # ))], - # ) - # http_archive( - # name = "io_kubernetes_build", - # sha256 = k8s_repo_tools_sha, - # strip_prefix = k8s_repo_tools_prefix, - # urls = ["https://github.com/{}/{}/archive/{}.tar.gz".format( - # _k8s_org, - # _k8s_repo_tools_repo, - # k8s_repo_tools_commit, - # )], - # ) - # if "kubectl_path" in kwargs: - # _kubectl_configure(name = name, kubectl_path = kwargs["kubectl_path"]) - # else: - # _kubectl_configure(name = name, build_srcs = build_srcs) + _node_configure( + name = "nodejs_config_%s_%s" % (os, arch), + os = os, + arch = arch, + host_tool = "@nodejs//:node" + node_repository_names = node_repositories + )