From dfcfb2795f7ab64eab87632c0b4e8ecb8f543150 Mon Sep 17 00:00:00 2001 From: Corbin McNeely-Smith <58151731+restingbull@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:38:49 -0600 Subject: [PATCH] Make loading the kotlinc repository lazy * Also, cleanup the imports of libraries to be easier to reason about. * .klib remains an open question. Will need ot address in the near future. fixes 1106 --- kotlin/compiler/compiler.bzl | 121 ++++-------------- kotlin/internal/jvm/impl.bzl | 2 +- src/main/starlark/core/repositories/BUILD | 3 +- .../BUILD.com_github_jetbrains_kotlin.bazel | 8 ++ .../BUILD.kotlin_capabilities.bazel | 35 +++++ .../starlark/core/repositories/compiler.bzl | 72 +++++++++-- .../core/repositories/kotlin/artifacts.bzl | 88 +++++++++++++ .../io/bazel/kotlin/builder/tasks/BUILD.bazel | 2 +- 8 files changed, 218 insertions(+), 113 deletions(-) create mode 100644 src/main/starlark/core/repositories/BUILD.kotlin_capabilities.bazel create mode 100644 src/main/starlark/core/repositories/kotlin/artifacts.bzl diff --git a/kotlin/compiler/compiler.bzl b/kotlin/compiler/compiler.bzl index 0fac184d2..25c3c6611 100644 --- a/kotlin/compiler/compiler.bzl +++ b/kotlin/compiler/compiler.bzl @@ -16,8 +16,25 @@ load("@rules_java//java:defs.bzl", "java_import") load("//kotlin:js.bzl", "kt_js_import") load("//kotlin:jvm.bzl", "kt_jvm_import") load("//kotlin/internal:defs.bzl", _KT_COMPILER_REPO = "KT_COMPILER_REPO") - -_KT_COMPILER_REPO_PREFIX = "@" + _KT_COMPILER_REPO + "//:" +load("@com_github_jetbrains_kotlin//:artifacts.bzl", "KOTLINC_ARTIFACTS") + +def _import_artifacts(artifacts, rule_kind): + _import_labels(artifacts.plugin, rule_kind) + _import_labels(artifacts.runtime, rule_kind) + _import_labels(artifacts.compile, rule_kind, neverlink = 1) + +def _import_labels(labels, rule_kind, **rule_args): + for label in labels: + if "-sources" in label: + continue + args = dict(rule_args.items()) + args["visibility"] = ["//visibility:public"] + args["name"] = label + args["jars"] = ["@%s//:%s" % (_KT_COMPILER_REPO, label)] + sources = label + "-sources" + if sources in labels: + args["srcjar"] = "@%s//:%s" % (_KT_COMPILER_REPO, sources) + rule_kind(**args) def kt_configure_compiler(): """ @@ -28,100 +45,6 @@ def kt_configure_compiler(): if native.package_name() != "kotlin/compiler": fail("kt_configure_compiler must be called in kotlin/compiler not %s" % native.package_name()) - kt_jvm_import( - name = "annotations", - jar = _KT_COMPILER_REPO_PREFIX + "lib/annotations-13.0.jar", - neverlink = 1, - ) - - kt_jvm_import( - name = "jvm-abi-gen", - jar = _KT_COMPILER_REPO_PREFIX + "lib/jvm-abi-gen.jar", - ) - - # Kotlin dependencies that are internal to this repo and are meant to be loaded manually into a classloader. - [ - kt_jvm_import( - name = "kotlin-%s" % art, - jar = _KT_COMPILER_REPO_PREFIX + "lib/kotlin-%s.jar" % art, - neverlink = 1, - ) - for art in [ - "annotation-processing", - "annotation-processing-runtime", - "compiler", - ] - ] - - kt_jvm_import( - name = "kotlinx-serialization-compiler-plugin", - jar = _KT_COMPILER_REPO_PREFIX + "lib/kotlinx-serialization-compiler-plugin.jar", - ) - - kt_jvm_import( - name = "allopen-compiler-plugin", - jar = _KT_COMPILER_REPO_PREFIX + "lib/allopen-compiler-plugin.jar", - ) - - kt_jvm_import( - name = "noarg-compiler-plugin", - jar = _KT_COMPILER_REPO_PREFIX + "lib/noarg-compiler-plugin.jar", - ) - - kt_jvm_import( - name = "sam-with-receiver-compiler-plugin", - jar = _KT_COMPILER_REPO_PREFIX + "lib/sam-with-receiver-compiler-plugin.jar", - ) - - kt_jvm_import( - name = "parcelize-compiler-plugin", - jar = _KT_COMPILER_REPO_PREFIX + "lib/parcelize-compiler.jar", - ) - - kt_jvm_import( - name = "parcelize-runtime", - jar = _KT_COMPILER_REPO_PREFIX + "lib/parcelize-runtime.jar", - ) - - # Kotlin dependencies that are internal to this repo and may be linked. - [ - java_import( - name = "kotlin-%s" % art, - jars = [_KT_COMPILER_REPO_PREFIX + "lib/kotlin-%s.jar" % art], - ) - for art in [ - "preloader", - ] - ] - - # The Kotlin standard libraries. These should be setup in a Toolchain. - [ - kt_jvm_import( - name = "kotlin-%s" % art, - jar = _KT_COMPILER_REPO_PREFIX + "lib/kotlin-%s.jar" % art, - srcjar = _KT_COMPILER_REPO_PREFIX + "lib/kotlin-%s-sources.jar" % art, - visibility = ["//visibility:public"], - ) - for art in [ - "stdlib", - "stdlib-jdk7", - "stdlib-jdk8", - "reflect", - "test", - "script-runtime", - ] - ] - - # The Kotlin JS standard libraries. These should be setup in a Toolchain. - [ - kt_js_import( - name = "kotlin-%s" % art, - jars = [_KT_COMPILER_REPO_PREFIX + "lib/kotlin-%s.jar" % art], - srcjar = _KT_COMPILER_REPO_PREFIX + "lib/kotlin-%s-sources.jar" % art, - visibility = ["//visibility:public"], - ) - for art in [ - "test-js", - "stdlib-js", - ] - ] + _import_artifacts(KOTLINC_ARTIFACTS.js, kt_js_import) + _import_artifacts(KOTLINC_ARTIFACTS.jvm, kt_jvm_import) + _import_artifacts(KOTLINC_ARTIFACTS.core, kt_jvm_import) diff --git a/kotlin/internal/jvm/impl.bzl b/kotlin/internal/jvm/impl.bzl index 2f9e8f372..75f28a186 100644 --- a/kotlin/internal/jvm/impl.bzl +++ b/kotlin/internal/jvm/impl.bzl @@ -167,7 +167,7 @@ def _unify_jars(ctx): elif file.basename.endswith(".jar"): jars.append(file) else: - fail("a jar pointing to a filegroup must either end with -sources.jar or .jar") + fail("a jar pointing to a filegroup must either end with -sources.jar or .jar: %s", file) if len(jars) > 1: fail("Got more than one jar, this is an error create an issue: %s" % jars) diff --git a/src/main/starlark/core/repositories/BUILD b/src/main/starlark/core/repositories/BUILD index 567e32505..2bdf154b0 100644 --- a/src/main/starlark/core/repositories/BUILD +++ b/src/main/starlark/core/repositories/BUILD @@ -1,5 +1,3 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") - # Copyright 2018 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +11,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//src/main/starlark/release:packager.bzl", "release_archive") release_archive( diff --git a/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel b/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel index 03467c3dc..6ffe130ac 100644 --- a/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel +++ b/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel @@ -16,6 +16,14 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") package(default_visibility = ["//visibility:public"]) # Kotlin home filegroup containing everything that is needed. +[ + filegroup( + name = name.replace(".", "_"), + srcs = glob(["" + name]), + ) + for name in glob(["lib/**"]) +] + filegroup( name = "home", srcs = glob(["**"]), diff --git a/src/main/starlark/core/repositories/BUILD.kotlin_capabilities.bazel b/src/main/starlark/core/repositories/BUILD.kotlin_capabilities.bazel new file mode 100644 index 000000000..2b6103be2 --- /dev/null +++ b/src/main/starlark/core/repositories/BUILD.kotlin_capabilities.bazel @@ -0,0 +1,35 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load(":artifacts.bzl", "KOTLINC_ARTIFACT_LIST") + +bzl_library( + name = "capabilities", + srcs = glob(["*.bzl"]), +) + +alias( + name = "home", + actual = "@$git_repo$//:home", +) + +[ + alias( + name = label, + actual = "@$git_repo$//:%s" % file.replace(".", "_"), + ) + for (label, file) in KOTLINC_ARTIFACT_LIST.items() +] diff --git a/src/main/starlark/core/repositories/compiler.bzl b/src/main/starlark/core/repositories/compiler.bzl index 724af7de4..2c947c6d0 100644 --- a/src/main/starlark/core/repositories/compiler.bzl +++ b/src/main/starlark/core/repositories/compiler.bzl @@ -1,11 +1,23 @@ +""" +Defines kotlin compiler repositories. +""" + def _kotlin_compiler_impl(repository_ctx): - """Creates the kotlinc repository.""" attr = repository_ctx.attr repository_ctx.download_and_extract( attr.urls, sha256 = attr.sha256, stripPrefix = "kotlinc", ) + repository_ctx.template( + "BUILD.bazel", + attr._template, + executable = False, + ) + +def _kotlin_capabilities_impl(repository_ctx): + """Creates the kotlinc repository.""" + attr = repository_ctx.attr repository_ctx.file( "WORKSPACE", content = """workspace(name = "%s")""" % attr.name, @@ -14,8 +26,15 @@ def _kotlin_compiler_impl(repository_ctx): "BUILD.bazel", attr._template, executable = False, + substitutions = { + "$git_repo$": attr.git_repository_name, + }, + ) + repository_ctx.template( + "artifacts.bzl", + attr._artifacts_template, + executable = False, ) - repository_ctx.template( "capabilities.bzl", _get_capability_template(attr.compiler_version, attr._capabilities_templates), @@ -45,7 +64,31 @@ _CAPABILITIES_TEMPLATES = { "2.0": "//src/main/starlark/core/repositories/kotlin:capabilities_2.0.bzl.com_github_jetbrains_kotlin.bazel", } -kotlin_compiler_repository = repository_rule( +kotlin_capabilities_repository = repository_rule( + implementation = _kotlin_capabilities_impl, + attrs = { + "git_repository_name": attr.string( + doc = "Name of the repository containing kotlin compiler libraries", + ), + "compiler_version": attr.string( + doc = "compiler version", + ), + "_capabilities_templates": attr.label_list( + doc = "compiler capabilities file templates", + default = _CAPABILITIES_TEMPLATES.values(), + ), + "_template": attr.label( + doc = "repository build file template", + default = ":BUILD.kotlin_capabilities.bazel", + ), + "_artifacts_template": attr.label( + doc = "kotlinc artifacts template", + default = "//src/main/starlark/core/repositories/kotlin:artifacts.bzl", + ), + }, +) + +kotlin_compiler_git_repository = repository_rule( implementation = _kotlin_compiler_impl, attrs = { "urls": attr.string_list( @@ -55,16 +98,25 @@ kotlin_compiler_repository = repository_rule( "sha256": attr.string( doc = "sha256 of the compiler archive", ), - "compiler_version": attr.string( - doc = "compiler version", - ), "_template": attr.label( doc = "repository build file template", default = ":BUILD.com_github_jetbrains_kotlin.bazel", ), - "_capabilities_templates": attr.label_list( - doc = "compiler capabilities file templates", - default = _CAPABILITIES_TEMPLATES.values(), - ), }, ) + +def kotlin_compiler_repository(name, urls, sha256, compiler_version): + """ + Creates two repositories, necessary for lazily loading the kotlin compiler binaries for git. + """ + git_repo = name + "_git" + kotlin_compiler_git_repository( + name = git_repo, + urls = urls, + sha256 = sha256, + ) + kotlin_capabilities_repository( + name = name, + git_repository_name = git_repo, + compiler_version = compiler_version, + ) diff --git a/src/main/starlark/core/repositories/kotlin/artifacts.bzl b/src/main/starlark/core/repositories/kotlin/artifacts.bzl new file mode 100644 index 000000000..b46bc5243 --- /dev/null +++ b/src/main/starlark/core/repositories/kotlin/artifacts.bzl @@ -0,0 +1,88 @@ +"""A map of label to artifacts made available by the kotlinc github repo""" + +KOTLINC_ARTIFACTS = struct( + js = struct( + plugin = {}, + runtime = { + "kotlin-stdlib-js": "lib/kotlin-stdlib-js.jar", + "kotlin-stdlib-js-sources": "lib/kotlin-stdlib-js-sources.jar", + "kotlin-test-js": "lib/kotlin-test-js.jar", + "kotlin-test-js-sources": "lib/kotlin-test-js-sources.jar", + }, + compile = {}, + ), + jvm = struct( + plugin = { + "allopen-compiler-plugin": "lib/allopen-compiler-plugin.jar", + "assignment-compiler-plugin": "lib/assignment-compiler-plugin.jar", + "kotlin-imports-dumper-compiler-plugin": "lib/kotlin-imports-dumper-compiler-plugin.jar", + "kotlin-serialization-compiler-plugin": "lib/kotlin-serialization-compiler-plugin.jar", + "kotlinx-serialization-compiler-plugin": "lib/kotlinx-serialization-compiler-plugin.jar", + "lombok-compiler-plugin": "lib/lombok-compiler-plugin.jar", + "mutability-annotations-compat": "lib/mutability-annotations-compat.jar", + "noarg-compiler-plugin": "lib/noarg-compiler-plugin.jar", + "sam-with-receiver-compiler-plugin": "lib/sam-with-receiver-compiler-plugin.jar", + }, + runtime = { + "jvm-abi-gen": "lib/jvm-abi-gen.jar", + "kotlin-stdlib": "lib/kotlin-stdlib.jar", + "kotlin-stdlib-jdk7": "lib/kotlin-stdlib-jdk7.jar", + "kotlin-stdlib-jdk7-sources": "lib/kotlin-stdlib-jdk7-sources.jar", + "kotlin-stdlib-jdk8": "lib/kotlin-stdlib-jdk8.jar", + "kotlin-stdlib-jdk8-sources": "lib/kotlin-stdlib-jdk8-sources.jar", + "kotlin-stdlib-sources": "lib/kotlin-stdlib-sources.jar", + "kotlin-test-junit": "lib/kotlin-test-junit.jar", + "kotlin-test-junit-sources": "lib/kotlin-test-junit-sources.jar", + "kotlin-test-junit5": "lib/kotlin-test-junit5.jar", + "kotlin-test-junit5-sources": "lib/kotlin-test-junit5-sources.jar", + "kotlin-test-testng": "lib/kotlin-test-testng.jar", + "parcelize-runtime": "lib/parcelize-runtime.jar", + }, + compile = {}, + ), + core = struct( + plugin = {}, + runtime = { + "kotlin-reflect": "lib/kotlin-reflect.jar", + "kotlin-reflect-sources": "lib/kotlin-reflect-sources.jar", + "kotlin-script-runtime": "lib/kotlin-script-runtime.jar", + "kotlin-script-runtime-sources": "lib/kotlin-script-runtime-sources.jar", + "kotlin-test": "lib/kotlin-test.jar", + "kotlin-test-sources": "lib/kotlin-test-sources.jar", + "kotlin-test-testng-sources": "lib/kotlin-test-testng-sources.jar", + "kotlin-preloader": "lib/kotlin-preloader.jar", + }, + compile = { + "android-extensions-compiler": "lib/android-extensions-compiler.jar", + "android-extensions-runtime": "lib/android-extensions-runtime.jar", + "annotations": "lib/annotations-13_0.jar", + "js_engines": "lib/js_engines.jar", + "kotlin-annotation-processing": "lib/kotlin-annotation-processing.jar", + "kotlin-annotation-processing-cli": "lib/kotlin-annotation-processing-cli.jar", + "kotlin-annotation-processing-compiler": "lib/kotlin-annotation-processing-compiler.jar", + "kotlin-annotation-processing-runtime": "lib/kotlin-annotation-processing-runtime.jar", + "kotlin-annotations-jvm": "lib/kotlin-annotations-jvm.jar", + "kotlin-annotations-jvm-sources": "lib/kotlin-annotations-jvm-sources.jar", + "kotlin-compiler": "lib/kotlin-compiler.jar", + "kotlin-daemon": "lib/kotlin-daemon.jar", + "kotlin-daemon-client": "lib/kotlin-daemon-client.jar", + "kotlin-main-kts": "lib/kotlin-main-kts.jar", + "kotlin-runner": "lib/kotlin-runner.jar", + "kotlin-scripting-common": "lib/kotlin-scripting-common.jar", + "kotlin-scripting-compiler": "lib/kotlin-scripting-compiler.jar", + "kotlin-scripting-compiler-impl": "lib/kotlin-scripting-compiler-impl.jar", + "kotlin-scripting-jvm": "lib/kotlin-scripting-jvm.jar", + "kotlinx-coroutines-core-jvm": "lib/kotlinx-coroutines-core-jvm.jar", + "parcelize-compiler": "lib/parcelize-compiler.jar", + "scripting-compiler": "lib/scripting-compiler.jar", + "trove4j": "lib/trove4j.jar", + }, + ), +) + +KOTLINC_ARTIFACT_LIST = { + label: file + for lang in ["js", "jvm", "core"] + for type in ["compile", "plugin", "runtime"] + for (label, file) in getattr(getattr(KOTLINC_ARTIFACTS, lang), type).items() +} diff --git a/src/test/kotlin/io/bazel/kotlin/builder/tasks/BUILD.bazel b/src/test/kotlin/io/bazel/kotlin/builder/tasks/BUILD.bazel index fb51348e7..f126be8f6 100644 --- a/src/test/kotlin/io/bazel/kotlin/builder/tasks/BUILD.bazel +++ b/src/test/kotlin/io/bazel/kotlin/builder/tasks/BUILD.bazel @@ -103,7 +103,7 @@ kt_rules_test( name = "KotlinBuilderJsTest", srcs = ["js/KotlinBuilderJsTest.java"], data = [ - "@com_github_jetbrains_kotlin//:lib/kotlin-stdlib-js.jar", + "//kotlin/compiler:kotlin-stdlib-js.js", ], )