From 507ba36e299599ceca0c537aa95e416c996f1fcc Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 16 Jul 2024 03:38:36 -0700 Subject: [PATCH] Support building //java with bzlmod Create WORKSPACE.bzlmod. Before building with Bzlmod resulted in use of full WORKSPACE. Some repos are still there, but the file should eventually be empty. Add dep to rules_kotlin 1.9.0. This was the first version available on BCR. It pushed upgrade of rules_jvm_external to 6.0 and rules_java to 6.5.2 (keep 6.0.0 on Bazel 6.3.0). Add missing maven and other deps to MODULE.bazel CI changes: Disable Bazel 6.4.0 with bzlmod. rules_jvm_external 6.0 use use_repo_rule, which is not supported by Bazel 6. Add C++ build "Bazel7 with Bzlmod" enabled. Add Java builds with "Bazel 7 with/without Bzlmod". Fixes: https://github.com/protocolbuffers/protobuf/issues/17176 PiperOrigin-RevId: 652773197 --- .github/workflows/test_bazel.yml | 4 +-- .github/workflows/test_cpp.yml | 14 ++++++++-- .github/workflows/test_java.yml | 12 ++++++++- MODULE.bazel | 27 +++++++++++++++++++- WORKSPACE | 4 +-- WORKSPACE.bzlmod | 40 +++++++++++++++++++++++++++++ java/kotlin-lite/BUILD.bazel | 6 ++--- java/kotlin/BUILD.bazel | 10 ++++---- protobuf_deps.bzl | 44 ++++++++++++++++++++++---------- 9 files changed, 131 insertions(+), 30 deletions(-) create mode 100644 WORKSPACE.bzlmod diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml index 0569790b9bac..a61c66271141 100644 --- a/.github/workflows/test_bazel.yml +++ b/.github/workflows/test_bazel.yml @@ -22,9 +22,7 @@ jobs: include: - runner: ubuntu bazelversion: '6.4.0' - bzlmod: true - - runner: ubuntu - bazelversion: '6.4.0' + # Not running Bazel 6 with bzlmod, because it doesn't support use_repo_rule in rules_jvm_external bzlmod: false runs-on: ${{ matrix.runner }}-latest name: Examples ${{ matrix.runner }} ${{ matrix.bazelversion }}${{ matrix.bzlmod && ' (bzlmod)' || '' }} diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 35ecae8633b6..cc35286684b5 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -30,13 +30,22 @@ jobs: - targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/... # Override cases with custom images - - config: { name: "Bazel7" } + - config: { name: "Bazel7", flags: --noenable_bzlmod } + version: Bazel7 image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e" targets: "//src/... //third_party/utf8_range/..." + - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod } + version: Bazel7bzlmod + image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e" + targets: "//src/... //third_party/utf8_range/..." + # TODO: enable back command_line_interface_unittest after bug is fixed + exclude-targets: "-//src/google/protobuf/compiler:command_line_interface_unittest" - config: { name: "TCMalloc" } + version: TcMalloc image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:1c5133455481f4d1bb8afa477029604f41f1a3c46cebe4d9958cf1af95b5c87c" targets: "//src/... //third_party/utf8_range/..." - config: { name: "aarch64" } + version: TcMalloc targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test //third_party/utf8_range/..." image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:6.3.0-aarch64-68e662b3a56b881804dc4e9d45f949791cbc4b94" name: Linux ${{ matrix.config.name }} @@ -51,8 +60,9 @@ jobs: with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel-cache: cpp_linux/${{ matrix.config.name }} + bazel-cache: cpp_linux/${{ matrix.version }} bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} + exclude-targets: ${{ matrix.exclude-targets }} linux-gcc: strategy: diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index e0cb69558f02..9762ddc94bb2 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -31,6 +31,16 @@ jobs: version: '17' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 targets: //java/... //java/internal:java_version //compatibility/... + - name: Bazel7 + version: 'bazel7nobzlmod' + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e + targets: //java/... //java/internal:java_version //compatibility/... + flags: --noenable_bzlmod + - name: Bazel7 with Bzlmod + version: 'bazel7bzlmod' + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e + targets: //java/... //java/internal:java_version //compatibility/... + flags: --enable_bzlmod - name: aarch64 version: 'aarch64' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17 @@ -49,7 +59,7 @@ jobs: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: java_linux/${{ matrix.version }} - bazel: test ${{ matrix.targets }} --test_env=KOKORO_JAVA_VERSION + bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_JAVA_VERSION # TODO restore this test (or a better one) when gRPC has rebuilt with 26.x # linkage-monitor: diff --git a/MODULE.bazel b/MODULE.bazel index f370bfe8c2d5..b9f9fbf24dbe 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,7 +18,8 @@ bazel_dep(name = "jsoncpp", version = "1.9.5") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "rules_fuzzing", version = "0.5.2") bazel_dep(name = "rules_java", version = "5.3.5") -bazel_dep(name = "rules_jvm_external", version = "5.1") +bazel_dep(name = "rules_jvm_external", version = "6.0") +bazel_dep(name = "rules_kotlin", version = "1.9.0") bazel_dep(name = "rules_pkg", version = "0.7.0") bazel_dep(name = "rules_python", version = "0.28.0") bazel_dep(name = "rules_rust", version = "0.45.1") @@ -72,5 +73,29 @@ crate.spec( crate.from_specs() use_repo(crate, crate_index = "crates") +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +maven.install( + artifacts = [ + "com.google.caliper:caliper:1.0-beta-3", + "com.google.code.findbugs:jsr305:3.0.2", + "com.google.code.gson:gson:2.8.9", + "com.google.errorprone:error_prone_annotations:2.5.1", + "com.google.j2objc:j2objc-annotations:2.8", + "com.google.guava:guava:32.0.1-jre", + "com.google.guava:guava-testlib:32.0.1-jre", + "com.google.truth:truth:1.1.2", + "junit:junit:4.13.2", + "org.mockito:mockito-core:4.3.1", + "biz.aQute.bnd:biz.aQute.bndlib:6.4.0", + "info.picocli:picocli:4.6.3", + ], + repositories = [ + "https://repo1.maven.org/maven2", + "https://repo.maven.apache.org/maven2", + ], +) +use_repo(maven, "maven") + # Development dependencies +bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest", dev_dependency = True) bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True) diff --git a/WORKSPACE b/WORKSPACE index e1b761e3af71..5ca0070466cb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -99,11 +99,11 @@ load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies") rules_cc_dependencies() # For `kt_jvm_library` -load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") +load("@rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") kotlin_repositories() -load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") +load("@rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") kt_register_toolchains() diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod new file mode 100644 index 000000000000..d6e152777b60 --- /dev/null +++ b/WORKSPACE.bzlmod @@ -0,0 +1,40 @@ +# This is a WORKSPACE file used by bzlmod in combination with MODULE.bazel. +# It's used for a gradual migration and it should be empty. +# Don't remove this file. If the file doesn't exist, bzlmod falls back to WORKSPACE file. + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# TODO: either replace rules_ruby with a maintained version on BCR +# or use bzlmod extensions to depend on this specific repo +http_archive( + name = "rules_ruby", + urls = [ + "https://github.com/protocolbuffers/rules_ruby/archive/b7f3e9756f3c45527be27bc38840d5a1ba690436.zip" + ], + strip_prefix = "rules_ruby-b7f3e9756f3c45527be27bc38840d5a1ba690436", + sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8", +) + +load("@rules_ruby//ruby:defs.bzl", "ruby_runtime") + +ruby_runtime("system_ruby") + +register_toolchains("@system_ruby//:toolchain") + +# Follwing are just needed to run conformance tests, not really needed to support them via MODULE.bazel + +# For testing runtime against old gencode from a previous major version. +http_archive( + name = "com_google_protobuf_v25.0", + strip_prefix = "protobuf-25.0", + url = "https://github.com/protocolbuffers/protobuf/releases/download/v25.0/protobuf-25.0.tar.gz", +) + +# Needed as a dependency of @com_google_protobuf_v25.x, which was before +# utf8_range was merged in. +http_archive( + name = "utf8_range", + strip_prefix = "utf8_range-d863bc33e15cba6d873c878dcca9e6fe52b2f8cb", + url = "https://github.com/protocolbuffers/utf8_range/archive/d863bc33e15cba6d873c878dcca9e6fe52b2f8cb.zip", +) + diff --git a/java/kotlin-lite/BUILD.bazel b/java/kotlin-lite/BUILD.bazel index 362a8bf3051d..ed3e1c4c1de2 100644 --- a/java/kotlin-lite/BUILD.bazel +++ b/java/kotlin-lite/BUILD.bazel @@ -1,5 +1,5 @@ -load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_jvm_external//:kt_defs.bzl", "kt_jvm_export") +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("//:protobuf.bzl", "internal_gen_kt_protos") load("//:protobuf_version.bzl", "PROTOBUF_JAVA_VERSION") @@ -50,7 +50,7 @@ kt_jvm_library( kt_jvm_export( name = "kotlin-lite_mvn", deploy_env = [ - "@com_github_jetbrains_kotlin//:kotlin-stdlib", + "@rules_kotlin//kotlin/compiler:kotlin-stdlib", "//java/lite", ], maven_coordinates = "com.google.protobuf:protobuf-kotlin-lite:%s" % PROTOBUF_JAVA_VERSION, @@ -99,9 +99,9 @@ kt_jvm_library( "//java/kotlin:only_for_use_in_proto_generated_code_its_generator_and_tests", "//java/kotlin:shared_runtime", "//java/lite", - "@com_github_jetbrains_kotlin//:kotlin-test", "@maven//:com_google_truth_truth", "@maven//:junit_junit", + "@rules_kotlin//kotlin/compiler:kotlin-test", ], ) diff --git a/java/kotlin/BUILD.bazel b/java/kotlin/BUILD.bazel index 20dd242aea6f..3af561b74d54 100644 --- a/java/kotlin/BUILD.bazel +++ b/java/kotlin/BUILD.bazel @@ -1,5 +1,5 @@ -load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_jvm_external//:kt_defs.bzl", "kt_jvm_export") +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("//:protobuf.bzl", "internal_gen_kt_protos") load("//:protobuf_version.bzl", "PROTOBUF_JAVA_VERSION") @@ -53,7 +53,7 @@ kt_jvm_library( kt_jvm_export( name = "kotlin_mvn", deploy_env = [ - "@com_github_jetbrains_kotlin//:kotlin-stdlib", + "@rules_kotlin//kotlin/compiler:kotlin-stdlib", "//java/core", ], maven_coordinates = "com.google.protobuf:protobuf-kotlin:%s" % PROTOBUF_JAVA_VERSION, @@ -101,9 +101,9 @@ kt_jvm_library( deps = [ ":bytestring_lib", "//java/lite", - "@com_github_jetbrains_kotlin//:kotlin-test", "@maven//:com_google_truth_truth", "@maven//:junit_junit", + "@rules_kotlin//kotlin/compiler:kotlin-test", ], ) @@ -136,10 +136,10 @@ kt_jvm_library( ":example_extensible_message_java_proto", ":only_for_use_in_proto_generated_code_its_generator_and_tests", ":shared_runtime", - "@com_github_jetbrains_kotlin//:kotlin-test", "@maven//:com_google_guava_guava_testlib", "@maven//:com_google_truth_truth", "@maven//:junit_junit", + "@rules_kotlin//kotlin/compiler:kotlin-test", ], ) @@ -162,9 +162,9 @@ kt_jvm_library( ":only_for_use_in_proto_generated_code_its_generator_and_tests", ":shared_runtime", "//java/core", - "@com_github_jetbrains_kotlin//:kotlin-test", "@maven//:com_google_truth_truth", "@maven//:junit_junit", + "@rules_kotlin//kotlin/compiler:kotlin-test", ], ) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 92ce54fb771e..9dcf0c042887 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -98,11 +98,27 @@ def protobuf_deps(): ) if not native.existing_rule("rules_java"): - http_archive( - name = "rules_java", - url = "https://github.com/bazelbuild/rules_java/releases/download/6.0.0/rules_java-6.0.0.tar.gz", - sha256 = "469b7f3b580b4fcf8112f4d6d0d5a4ce8e1ad5e21fee67d8e8335d5f8b3debab", - ) + bazel_version = native.bazel_version or "999999.999999.999999" + version_parts = bazel_version.split("-")[0].split(".") + if len(version_parts) != 3: + fail("invalid Bazel version '{}': got {} dot-separated segments, want 3".format(bazel_version, len(version_parts))) + major_version_int = int(version_parts[0]) + minor_version_int = int(version_parts[1]) + + if major_version_int < 6 or (major_version_int == 6 and minor_version_int <= 3): + # Works with Bazel 6.3.0, but not higher + http_archive( + name = "rules_java", + url = "https://github.com/bazelbuild/rules_java/releases/download/6.0.0/rules_java-6.0.0.tar.gz", + sha256 = "469b7f3b580b4fcf8112f4d6d0d5a4ce8e1ad5e21fee67d8e8335d5f8b3debab", + ) + else: + # Version 6.5.2 works both with Bazel 6.4.0 and Bazel 7 + http_archive( + name = "rules_java", + url = "https://github.com/bazelbuild/rules_java/releases/download/6.5.0/rules_java-6.5.0.tar.gz", + sha256 = "160d1ebf33763124766fb35316329d907ca67f733238aa47624a8e3ff3cf2ef4", + ) # TODO: remove after toolchain types are moved to protobuf if not native.existing_rule("rules_proto"): @@ -133,11 +149,12 @@ def protobuf_deps(): ) if not native.existing_rule("rules_jvm_external"): - _github_archive( + # Version 6.0 is the lowest that works with rules_kotlin 1.9.0 + http_archive( name = "rules_jvm_external", - repo = "https://github.com/bazelbuild/rules_jvm_external", - commit = "906875b0d5eaaf61a8ca2c9c3835bde6f435d011", - sha256 = "744bd7436f63af7e9872948773b8b106016dc164acb3960b4963f86754532ee7", + strip_prefix = "rules_jvm_external-6.0", + sha256 = "85fd6bad58ac76cc3a27c8e051e4255ff9ccd8c92ba879670d195622e7c0a9b7", + url = "https://github.com/bazelbuild/rules_jvm_external/releases/download/6.0/rules_jvm_external-6.0.tar.gz", ) if not native.existing_rule("rules_pkg"): @@ -164,11 +181,12 @@ def protobuf_deps(): url = "https://github.com/bazelbuild/apple_support/releases/download/1.12.0/apple_support.1.12.0.tar.gz", ) - if not native.existing_rule("io_bazel_rules_kotlin"): + if not native.existing_rule("rules_kotlin"): + # Version 1.9.0 is the lowest available on BCR http_archive( - name = "io_bazel_rules_kotlin", - urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v1.8.1/rules_kotlin_release.tgz"], - sha256 = "a630cda9fdb4f56cf2dc20a4bf873765c41cf00e9379e8d59cd07b24730f4fde", + name = "rules_kotlin", + sha256 = "5766f1e599acf551aa56f49dab9ab9108269b03c557496c54acaf41f98e2b8d6", + url = "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.0/rules_kotlin-v1.9.0.tar.gz", ) # Python Downloads