[6.5.0] Proto toolchainisation cherrypicks (#20925)

Fixes: #20921 List of cherrypicks: 5a24c8a [email protected] Mon Oct 9 02:27:10 2023 -0700 Support automatic exec groups in proto_common.compile 3e1e061 [email protected] Tue Oct 3 03:32:01 2023 -0700 Use proto toolchains in cc_proto_library 8c38be3 [email protected] Mon Oct 2 07:08:55 2023 -0700 Use proto toolchains in java_lite_proto_library 51970d2 [email protected] Mon Oct 2 07:05:16 2023 -0700 Use proto toolchains in java_proto_library 20bc11f [email protected] Mon Oct 2 06:56:47 2023 -0700 Decouple java_lite_proto_library from java_proto_library 3b18d3f [email protected] Mon Oct 2 04:11:24 2023 -0700 Refactor proto toolchainsation support utilities 42800a8 [email protected] Fri Sep 22 03:04:18 2023 -0700 Use proto compiler from proto_toolchain rule d435c6d [email protected] Thu Sep 21 07:05:40 2023 -0700 Use proto_toolchain in proto_library f5fb2f6 [email protected] Wed Sep 20 05:50:59 2023 -0700 Remove protoc from proto_lang_toolchain rule 108ef55 [email protected] Tue Sep 19 08:40:00 2023 -0700 Use MockProtoSupport.setup where protos are used 11cf1b7 [email protected] Sun Sep 17 21:57:54 2023 -0700 Implement incompatible_enable_proto_toolchain_resolution
bazelbuild · Jan 18, 2024 · d2daa9b · d2daa9b
1 parent 29bbe31
commit d2daa9b
Show file tree

Hide file tree

Showing 29 changed files with 373 additions and 231 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java
@@ -696,6 +696,18 @@ public final class BuildLanguageOptions extends OptionsBase {
               + " Label.relative) can be used.")
   public boolean enableDeprecatedLabelApis;
 
+  // Flip when dependencies to rules_* repos are upgraded and protobuf registers toolchains
+  @Option(
+      name = "incompatible_enable_proto_toolchain_resolution",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.TOOLCHAIN,
+      effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS},
+      metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE},
+      help =
+          "If true, proto lang rules define toolchains from rules_proto, rules_java, rules_cc"
+              + " repositories.")
+  public boolean incompatibleEnableProtoToolchainResolution;
+
   /**
    * An interner to reduce the number of StarlarkSemantics instances. A single Blaze instance should
    * never accumulate a large number of these and being able to shortcut on object identity makes a
@@ -795,6 +807,9 @@ public StarlarkSemantics toStarlarkSemantics() {
                 INCOMPATIBLE_DISABLE_OBJC_LIBRARY_TRANSITION,
                 incompatibleDisableObjcLibraryTransition)
             .setBool(INCOMPATIBLE_ENABLE_DEPRECATED_LABEL_APIS, enableDeprecatedLabelApis)
+            .setBool(
+                INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION,
+                incompatibleEnableProtoToolchainResolution)
             .build();
     return INTERNER.intern(semantics);
   }
@@ -891,6 +906,8 @@ public StarlarkSemantics toStarlarkSemantics() {
       "-incompatible_disable_objc_library_transition";
   public static final String INCOMPATIBLE_ENABLE_DEPRECATED_LABEL_APIS =
       "+incompatible_enable_deprecated_label_apis";
+  public static final String INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION =
+      "-incompatible_enable_proto_toolchain_resolution";
 
   // non-booleans
   public static final StarlarkSemantics.Key<String> EXPERIMENTAL_BUILTINS_BZL_PATH =

diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/BUILD b/src/main/java/com/google/devtools/build/lib/rules/proto/BUILD
@@ -40,6 +40,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
         "//src/main/java/com/google/devtools/build/lib/concurrent",
         "//src/main/java/com/google/devtools/build/lib/packages",
+        "//src/main/java/com/google/devtools/build/lib/packages/semantics",
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi",
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/proto",
         "//src/main/java/com/google/devtools/build/lib/util:filetype",

diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/BazelProtoCommon.java b/src/main/java/com/google/devtools/build/lib/rules/proto/BazelProtoCommon.java
@@ -22,6 +22,12 @@
 import net.starlark.java.annot.StarlarkMethod;
 import net.starlark.java.eval.EvalException;
 import net.starlark.java.eval.StarlarkList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.packages.BuiltinRestriction;
+import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
+import com.google.devtools.build.lib.starlarkbuildapi.proto.ProtoCommonApi;
+import net.starlark.java.annot.StarlarkMethod;
+import net.starlark.java.eval.EvalException;
 import net.starlark.java.eval.StarlarkThread;
 
 /** Protocol buffers support for Starlark. */
@@ -87,4 +93,15 @@ public ProtoInfo protoInfo(
         Depset.cast(transitiveDescriptorSets, Artifact.class, "transitive_descriptor_set"),
         Depset.cast(exportedSources, ProtoSource.class, "exported_sources"));
   }
+
+  @StarlarkMethod(
+      name = "incompatible_enable_proto_toolchain_resolution",
+      useStarlarkThread = true,
+      documented = false)
+  public boolean getDefineProtoToolchains(StarlarkThread thread) throws EvalException {
+    ProtoCommon.checkPrivateStarlarkificationAllowlist(thread);
+    return thread
+        .getSemantics()
+        .getBool(BuildLanguageOptions.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION);
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConfiguration.java
@@ -31,6 +31,7 @@
 import com.google.devtools.common.options.OptionEffectTag;
 import com.google.devtools.common.options.OptionMetadataTag;
 import java.util.List;
+import javax.annotation.Nullable;
 import net.starlark.java.annot.StarlarkMethod;
 import net.starlark.java.eval.EvalException;
 import net.starlark.java.eval.StarlarkThread;
@@ -65,13 +66,12 @@ public static class Options extends FragmentOptions {
     public List<String> protocOpts;
 
     @Option(
-      name = "experimental_proto_extra_actions",
-      defaultValue = "false",
-      documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION,
-      effectTags = {OptionEffectTag.AFFECTS_OUTPUTS, OptionEffectTag.LOADING_AND_ANALYSIS},
-      metadataTags = {OptionMetadataTag.EXPERIMENTAL},
-      help = "Run extra actions for alternative Java api versions in a proto_library."
-    )
+        name = "experimental_proto_extra_actions",
+        defaultValue = "false",
+        documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION,
+        effectTags = {OptionEffectTag.AFFECTS_OUTPUTS, OptionEffectTag.LOADING_AND_ANALYSIS},
+        metadataTags = {OptionMetadataTag.EXPERIMENTAL},
+        help = "Run extra actions for alternative Java api versions in a proto_library.")
     public boolean experimentalProtoExtraActions;
 
     @Option(
@@ -240,8 +240,8 @@ public boolean experimentalProtoDescriptorSetsIncludeSourceInfo() {
   }
 
   /**
-   * Returns true if we will run extra actions for actions that are not run by default. If this
-   * is enabled, e.g. all extra_actions for alternative api-versions or language-flavours of a
+   * Returns true if we will run extra actions for actions that are not run by default. If this is
+   * enabled, e.g. all extra_actions for alternative api-versions or language-flavours of a
    * proto_library target are run.
    */
   public boolean runExperimentalProtoExtraActions() {
@@ -252,6 +252,7 @@ public boolean runExperimentalProtoExtraActions() {
       name = "proto_compiler",
       doc = "Label for the proto compiler.",
       defaultLabel = ProtoConstants.DEFAULT_PROTOC_LABEL)
+  @Nullable
   public Label protoCompiler() {
     return options.protoCompiler;
   }
@@ -260,10 +261,12 @@ public Label protoCompiler() {
       name = "proto_toolchain_for_java",
       doc = "Label for the java proto toolchains.",
       defaultLabel = ProtoConstants.DEFAULT_JAVA_PROTO_LABEL)
+  @Nullable
   public Label protoToolchainForJava() {
     return options.protoToolchainForJava;
   }
 
+  @Nullable
   public Label protoToolchainForJ2objc() {
     return options.protoToolchainForJ2objc;
   }
@@ -272,6 +275,7 @@ public Label protoToolchainForJ2objc() {
       name = "proto_toolchain_for_java_lite",
       doc = "Label for the java lite proto toolchains.",
       defaultLabel = ProtoConstants.DEFAULT_JAVA_LITE_PROTO_LABEL)
+  @Nullable
   public Label protoToolchainForJavaLite() {
     return options.protoToolchainForJavaLite;
   }
@@ -280,6 +284,7 @@ public Label protoToolchainForJavaLite() {
       name = "proto_toolchain_for_cc",
       doc = "Label for the cc proto toolchains.",
       defaultLabel = ProtoConstants.DEFAULT_CC_PROTO_LABEL)
+  @Nullable
   public Label protoToolchainForCc() {
     return options.protoToolchainForCc;
   }

diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConstants.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConstants.java
@@ -17,7 +17,7 @@
 /** Constants used in Proto rules. */
 public final class ProtoConstants {
   /** Default label for proto compiler. */
-  static final String DEFAULT_PROTOC_LABEL =  "@bazel_tools//tools/proto:protoc";
+  public static final String DEFAULT_PROTOC_LABEL =  "@bazel_tools//tools/proto:protoc";
 
   /** Default label for java proto toolchains. */
   static final String DEFAULT_JAVA_PROTO_LABEL = "@bazel_tools//tools/proto:java_toolchain";

diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_proto_library.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_proto_library.bzl
@@ -15,7 +15,8 @@
 """Starlark implementation of cc_proto_library"""
 
 load(":common/cc/cc_helper.bzl", "cc_helper")
-load(":common/proto/proto_common.bzl", "ProtoLangToolchainInfo", proto_common = "proto_common_do_not_use")
+load(":common/proto/proto_common.bzl", "toolchains", "ProtoLangToolchainInfo", proto_common = "proto_common_do_not_use")
+load(":common/cc/semantics.bzl", "semantics")
 
 ProtoInfo = _builtins.toplevel.ProtoInfo
 CcInfo = _builtins.toplevel.CcInfo
@@ -24,15 +25,15 @@ cc_common = _builtins.toplevel.cc_common
 ProtoCcFilesInfo = provider(fields = ["files"], doc = "Provide cc proto files.")
 ProtoCcHeaderInfo = provider(fields = ["headers"], doc = "Provide cc proto headers.")
 
-def _are_srcs_excluded(ctx, target):
-    return not proto_common.experimental_should_generate_code(target[ProtoInfo], ctx.attr._aspect_cc_proto_toolchain[ProtoLangToolchainInfo], "cc_proto_library", target.label)
+def _are_srcs_excluded(proto_toolchain, target):
+    return not proto_common.experimental_should_generate_code(target[ProtoInfo], proto_toolchain, "cc_proto_library", target.label)
 
-def _get_feature_configuration(ctx, target, cc_toolchain, proto_info):
+def _get_feature_configuration(ctx, target, cc_toolchain, proto_info, proto_toolchain):
     requested_features = list(ctx.features)
     unsupported_features = list(ctx.disabled_features)
     unsupported_features.append("parse_headers")
     unsupported_features.append("layering_check")
-    if not _are_srcs_excluded(ctx, target) and len(proto_info.direct_sources) != 0:
+    if not _are_srcs_excluded(proto_toolchain, target) and len(proto_info.direct_sources) != 0:
         requested_features.append("header_modules")
     else:
         unsupported_features.append("header_modules")
@@ -78,12 +79,13 @@ def _get_strip_include_prefix(ctx, proto_info):
 
 def _aspect_impl(target, ctx):
     cc_toolchain = cc_helper.find_cpp_toolchain(ctx)
+    proto_toolchain = toolchains.find_toolchain(ctx, "_aspect_cc_proto_toolchain", semantics.CC_PROTO_TOOLCHAIN)
     proto_info = target[ProtoInfo]
-    feature_configuration = _get_feature_configuration(ctx, target, cc_toolchain, proto_info)
+    feature_configuration = _get_feature_configuration(ctx, target, cc_toolchain, proto_info, proto_toolchain)
     proto_configuration = ctx.fragments.proto
 
     deps = []
-    runtime = ctx.attr._aspect_cc_proto_toolchain[ProtoLangToolchainInfo].runtime
+    runtime = proto_toolchain.runtime
     if runtime != None:
         deps.append(runtime)
     deps.extend(getattr(ctx.rule.attr, "deps", []))
@@ -100,7 +102,7 @@ def _aspect_impl(target, ctx):
     textual_hdrs = []
     additional_exported_hdrs = []
 
-    if _are_srcs_excluded(ctx, target):
+    if _are_srcs_excluded(proto_toolchain, target):
         header_provider = None
 
         # Hack: This is a proto_library for descriptor.proto or similar.
@@ -155,7 +157,7 @@ def _aspect_impl(target, ctx):
     proto_common.compile(
         actions = ctx.actions,
         proto_info = proto_info,
-        proto_lang_toolchain_info = ctx.attr._aspect_cc_proto_toolchain[ProtoLangToolchainInfo],
+        proto_lang_toolchain_info = proto_toolchain,
         generated_files = outputs,
         experimental_output_files = "multiple",
     )
@@ -250,12 +252,11 @@ cc_proto_aspect = aspect(
     required_providers = [ProtoInfo],
     provides = [CcInfo],
     attrs = {
-        "_aspect_cc_proto_toolchain": attr.label(
-            default = configuration_field(fragment = "proto", name = "proto_toolchain_for_cc"),
-        ),
         "_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),
-    },
-    toolchains = cc_helper.use_cpp_toolchain(),
+    } | toolchains.if_legacy_toolchain({"_aspect_cc_proto_toolchain": attr.label(
+        default = configuration_field(fragment = "proto", name = "proto_toolchain_for_cc"),
+    )}),
+    toolchains = cc_helper.use_cpp_toolchain() + toolchains.use_toolchain(semantics.CC_PROTO_TOOLCHAIN),
 )
 
 def _impl(ctx):
@@ -283,4 +284,5 @@ cc_proto_library = rule(
         ),
     },
     provides = [CcInfo],
+    toolchains = toolchains.use_toolchain(semantics.CC_PROTO_TOOLCHAIN),
 )
diff --git a/src/main/starlark/builtins_bzl/common/cc/semantics.bzl b/src/main/starlark/builtins_bzl/common/cc/semantics.bzl
@@ -198,4 +198,5 @@ semantics = struct(
     get_coverage_env = _get_coverage_env,
     get_proto_aspects = _get_proto_aspects,
     incompatible_disable_objc_library_transition = _incompatible_disable_objc_library_transition,
+    CC_PROTO_TOOLCHAIN = "@rules_cc//cc/proto:toolchain_type",
 )
diff --git a/src/main/starlark/builtins_bzl/common/exports.bzl b/src/main/starlark/builtins_bzl/common/exports.bzl
@@ -24,7 +24,7 @@ load("@_builtins//:common/objc/compilation_support.bzl", "compilation_support")
 load("@_builtins//:common/objc/linking_support.bzl", "linking_support")
 load("@_builtins//:common/proto/proto_common.bzl", "proto_common_do_not_use")
 load("@_builtins//:common/proto/proto_library.bzl", "proto_library")
-load("@_builtins//:common/proto/proto_lang_toolchain_wrapper.bzl", "proto_lang_toolchain")
+load("@_builtins//:common/proto/proto_lang_toolchain.bzl", "proto_lang_toolchain")
 load("@_builtins//:common/python/py_runtime_macro.bzl", "py_runtime")
 load("@_builtins//:common/python/providers.bzl", "PyInfo", "PyRuntimeInfo")
 load("@_builtins//:common/java/proto/java_lite_proto_library.bzl", "java_lite_proto_library")

diff --git a/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl b/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl
@@ -73,4 +73,6 @@ semantics = struct(
     check_proto_registry_collision = _check_proto_registry_collision,
     get_coverage_runner = _get_coverage_runner,
     add_constraints = _add_constraints,
+    JAVA_PROTO_TOOLCHAIN = "@rules_java//java/proto:toolchain_type",
+    JAVA_LITE_PROTO_TOOLCHAIN = "@rules_java//java/proto:lite_toolchain_type",
 )
diff --git a/src/main/starlark/builtins_bzl/common/java/proto/java_lite_proto_library.bzl b/src/main/starlark/builtins_bzl/common/java/proto/java_lite_proto_library.bzl
@@ -15,8 +15,8 @@
 """A Starlark implementation of the java_lite_proto_library rule."""
 
 load(":common/java/java_semantics.bzl", "semantics")
-load(":common/proto/proto_common.bzl", "ProtoLangToolchainInfo", proto_common = "proto_common_do_not_use")
-load(":common/java/proto/java_proto_library.bzl", "JavaProtoAspectInfo", "bazel_java_proto_library_rule", "java_compile_for_protos")
+load(":common/java/proto/java_proto_library.bzl", "JavaProtoAspectInfo", "java_compile_for_protos")
+load(":common/proto/proto_common.bzl", "toolchains", "ProtoLangToolchainInfo", proto_common = "proto_common_do_not_use")
 
 PROTO_TOOLCHAIN_ATTR = "_aspect_proto_toolchain_for_javalite"
 
@@ -40,7 +40,11 @@ def _aspect_impl(target, ctx):
 
     deps = [dep[JavaInfo] for dep in ctx.rule.attr.deps]
     exports = [exp[JavaInfo] for exp in ctx.rule.attr.exports]
-    proto_toolchain_info = ctx.attr._aspect_proto_toolchain_for_javalite[ProtoLangToolchainInfo]
+    proto_toolchain_info = toolchains.find_toolchain(
+        ctx,
+        "_aspect_proto_toolchain_for_javalite",
+        semantics.JAVA_LITE_PROTO_TOOLCHAIN,
+    )
     source_jar = None
 
     if proto_common.experimental_should_generate_code(target[ProtoInfo], proto_toolchain_info, "java_lite_proto_library", target.label):
@@ -74,15 +78,16 @@ def _aspect_impl(target, ctx):
 java_lite_proto_aspect = aspect(
     implementation = _aspect_impl,
     attr_aspects = ["deps", "exports"],
-    attrs = {
+    attrs = toolchains.if_legacy_toolchain({
         PROTO_TOOLCHAIN_ATTR: attr.label(
             default = configuration_field(fragment = "proto", name = "proto_toolchain_for_java_lite"),
         ),
-    },
+    }),
     fragments = ["java"],
     required_providers = [ProtoInfo],
     provides = [JavaInfo, JavaProtoAspectInfo],
-    toolchains = [semantics.JAVA_TOOLCHAIN],
+    toolchains = [semantics.JAVA_TOOLCHAIN] +
+                 toolchains.use_toolchain(semantics.JAVA_LITE_PROTO_TOOLCHAIN),
 )
 
 def _rule_impl(ctx):
@@ -98,31 +103,44 @@ def _rule_impl(ctx):
       ([JavaInfo, DefaultInfo, OutputGroupInfo, ProguardSpecProvider])
     """
 
-    proto_toolchain_info = ctx.attr._aspect_proto_toolchain_for_javalite[ProtoLangToolchainInfo]
+    proto_toolchain_info = toolchains.find_toolchain(
+        ctx,
+        "_aspect_proto_toolchain_for_javalite",
+        semantics.JAVA_LITE_PROTO_TOOLCHAIN,
+    )
     runtime = proto_toolchain_info.runtime
 
     if runtime:
         proguard_provider_specs = runtime[ProguardSpecProvider]
     else:
         proguard_provider_specs = ProguardSpecProvider(depset())
 
-    java_info, DefaultInfo, OutputGroupInfo = bazel_java_proto_library_rule(ctx)
+    java_info = java_common.merge([dep[JavaInfo] for dep in ctx.attr.deps], merge_java_outputs = False)
+
+    transitive_src_and_runtime_jars = depset(transitive = [dep[JavaProtoAspectInfo].jars for dep in ctx.attr.deps])
+    transitive_runtime_jars = depset(transitive = [java_info.transitive_runtime_jars])
+
     java_info = semantics.add_constraints(java_info, ["android"])
 
     return [
         java_info,
-        DefaultInfo,
-        OutputGroupInfo,
+        DefaultInfo(
+            files = transitive_src_and_runtime_jars,
+            runfiles = ctx.runfiles(transitive_files = transitive_runtime_jars),
+        ),
+        OutputGroupInfo(default = depset()),
         proguard_provider_specs,
     ]
 
 java_lite_proto_library = rule(
     implementation = _rule_impl,
     attrs = {
         "deps": attr.label_list(providers = [ProtoInfo], aspects = [java_lite_proto_aspect]),
+    } | toolchains.if_legacy_toolchain({
         PROTO_TOOLCHAIN_ATTR: attr.label(
             default = configuration_field(fragment = "proto", name = "proto_toolchain_for_java_lite"),
         ),
-    },
+    }),
     provides = [JavaInfo],
+    toolchains = toolchains.use_toolchain(semantics.JAVA_LITE_PROTO_TOOLCHAIN),
 )