Use relocated protobuf runtime

Generated protobuf code must be run with the protobuf runtime using the exact same version as the generator used. This is currently a soft constraint, but it is likely that protobuf will enforce that in future protobuf versions. Using a different version of the protobuf runtime can (and does) lead to issues at runtime, see [this issue for an example](protocolbuffers/protobuf#11986).

This change adds a new Nessie artifact `nessie-protobuf-relocated` which contains a relocated protobuf runtime that is used by all other Nessie artifacts.

Also:
* Move `*.proto` files into the directories corresponding to the Java package names.
* Remove the `ProtobufHelperPlugin`, as the underlying issue (not exposing the generated java source folders in IDEs) has been fixed.
* Do not apply checkstyle to Gradle projects that just generate protobuf code (the `ProtobufHelperPlugin` did that before).

Side note: CEL-Java heavily depends on protobuf itself via its API and not just as a runtime/compile-only dependency. As long as no generated protobuf code is being used, it _should_ be okay to use a different protobuf version at runtime, but there is and will be no guarantee.

buildSrc/build.gradle.kts

@@ -30,7 +30,6 @@ dependencies {
   implementation(baselibs.jandex)
   implementation(baselibs.idea.ext)
   implementation(baselibs.shadow)
-  implementation(baselibs.protobuf)
   implementation(baselibs.errorprone)
   implementation(baselibs.nessie.buildsupport.reflectionconfig)
 

buildSrc/src/main/kotlin/Checkstyle.kt

@@ -29,6 +29,10 @@ import org.gradle.kotlin.dsl.withType
 class NessieCheckstylePlugin : Plugin<Project> {
   override fun apply(project: Project): Unit =
     project.run {
+      if (project.name.endsWith("-proto")) {
+        return
+      }
+
       apply<CheckstylePlugin>()
       configure<CheckstyleExtension> {
         toolVersion = libsRequiredVersion("checkstyle")

gradle/baselibs.versions.toml

@@ -4,8 +4,7 @@
 errorpronePlugin = "3.0.1"
 jandexPlugin = "1.87"
 junit = "5.9.2"
-nessieBuildPlugins = "0.2.20"
-protobufPlugin = "0.9.2"
+nessieBuildPlugins = "0.2.21"
 shadowPlugin = "8.1.1"
 spotlessPlugin = "6.17.0"
 
@@ -21,4 +20,3 @@ junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" }
 nessie-buildsupport-reflectionconfig = { module = "org.projectnessie.buildsupport:reflection-config", version.ref = "nessieBuildPlugins" }
 shadow = { module = "com.github.johnrengelman:shadow", version.ref = "shadowPlugin" }
 spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotlessPlugin" }
-protobuf = { module = "com.google.protobuf:protobuf-gradle-plugin", version.ref = "protobufPlugin" }

gradle/libs.versions.toml

@@ -226,4 +226,5 @@ jmh = { id = "me.champeau.jmh", version = "0.7.0" }
 nessie-run = { id = "org.projectnessie", version = "0.29.0" }
 nexus-publish-plugin = { id = "io.github.gradle-nexus.publish-plugin", version = "1.3.0" }
 node-gradle = { id = "com.github.node-gradle.node", version = "3.5.1" }
+protobuf = { id = "com.google.protobuf", version = "0.9.2" }
 quarkus = { id = "io.quarkus", version.ref = "quarkus" }

gradle/projects.main.properties

@@ -26,6 +26,7 @@ nessie-services=servers/services
 nessie-server-store=servers/store
 nessie-server-store-proto=servers/store-proto
 nessie-content-generator=tools/content-generator
+nessie-protobuf-relocated=tools/protobuf-relocated
 nessie-ui=ui
 nessie-versioned-persist-adapter=versioned/persist/adapter
 nessie-versioned-persist-bench=versioned/persist/bench

servers/quarkus-cli/build.gradle.kts

@@ -60,8 +60,6 @@ dependencies {
   implementation(enforcedPlatform(libs.quarkus.amazon.services.bom))
   implementation("io.quarkus:quarkus-picocli")
 
-  implementation(libs.protobuf.java)
-
   // javax/jakarta
   compileOnly(libs.jakarta.annotation.api)
   compileOnly(libs.findbugs.jsr305)

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/BaseContentTest.java

@@ -19,7 +19,6 @@
 
 import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.main.LaunchResult;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
 import java.io.File;
@@ -32,6 +31,7 @@
 import org.projectnessie.model.ContentKey;
 import org.projectnessie.model.IcebergTable;
 import org.projectnessie.model.Namespace;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.BranchName;
 import org.projectnessie.versioned.CommitMetaSerializer;
 import org.projectnessie.versioned.persist.adapter.ContentId;

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/ITCheckContent.java

@@ -21,7 +21,6 @@
 import static org.projectnessie.versioned.store.DefaultStoreWorker.payloadForContent;
 import static org.projectnessie.versioned.testworker.OnRefOnly.onRef;
 
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
 import io.quarkus.test.junit.main.QuarkusMainTest;
@@ -33,6 +32,7 @@
 import org.projectnessie.model.Content;
 import org.projectnessie.model.ContentKey;
 import org.projectnessie.model.IcebergTable;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.BranchName;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.ReferenceInfo;

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/ITContentInfo.java

@@ -20,7 +20,6 @@
 import static org.projectnessie.model.Content.Type.ICEBERG_TABLE;
 import static org.projectnessie.model.Content.Type.NAMESPACE;
 
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
 import io.quarkus.test.junit.main.QuarkusMainTest;
@@ -31,6 +30,7 @@
 import org.junit.jupiter.params.provider.ValueSource;
 import org.projectnessie.model.ContentKey;
 import org.projectnessie.model.IcebergTable;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.server.store.proto.ObjectTypes;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.ReferenceInfo;

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/ITEraseRepository.java

@@ -17,13 +17,13 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
 import io.quarkus.test.junit.main.QuarkusMainTest;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.ReferenceInfo;
 import org.projectnessie.versioned.ReferenceNotFoundException;

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/ITExportImport.java

@@ -20,7 +20,6 @@
 import static org.projectnessie.quarkus.cli.ImportRepository.ERASE_BEFORE_IMPORT;
 import static org.projectnessie.versioned.store.DefaultStoreWorker.payloadForContent;
 
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.main.LaunchResult;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
@@ -39,6 +38,7 @@
 import org.projectnessie.model.ContentKey;
 import org.projectnessie.model.IcebergTable;
 import org.projectnessie.model.Namespace;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.quarkus.cli.ExportRepository.Format;
 import org.projectnessie.versioned.BranchName;
 import org.projectnessie.versioned.CommitMetaSerializer;

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/ITExportImportPersist.java

@@ -29,7 +29,6 @@
 import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID;
 import static org.projectnessie.versioned.store.DefaultStoreWorker.payloadForContent;
 
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.main.LaunchResult;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
@@ -45,6 +44,7 @@
 import org.junit.jupiter.api.io.TempDir;
 import org.projectnessie.api.NessieVersion;
 import org.projectnessie.model.IcebergTable;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.quarkus.cli.ExportRepository.Format;
 import org.projectnessie.versioned.StoreWorker;
 import org.projectnessie.versioned.storage.common.indexes.StoreKey;

servers/quarkus-cli/src/intTest/java/org/projectnessie/quarkus/cli/ITNessieInfo.java

@@ -17,14 +17,14 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import com.google.protobuf.ByteString;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 import io.quarkus.test.junit.main.QuarkusMainLauncher;
 import io.quarkus.test.junit.main.QuarkusMainTest;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.BranchName;
 import org.projectnessie.versioned.Hash;
 import org.projectnessie.versioned.ReferenceConflictException;

servers/quarkus-cli/src/main/java/org/projectnessie/quarkus/cli/CheckContent.java

@@ -20,7 +20,6 @@
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.protobuf.ByteString;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -31,6 +30,7 @@
 import java.util.stream.Stream;
 import org.projectnessie.model.Content;
 import org.projectnessie.model.ContentKey;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.Hash;
 import org.projectnessie.versioned.ReferenceInfo;

servers/quarkus-cli/src/main/java/org/projectnessie/quarkus/cli/ContentInfo.java

@@ -22,7 +22,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.google.errorprone.annotations.MustBeClosed;
-import com.google.protobuf.ByteString;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -34,6 +33,7 @@
 import java.util.stream.Stream;
 import org.projectnessie.model.Content;
 import org.projectnessie.model.ContentKey;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.DetachedRef;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.Hash;

servers/quarkus-cli/src/main/java/org/projectnessie/quarkus/cli/ImportRepository.java

@@ -17,14 +17,14 @@
 
 import static org.projectnessie.versioned.storage.common.logic.Logics.repositoryLogic;
 
-import com.google.protobuf.ByteString;
 import java.io.PrintWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.Hash;
 import org.projectnessie.versioned.ReferenceInfo;

servers/quarkus-cli/src/main/java/org/projectnessie/quarkus/cli/NessieInfo.java

@@ -19,11 +19,11 @@
 import static org.projectnessie.versioned.storage.common.logic.Logics.commitLogic;
 import static org.projectnessie.versioned.storage.common.logic.Logics.referenceLogic;
 
-import com.google.protobuf.ByteString;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.stream.Collectors;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.versioned.GetNamedRefsParams;
 import org.projectnessie.versioned.GetNamedRefsParams.RetrieveOptions;
 import org.projectnessie.versioned.ReferenceInfo;

servers/quarkus-common/build.gradle.kts

@@ -67,8 +67,6 @@ dependencies {
   compileOnly(libs.jakarta.validation.api)
   compileOnly(libs.javax.validation.api)
 
-  implementation(libs.protobuf.java)
-
   compileOnly(libs.graalvm.nativeimage.svm)
 
   compileOnly(platform(libs.jackson.bom))

servers/quarkus-server/build.gradle.kts

@@ -102,7 +102,6 @@ dependencies {
   testFixturesApi("io.quarkus:quarkus-test-oidc-server")
   testFixturesApi("io.quarkus:quarkus-jacoco")
   testFixturesImplementation(libs.guava)
-  testFixturesImplementation(libs.protobuf.java)
   testFixturesImplementation(libs.microprofile.openapi)
 
   testFixturesApi(platform(libs.junit.bom))

servers/store-proto/build.gradle.kts

@@ -14,21 +14,22 @@
  * limitations under the License.
  */
 
+import com.google.protobuf.gradle.GenerateProtoTask
 import com.google.protobuf.gradle.ProtobufExtension
+import com.google.protobuf.gradle.ProtobufExtract
 
 plugins {
   `java-library`
   `maven-publish`
   signing
   id("org.projectnessie.buildsupport.reflectionconfig")
   `nessie-conventions`
+  alias(libs.plugins.protobuf)
 }
 
-apply<ProtobufHelperPlugin>()
-
 extra["maven.name"] = "Nessie - Server - Store (Proto)"
 
-dependencies { api(libs.protobuf.java) }
+dependencies { api(project(path = ":nessie-protobuf-relocated", configuration = "shadow")) }
 
 // *.proto files taken from https://github.com/googleapis/googleapis/ repo, available as a git
 // submodule
@@ -40,16 +41,28 @@ extensions.configure<ProtobufExtension> {
   }
 }
 
+tasks.named<GenerateProtoTask>("generateProto") {
+  doLast {
+    fileTree("$buildDir/generated/source/proto/main").forEach {
+      it.writeText(
+        it.readText().replace("com.google.protobuf", "org.projectnessie.nessie.relocated.protobuf")
+      )
+    }
+  }
+}
+
 reflectionConfig {
   // Consider classes that extend one of these classes...
   classExtendsPatterns.set(
     listOf(
-      "com.google.protobuf.GeneratedMessageV3",
-      "com.google.protobuf.GeneratedMessageV3.Builder"
+      "org.projectnessie.nessie.relocated.protobuf.GeneratedMessageV3",
+      "org.projectnessie.nessie.relocated.protobuf.GeneratedMessageV3.Builder"
     )
   )
   // ... and classes the implement this interface.
-  classImplementsPatterns.set(listOf("com.google.protobuf.ProtocolMessageEnum"))
+  classImplementsPatterns.set(
+    listOf("org.projectnessie.nessie.relocated.protobuf.ProtocolMessageEnum")
+  )
   // Also include generated classes (e.g. google.protobuf.Empty) via the "runtimeClasspath",
   // which contains the the "com.google.protobuf:protobuf-java" dependency.
   includeConfigurations.set(listOf("runtimeClasspath"))
@@ -60,7 +73,7 @@ tasks.named<Jar>("sourcesJar") {
   dependsOn(tasks.named("generateProto"), tasks.named("generateReflectionConfig"))
 }
 
-tasks.withType(com.google.protobuf.gradle.ProtobufExtract::class).configureEach {
+tasks.withType(ProtobufExtract::class).configureEach {
   when (name) {
     "extractIncludeTestProto" -> dependsOn(tasks.named("processJandexIndex"))
     "extractIncludeTestFixturesProto" -> dependsOn(tasks.named("processJandexIndex"))

servers/store/src/main/java/org/projectnessie/server/store/BaseSerializer.java

@@ -15,8 +15,6 @@
  */
 package org.projectnessie.server.store;
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -29,6 +27,8 @@
 import org.projectnessie.model.ImmutableIcebergView;
 import org.projectnessie.model.ImmutableNamespace;
 import org.projectnessie.model.Namespace;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
+import org.projectnessie.nessie.relocated.protobuf.InvalidProtocolBufferException;
 import org.projectnessie.server.store.proto.ObjectTypes;
 import org.projectnessie.versioned.ContentAttachment;
 import org.projectnessie.versioned.ContentAttachmentKey;

servers/store/src/main/java/org/projectnessie/server/store/DeltaLakeTableSerializer.java

@@ -15,11 +15,11 @@
  */
 package org.projectnessie.server.store;
 
-import com.google.protobuf.ByteString;
 import java.util.function.Supplier;
 import org.projectnessie.model.Content;
 import org.projectnessie.model.DeltaLakeTable;
 import org.projectnessie.model.ImmutableDeltaLakeTable;
+import org.projectnessie.nessie.relocated.protobuf.ByteString;
 import org.projectnessie.server.store.proto.ObjectTypes;
 
 public final class DeltaLakeTableSerializer extends BaseSerializer<DeltaLakeTable> {