Merge pull request #150 from streem/protoc-via-maven

Download protoc from maven for all protoc tasks
streem · May 7, 2021 · e4655f6 · e4655f6
2 parents 76e5897 + 1c863f1
commit e4655f6
Show file tree

Hide file tree

Showing 19 changed files with 46,097 additions and 815 deletions.
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
@@ -4,7 +4,7 @@ env:
   GRADLE_OPTS: '-Dorg.gradle.daemon=false -Dkotlin.compiler.execution.strategy=in-process'
   JAVA_VERSION: 11
   NODEJS_VERSION: 14
-  PROTOBUF_VERSION: 3.10.1
+  PROTOBUF_CONFORMANCE_VERSION: 3.10.1
 
 on:
   pull_request:
@@ -42,32 +42,9 @@ jobs:
       - name: Build project and run tests
         run: ./gradlew build --stacktrace
 
-      - name: Cache protobuf ${{ env.PROTOBUF_VERSION }}
-        uses: actions/cache@v2
-        id: cache-protobuf
-        with:
-          path: ~/protobuf
-          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_VERSION }}
-
-      - name: Build protoc and protobuf conformance-test-runner
-        if: steps.cache-protobuf.outputs.cache-hit != 'true'
-        run: |
-          cd ~
-          curl -sSLO https://github.com/protocolbuffers/protobuf/releases/download/v${{ env.PROTOBUF_VERSION }}/protobuf-all-${{ env.PROTOBUF_VERSION }}.tar.gz
-          tar xzvf protobuf-all-${{ env.PROTOBUF_VERSION }}.tar.gz
-          mv protobuf-${{ env.PROTOBUF_VERSION }} protobuf
-          cd protobuf
-          ./configure --prefix="$(pwd)/install"
-          make
-          make install
-          cd conformance
-          make
-
       - name: Ensure bundled types are up-to-date
         run: |
-          export PATH="$PATH:$HOME/protobuf/install/bin"
-          ./gradlew -Dprotoc.path=$HOME/protobuf/install \
-            :runtime:generateWellKnownTypes \
+          ./gradlew :runtime:generateWellKnownTypes \
             :runtime:generateTestTypes \
             :protoc-gen-kotlin:protoc-gen-kotlin-lib:generateProto \
             :conformance:conformance-lib:generateProto
@@ -127,8 +104,32 @@ jobs:
           name: error-report
           path: build-reports.zip
 
+  build-conformance-test-runner:
+    runs-on: ubuntu-18.04
+
+    steps:
+      - name: Cache protobuf conformance-test-runner ${{ env.PROTOBUF_CONFORMANCE_VERSION }}
+        uses: actions/cache@v2
+        id: cache-protobuf
+        with:
+          path: ~/protobuf
+          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_CONFORMANCE_VERSION }}
+
+      - name: Build protobuf conformance-test-runner
+        if: steps.cache-protobuf.outputs.cache-hit != 'true'
+        run: |
+          cd ~
+          curl -sSLO https://github.com/protocolbuffers/protobuf/releases/download/v${{ env.PROTOBUF_CONFORMANCE_VERSION }}/protobuf-all-${{ env.PROTOBUF_CONFORMANCE_VERSION }}.tar.gz
+          tar xzvf protobuf-all-${{ env.PROTOBUF_CONFORMANCE_VERSION }}.tar.gz
+          mv protobuf-${{ env.PROTOBUF_CONFORMANCE_VERSION }} protobuf
+          cd protobuf
+          ./configure
+          make
+          cd conformance
+          make
+
   conformance:
-    needs: build
+    needs: [build, build-conformance-test-runner]
     strategy:
       matrix:
         platform: [jvm, js, linux]
@@ -148,11 +149,11 @@ jobs:
         with:
           node-version: ${{ env.NODEJS_VERSION }}
 
-      - name: Cache protobuf ${{ env.PROTOBUF_VERSION }}
+      - name: Cache protobuf conformance-test-runner ${{ env.PROTOBUF_CONFORMANCE_VERSION }}
         uses: actions/cache@v2
         with:
           path: ~/protobuf
-          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_VERSION }}
+          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_CONFORMANCE_VERSION }}
 
       - name: Download conformance test files
         uses: actions/download-artifact@v2

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -4,7 +4,7 @@ env:
   GRADLE_OPTS: '-Dorg.gradle.daemon=false -Dkotlin.compiler.execution.strategy=in-process'
   JAVA_VERSION: 11
   NODEJS_VERSION: 14
-  PROTOBUF_VERSION: 3.10.1
+  PROTOBUF_CONFORMANCE_VERSION: 3.10.1
   SIGNING_KEY_ASCII_ARMORED: ${{ secrets.SIGNING_KEY_ASCII_ARMORED }}
   SONATYPE_API_USER: ${{ secrets.SONATYPE_API_USER }}
   SONATYPE_API_KEY: ${{ secrets.SONATYPE_API_KEY }}
@@ -49,32 +49,9 @@ jobs:
       - name: Build project and run tests
         run: ./gradlew build --stacktrace
 
-      - name: Cache protobuf ${{ env.PROTOBUF_VERSION }}
-        uses: actions/cache@v2
-        id: cache-protobuf
-        with:
-          path: ~/protobuf
-          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_VERSION }}
-
-      - name: Build protoc and protobuf conformance-test-runner
-        if: steps.cache-protobuf.outputs.cache-hit != 'true'
-        run: |
-          cd ~
-          curl -sSLO https://github.com/protocolbuffers/protobuf/releases/download/v${{ env.PROTOBUF_VERSION }}/protobuf-all-${{ env.PROTOBUF_VERSION }}.tar.gz
-          tar xzvf protobuf-all-${{ env.PROTOBUF_VERSION }}.tar.gz
-          mv protobuf-${{ env.PROTOBUF_VERSION }} protobuf
-          cd protobuf
-          ./configure --prefix="$(pwd)/install"
-          make
-          make install
-          cd conformance
-          make
-
       - name: Ensure bundled types are up-to-date
         run: |
-          export PATH="$PATH:$HOME/protobuf/install/bin"
-          ./gradlew -Dprotoc.path=$HOME/protobuf/install \
-            :runtime:generateWellKnownTypes \
+          ./gradlew :runtime:generateWellKnownTypes \
             :runtime:generateTestTypes \
             :protoc-gen-kotlin:protoc-gen-kotlin-lib:generateProto \
             :conformance:conformance-lib:generateProto
@@ -134,8 +111,32 @@ jobs:
           name: error-report
           path: build-reports.zip
 
+  build-conformance-test-runner:
+    runs-on: ubuntu-18.04
+
+    steps:
+      - name: Cache protobuf conformance-test-runner ${{ env.PROTOBUF_CONFORMANCE_VERSION }}
+        uses: actions/cache@v2
+        id: cache-protobuf
+        with:
+          path: ~/protobuf
+          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_CONFORMANCE_VERSION }}
+
+      - name: Build protobuf conformance-test-runner
+        if: steps.cache-protobuf.outputs.cache-hit != 'true'
+        run: |
+          cd ~
+          curl -sSLO https://github.com/protocolbuffers/protobuf/releases/download/v${{ env.PROTOBUF_CONFORMANCE_VERSION }}/protobuf-all-${{ env.POTOBUF_CONFORMANCE_VERSION }}.tar.gz
+          tar xzvf protobuf-all-${{ env.PROTOBUF_CONFORMANCE_VERSION }}.tar.gz
+          mv protobuf-${{ env.PROTOBUF_CONFORMANCE_VERSION }} protobuf
+          cd protobuf
+          ./configure
+          make
+          cd conformance
+          make
+
   conformance:
-    needs: build
+    needs: [build, build-conformance-test-runner]
     strategy:
       matrix:
         platform: [jvm, js, linux]
@@ -155,11 +156,11 @@ jobs:
         with:
           node-version: ${{ env.NODEJS_VERSION }}
 
-      - name: Cache protobuf ${{ env.PROTOBUF_VERSION }}
+      - name: Cache protobuf conformance-test-runner ${{ env.PROTOBUF_CONFORMANCE_VERSION }}
         uses: actions/cache@v2
         with:
           path: ~/protobuf
-          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_VERSION }}
+          key: ${{ runner.os }}-protobuf-${{ env.PROTOBUF_CONFORMANCE_VERSION }}
 
       - name: Download conformance test files
         uses: actions/download-artifact@v2

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * `pbandk-runtime` on JVM and Android no longer depends on the `protobuf-java` library. It instead uses the same pure-Kotlin implementation of protobuf primitives that is used by the `pbandk-runtime` on Kotlin/Native. This avoids dependency conflicts with other libraries (such as `com.google.firebase:firebase-perf` on Android). (PRs [#124], [#148], [#151], fixes [#91], [#138]) (thanks @jeroenmols)
 * Updated protobuf well-known types to the versions shipped with protobuf 3.15.5.
 * Updated `pbandk-runtime-js` dependency on `protobuf.js` to `6.10.2`.
+* Building pbandk no longer requires downloading and compiling protobuf. Running conformance tests still requires compiling protobuf however.
 
 ### Fixed
 

diff --git a/README.md b/README.md
@@ -511,14 +511,12 @@ To build the runtime library for both JS and the JVM, run:
 
 If any changes are made to the generated code that is output by `protoc-gen-kotlin`, then the
 well-known types (and other proto types used by pbandk) need to be re-generated using the updated
-`protoc-gen-kotlin` binary. To do this, first download a recent [release of `protoc`](https://github.com/protocolbuffers/protobuf/releases),
-extract it to a local directory, and then run:
+`protoc-gen-kotlin` binary:
 
 ```
-./gradlew -Dprotoc.path=path/to/protobuf/install/directory \
-    :runtime:generateWellKnownTypes 
-    :runtime:generateTestTypes
-    :protoc-gen-kotlin:protoc-gen-kotlin-lib:generateProto
+./gradlew :runtime:generateWellKnownTypes \
+    :runtime:generateTestTypes \
+    :protoc-gen-kotlin:protoc-gen-kotlin-lib:generateProto \
     :conformance:conformance-lib:generateProto
 ```
 
@@ -554,7 +552,9 @@ export CONF_TEST_PATH="$(pwd)/conformance-test-runner"
 Now, back in `pbandk`, build all JS. JVM and native projects via:
 
 ```
-./gradlew :conformance:conformance-lib:assemble :conformance:conformance-jvm:installDist :conformance:conformance-native:build
+./gradlew :conformance:conformance-lib:assemble \
+    :conformance:conformance-jvm:installDist \
+    :conformance:conformance-native:build
 ```
 
 You are now ready to run the conformance tests.  Make sure `CONF_TEST_PATH` environment variable is set to `path/to/protobuf/conformance/conformance-test-runner` (see above).

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -9,8 +9,8 @@ plugins {
     kotlin("multiplatform") version Versions.kotlin apply false
     id("com.android.library") version Versions.androidGradlePlugin apply false
     id("org.springframework.boot") version Versions.springBootGradlePlugin apply false
-    id("com.google.osdetector") version Versions.osDetectorGradlePlugin apply false
 
+    id("com.google.osdetector") version Versions.osDetectorGradlePlugin
     id("binary-compatibility-validator") version Versions.binaryCompatibilityValidatorGradlePlugin
     id("io.github.gradle-nexus.publish-plugin") version Versions.nexusPublishGradlePlugin
 }
@@ -54,11 +54,22 @@ if (signingKeyAsciiArmored.isPresent) {
     logger.info("PGP signing key not defined, skipping signing configuration")
 }
 
+val downloadProtoc by configurations.creating {
+    isTransitive = false
+}
+
 val wellKnownTypes by configurations.creating {
     isTransitive = false
 }
 
 dependencies {
+    downloadProtoc(
+        group = "com.google.protobuf",
+        name = "protoc",
+        version = Versions.protoc,
+        classifier = osdetector.classifier,
+        ext = "exe"
+    )
     wellKnownTypes("com.google.protobuf:protobuf-java:${Versions.protobufJava}")
 }
 
@@ -89,4 +100,4 @@ allprojects {
             )
         }
     }
-}
+}
diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
@@ -8,4 +8,8 @@ repositories {
 
 dependencies {
     implementation("com.google.guava:guava:28.0-jre")
-}
+}
+
+kotlinDslPluginOptions {
+    experimentalWarning.set(false)
+}
diff --git a/buildSrc/src/main/kotlin/DescriptorProtocTask.kt b/buildSrc/src/main/kotlin/DescriptorProtocTask.kt
@@ -1,45 +1,7 @@
-import org.gradle.api.file.DirectoryProperty
-import org.gradle.api.file.FileCollection
-import org.gradle.api.provider.Property
-import org.gradle.api.tasks.*
-import org.gradle.kotlin.dsl.property
-
 @Suppress("UnstableApiUsage")
-open class DescriptorProtocTask : AbstractExecTask<DescriptorProtocTask>(DescriptorProtocTask::class.java) {
-    @InputDirectory
-    val includeDir: DirectoryProperty = project.objects.directoryProperty()
-
-    @OutputDirectory
-    val outputDir: DirectoryProperty = project.objects.directoryProperty()
-
-    @Input
-    val protoc: Property<String> = project.objects.property<String>().apply {
-        convention("protoc")
-    }
-
-    @Input
-    val descriptorSetOutput: Property<String> = project.objects.property<String>().apply {
-        convention("fileDescriptor.protoset")
-    }
-
-    private val protoFileDir: DirectoryProperty = project.objects.directoryProperty().apply {
-        convention(includeDir)
-    }
-
-    @InputFiles
-    @SkipWhenEmpty
-    val protoFiles: FileCollection = protoFileDir.asFileTree.matching {
-        this.include("*.proto")
-    }
-
-    override fun exec() {
-        executable = protoc.get()
-
-        args("--proto_path=${includeDir.get()}")
-        args("--descriptor_set_out=${outputDir.get().file(descriptorSetOutput.get()).asFile.absolutePath}")
-
-        args(protoFiles.map { it.absolutePath })
-
-        super.exec()
+open class DescriptorProtocTask : ProtocTask() {
+    init {
+        plugin.set("descriptor_set")
+        outputFileName.convention("fileDescriptor.protoset")
     }
 }