From 32c67e0efe161dc253857195977ca669a7cb25a8 Mon Sep 17 00:00:00 2001 From: "Nadav Sr. Samet" Date: Sat, 22 Aug 2015 12:17:31 -0700 Subject: [PATCH] Use an internal Base64 implementation. Fixes #34 --- build.sbt | 7 +- .../scalapb/compiler/ProtobufGenerator.scala | 6 +- scalapb-runtime/build.sbt | 4 +- .../com/trueaccord/scalapb/Encoding.scala | 70 +++++++++++++++++++ .../com/trueaccord/scalapb/EncodingSpec.scala | 28 ++++++++ 5 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 scalapb-runtime/src/main/scala/com/trueaccord/scalapb/Encoding.scala create mode 100644 scalapb-runtime/src/test/scala/com/trueaccord/scalapb/EncodingSpec.scala diff --git a/build.sbt b/build.sbt index a19a56872..58d0313f7 100644 --- a/build.sbt +++ b/build.sbt @@ -42,6 +42,7 @@ lazy val projectReleaseSettings = Seq( releasePublishArtifactsAction := {}, releasePublishArtifactsAction <<= releasePublishArtifactsAction.dependsOn( PgpKeys.publishSigned, + clean, publishLocal) ) @@ -55,8 +56,10 @@ lazy val root = lazy val runtime = project.in(file("scalapb-runtime")).settings( projectReleaseSettings:_*) -lazy val compilerPlugin = project.in(file("compiler-plugin")).settings( - projectReleaseSettings:_*) +lazy val compilerPlugin = project.in(file("compiler-plugin")) + .dependsOn(runtime) + .settings( + projectReleaseSettings:_*) lazy val proptest = project.in(file("proptest")) .dependsOn(runtime, compilerPlugin) diff --git a/compiler-plugin/src/main/scala/com/trueaccord/scalapb/compiler/ProtobufGenerator.scala b/compiler-plugin/src/main/scala/com/trueaccord/scalapb/compiler/ProtobufGenerator.scala index cfb0ac124..5144f9c11 100644 --- a/compiler-plugin/src/main/scala/com/trueaccord/scalapb/compiler/ProtobufGenerator.scala +++ b/compiler-plugin/src/main/scala/com/trueaccord/scalapb/compiler/ProtobufGenerator.scala @@ -905,9 +905,9 @@ class ProtobufGenerator(val params: GeneratorParams) extends DescriptorPimps { def generateFileDescriptor(file: FileDescriptor)(fp: FunctionalPrinter): FunctionalPrinter = { // Encoding the file descriptor proto in base64. JVM has a limit on string literal to be up // to 64k, so we chunk it into a sequence and combining in run time. The chunks are less - // than base64 to account for indentation and new lines. + // than 64k to account for indentation and new lines. val clearProto = file.toProto.toBuilder.clearSourceCodeInfo.build - val base64: Seq[Seq[String]] = javax.xml.bind.DatatypeConverter.printBase64Binary(clearProto.toByteArray) + val base64: Seq[Seq[String]] = com.trueaccord.scalapb.Encoding.toBase64(clearProto.toByteArray) .grouped(55000).map { group => val lines = ("\"\"\"" + group).grouped(100).toSeq @@ -915,7 +915,7 @@ class ProtobufGenerator(val params: GeneratorParams) extends DescriptorPimps { }.toSeq fp.add("lazy val descriptor: com.google.protobuf.Descriptors.FileDescriptor = {") .add(" val proto = com.google.protobuf.DescriptorProtos.FileDescriptorProto.parseFrom(") - .add(" javax.xml.bind.DatatypeConverter.parseBase64Binary(Seq(") + .add(" com.trueaccord.scalapb.Encoding.fromBase64(Seq(") .addGroupsWithDelimiter(",")(base64) .add(" ).mkString))") .add(" com.google.protobuf.Descriptors.FileDescriptor.buildFrom(proto, Array(") diff --git a/scalapb-runtime/build.sbt b/scalapb-runtime/build.sbt index 7789f6134..dca525675 100644 --- a/scalapb-runtime/build.sbt +++ b/scalapb-runtime/build.sbt @@ -4,7 +4,9 @@ name := "scalapb-runtime" libraryDependencies ++= Seq( "com.google.protobuf" % "protobuf-java" % "3.0.0-alpha-3", - "com.trueaccord.lenses" %% "lenses" % "0.4" + "com.trueaccord.lenses" %% "lenses" % "0.4", + "org.scalacheck" %% "scalacheck" % "1.12.4" % "test", + "org.scalatest" %% "scalatest" % "2.2.4" % "test" ) unmanagedResourceDirectories in Compile += baseDirectory.value / "../protobuf" diff --git a/scalapb-runtime/src/main/scala/com/trueaccord/scalapb/Encoding.scala b/scalapb-runtime/src/main/scala/com/trueaccord/scalapb/Encoding.scala new file mode 100644 index 000000000..3bdcb28d0 --- /dev/null +++ b/scalapb-runtime/src/main/scala/com/trueaccord/scalapb/Encoding.scala @@ -0,0 +1,70 @@ +package com.trueaccord.scalapb + +import scala.collection.mutable + +/** Utility functions to encode/decode byte arrays as Base64 strings. + * + * Used internally between the protocol buffer compiler and the runtime to encode + * messages. + * + * We could have used Apache Commons, but we would like to avoid an additional dependency. + * javax.xm.bind.DayaTypeConverter.parseBase64Binary is not available on Android. And the Java + * native java.util.Base64 is only available for Java 8... + */ +object Encoding { + private val alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" + + def fromBase64(textInput: String): Array[Byte] = { + fromBase64Inner(textInput.filter(alphabet.contains(_: Char))) + } + + private def fromBase64Inner(input: String): Array[Byte] = { + require(input.length % 4 == 0) + val lastEqualsIndex = input.indexOf('=') + val outputLength = (input.length * 3) / 4 - ( + if (lastEqualsIndex > 0) (input.length() - input.indexOf('=')) else 0) + val builder = mutable.ArrayBuilder.make[Byte] + builder.sizeHint(outputLength) + + for { i <- 0 until(input.length, 4) } { + val b = input.substring(i, i + 4).map(alphabet.indexOf(_: Char).toByte) + builder += ((b(0) << 2) | (b(1) >> 4)).toByte + if (b(2) < 64) { + builder += ((b(1) << 4) | (b(2) >> 2)).toByte + if (b(3) < 64) { + builder += ((b(2) << 6) | b(3)).toByte + } + } + } + builder.result() + } + + def toBase64(in: Array[Byte]): String = { + val out = mutable.StringBuilder.newBuilder + var b: Int = 0 + for { i <- 0 until (in.length, 3) } { + b = (in(i) & 0xFC) >> 2 + out.append(alphabet(b)) + b = (in(i) & 0x03) << 4 + if (i + 1 < in.length) { + b |= (in(i + 1) & 0xF0) >> 4 + out.append(alphabet(b)) + b = (in(i + 1) & 0x0F) << 2 + if (i + 2 < in.length) { + b |= (in(i + 2) & 0xC0) >> 6 + out.append(alphabet(b)) + b = in(i + 2) & 0x3F + out.append(alphabet(b)) + } else { + out.append(alphabet(b)) + out.append('=') + } + } else { + out.append(alphabet(b)) + out.append("==") + } + } + out.result() + } + +} diff --git a/scalapb-runtime/src/test/scala/com/trueaccord/scalapb/EncodingSpec.scala b/scalapb-runtime/src/test/scala/com/trueaccord/scalapb/EncodingSpec.scala new file mode 100644 index 000000000..dbcaadce9 --- /dev/null +++ b/scalapb-runtime/src/test/scala/com/trueaccord/scalapb/EncodingSpec.scala @@ -0,0 +1,28 @@ +package com.trueaccord.scalapb + +import org.scalatest._ +import org.scalatest.prop.GeneratorDrivenPropertyChecks + +class EncodingSpec extends PropSpec with GeneratorDrivenPropertyChecks with Matchers { + property("fromBase64 is the inverse of toBase64") { + forAll { + b: Array[Byte] => + Encoding.fromBase64(Encoding.toBase64(b)) should be(b) + } + } + + property("fromBase64 is compatible with javax.printBase64") { + forAll { + b: Array[Byte] => + Encoding.fromBase64(javax.xml.bind.DatatypeConverter.printBase64Binary(b)) should be(b) + } + } + + property("toBase64 is compatible with javax.parseBase64") { + forAll { + b: Array[Byte] => + javax.xml.bind.DatatypeConverter.parseBase64Binary( + Encoding.toBase64(b)) should be(b) + } + } +}