diff --git a/README.md b/README.md index 2cd2523..a8af2d1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ 1. play-json-tools - Set of implicit Play-JSON `Format` helper classes. Example in [FlatFormatSpec](play-json-tools/src/test/scala/com/evolutiongaming/util/FlatFormatSpec.scala) 1. play-json-generic - provides Format derivation for enum like adt's (sealed trait/case objects'). Examples in [EnumerationDerivalSpec](play-json-generic/src/test/scala/com/evolutiongaming/util/generic/EnumerationDerivalSpec.scala) +1. play-json-jsoniter - provides the fastest way to convert an instance of `play.api.libs.json.JsValue` to byte array and read it back. ## Setup diff --git a/build.sbt b/build.sbt index 9e76a71..f006c46 100644 --- a/build.sbt +++ b/build.sbt @@ -16,7 +16,6 @@ val commonSettings = Seq( crossScalaVersions := Seq("2.13.1", "2.12.10"), ) - lazy val root = project .in(file(".")) .disablePlugins(MimaPlugin) @@ -28,6 +27,7 @@ lazy val root = project .aggregate( playJsonTools, playJsonGeneric, + playJsonJsoniter ) @@ -55,4 +55,20 @@ lazy val playJsonTools = project playJson, nel, scalaTest % Test, - ).map(excludeLog4j))) \ No newline at end of file + ).map(excludeLog4j))) + +//++ 2.12.10 or ++ 2.13.1 +lazy val playJsonJsoniter = project + .in(file("play-json-jsoniter")) + .settings(commonSettings) + .settings(Seq( + moduleName := "play-json-jsoniter", + name := "play-json-jsoniter", + libraryDependencies ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, v)) if v >= 13 => + Seq(playJson, nel, jsoniter, scalaTest % Test, jsonGenerator % Test).map(excludeLog4j) + case _ => + Seq(playJson, nel, jsoniter, collectionCompact, scalaTest % Test).map(excludeLog4j) + } + })) diff --git a/play-json-jsoniter/src/main/scala/com/evolutiongaming/jsonitertool/PlayJsonJsoniter.scala b/play-json-jsoniter/src/main/scala/com/evolutiongaming/jsonitertool/PlayJsonJsoniter.scala new file mode 100644 index 0000000..e5df205 --- /dev/null +++ b/play-json-jsoniter/src/main/scala/com/evolutiongaming/jsonitertool/PlayJsonJsoniter.scala @@ -0,0 +1,105 @@ +package com.evolutiongaming.jsonitertool + +import com.github.plokhotnyuk.jsoniter_scala.core._ +import play.api.libs.json._ + +import scala.collection.IndexedSeq +import scala.util.Try + +object PlayJsonJsoniter { + + implicit val jsValueCodec: JsonValueCodec[JsValue] = + new JsonValueCodec[JsValue] { + + /** + * The implementation was borrowed from: https://github.com/plokhotnyuk/jsoniter-scala/blob/e80d51019b39efacff9e695de97dce0c23ae9135/jsoniter-scala-benchmark/src/main/scala/io/circe/CirceJsoniter.scala + * and adapted to meet PlayJson criteria. + */ + override def decodeValue(in: JsonReader, default: JsValue): JsValue = { + val b = in.nextToken() + if (b == 'n') in.readNullOrError(default, "expected `null` value") + else if (b == '"') { + in.rollbackToken() + JsString(in.readString(null)) + } else if (b == 'f' || b == 't') { + in.rollbackToken() + if (in.readBoolean()) JsTrue else JsFalse + } else if ((b >= '0' && b <= '9') || b == '-') { + in.rollbackToken() + val s = JsonParserSettings.settings.bigDecimalParseSettings + JsNumber(in.readBigDecimal(null, s.mathContext, s.scaleLimit, s.digitsLimit)) + } else if (b == '[') { + val array: IndexedSeq[JsValue] = + if (in.isNextToken(']')) new Array[JsValue](0) + else { + in.rollbackToken() + var i = 0 + var arr = new Array[JsValue](4) + do { + if (i == arr.length) arr = java.util.Arrays.copyOf(arr, i << 1) + arr(i) = decodeValue(in, default) + i += 1 + } while (in.isNextToken(',')) + + if (in.isCurrentToken(']')) + if (i == arr.length) arr else java.util.Arrays.copyOf(arr, i) + else in.arrayEndOrCommaError() + } + JsArray(array) + } else if (b == '{') { + /* + * Because of DoS vulnerability in Scala 2.12 HashMap https://github.com/scala/bug/issues/11203 + * we use a Java LinkedHashMap because it better handles hash code collisions for Comparable keys. + */ + val kvs = + if (in.isNextToken('}')) new java.util.LinkedHashMap[String, JsValue]() + else { + val underlying = new java.util.LinkedHashMap[String, JsValue]() + in.rollbackToken() + do { + underlying.put(in.readKeyAsString(), decodeValue(in, default)) + } while (in.isNextToken(',')) + + if (!in.isCurrentToken('}')) + in.objectEndOrCommaError() + + underlying + } + import scala.jdk.CollectionConverters._ + JsObject(kvs.asScala) + } else in.decodeError("expected JSON value") + } + + override def encodeValue(jsValue: JsValue, out: JsonWriter): Unit = + jsValue match { + case JsBoolean(b) => + out.writeVal(b) + case JsString(value) => + out.writeVal(value) + case JsNumber(value) => + out.writeVal(value) + case JsArray(items) => + out.writeArrayStart() + items.foreach(encodeValue(_, out)) + out.writeArrayEnd() + case JsObject(kvs) => + out.writeObjectStart() + kvs.foreach { + case (k, v) => + out.writeKey(k) + encodeValue(v, out) + } + out.writeObjectEnd() + case JsNull => + out.writeNull() + } + + override val nullValue: JsValue = JsNull + } + + def serialize(payload: JsValue): Array[Byte] = + writeToArray(payload) + + def deserialize(bytes: Array[Byte]): Try[JsValue] = + Try(readFromArray[JsValue](bytes)) +} diff --git a/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/PlayJsonImplicits.scala b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/PlayJsonImplicits.scala new file mode 100644 index 0000000..d287ff1 --- /dev/null +++ b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/PlayJsonImplicits.scala @@ -0,0 +1,14 @@ +package com.evolutiongaming.jsonitertool + +import play.api.libs.json.Json +import com.evolutiongaming.jsonitertool.TestDataGenerators.{Address, User} + +trait PlayJsonImplicits { + + implicit val e = Json.reads[Address] + implicit val f = Json.writes[Address] + + implicit val g = Json.reads[User] + implicit val h = Json.writes[User] + +} diff --git a/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/PlayJsonWithJsoniterBackendSpec.scala b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/PlayJsonWithJsoniterBackendSpec.scala new file mode 100644 index 0000000..6af6df8 --- /dev/null +++ b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/PlayJsonWithJsoniterBackendSpec.scala @@ -0,0 +1,49 @@ +package com.evolutiongaming.jsonitertool + +import com.evolutiongaming.jsonitertool.TestDataGenerators.{User, genUser} +import org.scalacheck.Prop.forAll +import org.scalacheck.{Arbitrary, Gen, Test} +import play.api.libs.json.{JsSuccess, Json} + +//sbt playJsonJsoniter/test:"runMain com.evolutiongaming.jsonitertool.PlayJsonWithJsoniterBackendSpec" +object PlayJsonWithJsoniterBackendSpec extends org.scalacheck.Properties("PlayJsonWithJsoniterBackend") { + val Size = 200 + + override def overrideParameters(p: Test.Parameters): Test.Parameters = + p.withMinSuccessfulTests(Size) + + implicit def generator: Arbitrary[User] = Arbitrary(genUser) + + property("Write using PlayJson -> Read using Jsoniter") = forAll { user: User => + val jsValue = Json.toJson(user) + val bts = Json.toBytes(jsValue) + val actJsValue = PlayJsonJsoniter.deserialize(bts).map(Json.fromJson[User](_)) + JsSuccess(user) == actJsValue.get + } + + property("Write using PlayJson -> Read using Jsoniter. Batch") = forAll( + Gen.containerOfN[Vector, User](Size, genUser), + ) { batch: Vector[User] => + val bools = batch.map { user => + val jsValue = Json.toJson(user) + val bts = Json.toBytes(jsValue) + val actJsValue = PlayJsonJsoniter.deserialize(bts).map(Json.fromJson[User](_)).get + JsSuccess(user) == actJsValue + } + + bools.find(_ == false).isEmpty + } + + property("Write using Jsoniter -> Read using Jsoniter. Batch") = forAll( + Gen.containerOfN[Vector, User](Size, genUser), + ) { batch: Vector[User] => + val bools = batch.map { user => + val jsValue = Json.toJson(user) + val bts = PlayJsonJsoniter.serialize(jsValue) + val actJsValue = PlayJsonJsoniter.deserialize(bts).map(Json.fromJson[User](_)) + JsSuccess(user) == actJsValue.get + } + + bools.find(_ == false).isEmpty + } +} diff --git a/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/RandomJsonArraysSpec.scala b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/RandomJsonArraysSpec.scala new file mode 100644 index 0000000..6440962 --- /dev/null +++ b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/RandomJsonArraysSpec.scala @@ -0,0 +1,28 @@ +package com.evolutiongaming.jsonitertool + +import org.scalacheck.Prop.forAll +import org.scalacheck.{Arbitrary, Gen, Test} +import play.api.libs.json.Json +import valuegen.RandomJsArrayGen + +//sbt playJsonJsoniter/test:"runMain com.evolutiongaming.jsonitertool.RandomJsonArraysSpec" +object RandomJsonArraysSpec extends org.scalacheck.Properties("RandomJsonSpec") { + + val Size = 5000 + + //produces any imaginable Json array + def randomArrayGen: Gen[value.JsArray] = RandomJsArrayGen() + + implicit def generator: Arbitrary[value.JsArray] = Arbitrary(randomArrayGen) + + override def overrideParameters(p: Test.Parameters): Test.Parameters = + p.withMinSuccessfulTests(Size) + + property("Random json arrays") = forAll { array: value.JsArray => + val json = array.toString + val jsValue = Json.parse(json) + val bts = PlayJsonJsoniter.serialize(jsValue) + val actJsValue = PlayJsonJsoniter.deserialize(bts) + jsValue == actJsValue.get + } +} diff --git a/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/RandomJsonObjectsSpec.scala b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/RandomJsonObjectsSpec.scala new file mode 100644 index 0000000..6a13ed8 --- /dev/null +++ b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/RandomJsonObjectsSpec.scala @@ -0,0 +1,28 @@ +package com.evolutiongaming.jsonitertool + +import org.scalacheck.Prop.forAll +import org.scalacheck.{Arbitrary, Gen, Test} +import play.api.libs.json.Json +import valuegen.RandomJsObjGen + +//sbt playJsonJsoniter/test:"runMain com.evolutiongaming.jsonitertool.RandomJsonObjectsSpec" +object RandomJsonObjectsSpec extends org.scalacheck.Properties("RandomJsonObjectsSpec") { + + val Size = 5000 + + //produces any imaginable Json object + def randomObjGen: Gen[value.JsObj] = RandomJsObjGen() + + implicit def generator: Arbitrary[value.JsObj] = Arbitrary(randomObjGen) + + override def overrideParameters(p: Test.Parameters): Test.Parameters = + p.withMinSuccessfulTests(Size) + + property("Random json objects") = forAll { obj: value.JsObj => + val json = obj.toString + val jsValue = Json.parse(json) + val bts = PlayJsonJsoniter.serialize(jsValue) + val actJsValue = PlayJsonJsoniter.deserialize(bts) + jsValue == actJsValue.get + } +} diff --git a/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/TestDataGenerators.scala b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/TestDataGenerators.scala new file mode 100644 index 0000000..9d2dec6 --- /dev/null +++ b/play-json-jsoniter/src/test/scala-2.13/com/evolutiongaming/jsonitertool/TestDataGenerators.scala @@ -0,0 +1,60 @@ +package com.evolutiongaming.jsonitertool + +import play.api.libs.json.Json +import valuegen.Preamble._ +import valuegen.{JsArrayGen, JsObjGen} +import value.JsObj +import org.scalacheck.{Arbitrary, Gen} + +object TestDataGenerators extends PlayJsonImplicits { + + val ALPHABET: Vector[String] = "abcdefghijklmnopqrstuvwzyz".split("").toVector + + def str(len: Int): Gen[String] = Gen.listOfN(len, Gen.choose(0, ALPHABET.size - 1).map(ALPHABET(_))).map(_.mkString("")) + + def flagGen: Gen[Boolean] = Arbitrary.arbitrary[Boolean] + + def nameGen: Gen[String] = Gen.choose(5, 20).flatMap(str) + + def birthDateGen: Gen[String] = Gen.choose(5, 20).flatMap(str) + + def latitudeGen: Gen[Double] = Arbitrary.arbitrary[Double] + + def longitudeGen: Gen[Double] = Arbitrary.arbitrary[Double] + + def longGen: Gen[Long] = Arbitrary.arbitrary[Long] + + def ints: Gen[Int] = Arbitrary.arbitrary[Int] + + def emailGen: Gen[String] = Gen.choose(5, 20).flatMap(str) + + def countryGen: Gen[String] = Gen.choose(5, 20).flatMap(str) + + def friendGen(depth: Int): Gen[JsObj] = JsObjGen( + "name" -> nameGen, + "email" -> emailGen, + "flag" -> flagGen, + "l" -> longGen, + "from" -> JsObjGen("country" -> countryGen, "doubles" -> JsArrayGen(latitudeGen, longitudeGen)), + "metrics" -> JsArrayGen(ints, ints), + "friends" -> (if (depth > 0) JsArrayGen(friendGen(depth - 1)) else JsArrayGen()) + ) + + def objGen: Gen[JsObj] = JsObjGen( + "name" -> nameGen, + "email" -> emailGen, + "flag" -> flagGen, + "l" -> longGen, + "from" -> JsObjGen("country" -> countryGen, "doubles" -> JsArrayGen(latitudeGen, longitudeGen)), + "metrics" -> JsArrayGen(ints, ints, ints, ints, ints), + "friends" -> JsArrayGen(friendGen(depth = 50)) + ) + + def genUser: Gen[User] = objGen + .map(json => Json.fromJson[User](Json.parse(json.toString)).get) + + case class Address(country: String, doubles: List[Double]) + + case class User(name: String, email: String, flag: Boolean, from: Address, l: Long, metrics: List[Int], + friends: List[User]) +} diff --git a/play-json-jsoniter/src/test/scala/com/evolutiongaming/jsonitertool/JsoniterSpec.scala b/play-json-jsoniter/src/test/scala/com/evolutiongaming/jsonitertool/JsoniterSpec.scala new file mode 100644 index 0000000..265e932 --- /dev/null +++ b/play-json-jsoniter/src/test/scala/com/evolutiongaming/jsonitertool/JsoniterSpec.scala @@ -0,0 +1,71 @@ +package com.evolutiongaming.jsonitertool + +import com.evolutiongaming.jsonitertool.TestData.DataLine +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import play.api.libs.json.{JsSuccess, Json, JsonParserSettings} +import TestData._ + +import scala.util.Success + +class JsoniterSpec extends AnyFunSuite with Matchers { + + val maxDoubleStr = "179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368" + + test("Write using PlayJson -> Read using Jsoniter: Compare bytes") { + + val expected: DataLine = Json.fromJson[DataLine](Json.parse(TestData.jsonBody)) + .fold(errs => throw new Exception(s"Parsing error: ${errs.mkString(",")}"), identity) + + val jsValue = Json.toJson(expected) + val bts0 = PlayJsonJsoniter.serialize(jsValue) + val bts1 = Json.toBytes(jsValue) + java.util.Arrays.compare(bts0, bts1) shouldEqual 0 + } + + test("Write using PlayJson -> Read using Jsoniter: Compare objects") { + + val expected: DataLine = Json.fromJson[DataLine](Json.parse(TestData.jsonBody)) + .fold(errs => throw new Exception(s"Parsing error: ${errs.mkString(",")}"), identity) + + val bts = Json.toBytes(Json.toJson(expected)) + val jsValue = PlayJsonJsoniter.deserialize(bts).map(Json.fromJson[DataLine](_)) + Success(JsSuccess(expected)) shouldEqual jsValue + } + + test("Can write/read large number by play-json") { + //when number size hits length 35, equality comparison doesn't work anymore + val largeNum = "9999999999999999999999999999999911" + val jsValue = play.api.libs.json.JsNumber(BigDecimal(largeNum)) + val bytes = Json.toBytes(jsValue) + new String(bytes) shouldEqual largeNum + Json.parse(bytes) shouldEqual jsValue + } + + test("Can write/read large number by jsoniter without loosing precision") { + val largeNum = "9999999999999999999999999999999911" + val jsValue = play.api.libs.json.JsNumber(BigDecimal(largeNum)) + val bytes = PlayJsonJsoniter.serialize(jsValue) + new String(bytes) shouldEqual largeNum + PlayJsonJsoniter.deserialize(bytes) shouldEqual Success(jsValue) + } + + test("Can parse max double string as play json") { + val json = s"""{ "max":$maxDoubleStr }""" + val jsValue0 = Json.parse(json.getBytes) + jsValue0 shouldEqual PlayJsonJsoniter.deserialize(json.getBytes).get + } + + test("PlayJson and Jsoniter can parse max double") { + val maxDouble = maxDoubleStr + val jsValue0 = Json.parse(maxDouble.getBytes) + jsValue0 shouldEqual PlayJsonJsoniter.deserialize(maxDouble.getBytes).get + } + + test("PlayJson and Jsoniter can parse negative double max") { + val maxDouble = "-" + maxDoubleStr + val jsValue0 = Json.parse(maxDouble.getBytes) + JsonParserSettings.settings.bigDecimalParseSettings.digitsLimit shouldEqual maxDouble.length + jsValue0 shouldEqual PlayJsonJsoniter.deserialize(maxDouble.getBytes).get + } +} \ No newline at end of file diff --git a/play-json-jsoniter/src/test/scala/com/evolutiongaming/jsonitertool/TestData.scala b/play-json-jsoniter/src/test/scala/com/evolutiongaming/jsonitertool/TestData.scala new file mode 100644 index 0000000..a292219 --- /dev/null +++ b/play-json-jsoniter/src/test/scala/com/evolutiongaming/jsonitertool/TestData.scala @@ -0,0 +1,88 @@ +package com.evolutiongaming.jsonitertool + +import play.api.libs.json.Json + +object TestData { + + case class Friend(id: Int, name: String) + + case class DataLine( + id: String, + index: Int, + guid: String, + isActive: Boolean, + balance: String, + picture: String, + age: Int, + eyeColor: String, + name: String, + gender: String, + company: String, + phone: String, + address: String, + about: String, + registered: String, + latitude: Double, + longitude: Double, + tags: List[String], + friends: List[Friend], + greeting: String, + favoriteFruit: String + ) + + val jsonBody = + """ + | { + | "id": "576278df40bd978bc65ddb06", + | "index": 0, + | "guid": "e4c30481-84f3-4193-a83c-fe0435348774", + | "isActive": true, + | "balance": "$1,355.27", + | "picture": "http://placehold.it/32x32", + | "age": 26, + | "eyeColor": "green", + | "name": "Stewart Navarro", + | "gender": "male", + | "company": "NORSUP", + | "phone": "+1 (916) 543-2895", + | "address": "133 Stillwell Place, wrtywrt", + | "about": "Eiusmod excepteur do esse minim nisi occaecat enim non dolor labore ipsum ut. Fugiat deserunt est pariatur pariatur. Laboris aute cillum tempor in exercitation laboris laboris fugiat velit enim ut ad. Ea labore commodo consectetur ut anim anim sint consectetur commodo.\r\n", + | "registered": "2015-05-01T05:24:06 -10:00", + | "latitude": -29.132033, + | "longitude": -58.249295, + | "tags": [ + | "sunt", + | "Lorem", + | "adipisicing", + | "duis", + | "nulla", + | "sint", + | "fugiat" + | ], + | "friends": [ + | { + | "id": 0, + | "name": "Yates Pruitt" + | }, + | { + | "id": 1, + | "name": "Chen Henry" + | }, + | { + | "id": 2, + | "name": "Miranda Vincent" + | } + | ], + | "greeting": "Hello, Stewart Navarro! You have 7 unread messages", + | "favoriteFruit": "strawberry" + | } + """.stripMargin + + + implicit val a = Json.reads[Friend] + implicit val b = Json.writes[Friend] + + implicit val c = Json.reads[DataLine] + implicit val d = Json.writes[DataLine] + +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b935d4d..328b06f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,11 +2,14 @@ import sbt._ object Dependencies { - + val shapeless = "com.chuusai" %% "shapeless" % "2.3.3" val nel = "com.evolutiongaming" %% "nel" % "1.3.4" val playJson = "com.typesafe.play" %% "play-json" % "2.7.4" val scalaTest = "org.scalatest" %% "scalatest" % "3.1.0" + val jsoniter = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.0.4" + val jsonGenerator = "com.github.imrafaelmerino" %% "json-scala-values-generator" % "1.0.0" + val collectionCompact = "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.3" def excludeLog4j(moduleID: ModuleID): ModuleID = moduleID.excludeAll( ExclusionRule("log4j", "log4j"), diff --git a/project/build.properties b/project/build.properties index d761d99..6d7787a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.4 \ No newline at end of file +sbt.version=1.3.6 \ No newline at end of file