Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

902 - remove jackson json dependency #905

Merged
merged 2 commits into from
Sep 24, 2020
Merged

902 - remove jackson json dependency #905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
85346cb
Remove the dependency on jackson json library in favour of circe
ivanopagano Sep 3, 2020
22d477d
Bring the regression tests up-to-date as to be able to run them
ivanopagano Sep 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions conseil-api/src/test/scala/tech/cryptonomic/conseil/api/routes/info/AppInfoRouteTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,20 @@ package tech.cryptonomic.conseil.api.routes.info
import akka.http.scaladsl.model.{ContentTypes, StatusCodes}
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.testkit.ScalatestRouteTest
import org.scalatest.OptionValues
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import tech.cryptonomic.conseil.common.util.JsonUtil._
import tech.cryptonomic.conseil.api.routes.info.AppInfo.{GitInfo, Info}
import io.circe._
import io.circe.generic.extras.semiauto._
import io.circe.generic.extras.Configuration

class AppInfoRouteTest extends AnyWordSpec with Matchers with ScalatestRouteTest {
class AppInfoRouteTest extends AnyWordSpec with Matchers with OptionValues with ScalatestRouteTest {

implicit val derivation = Configuration.default
implicit val gitInfoDecoder = deriveDecoder[GitInfo]
implicit val infoDecoder = deriveDecoder[Info]

"The application info route" should {

Expand All @@ -20,10 +29,10 @@ class AppInfoRouteTest extends AnyWordSpec with Matchers with ScalatestRouteTest
Get("/info") ~> addHeader("apiKey", "hooman") ~> sut ~> check {
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val info: Map[String, Any] = toMap[Any](responseAs[String])
info("application") shouldBe "Conseil"
info("version").toString should fullyMatch regex """^0\.\d{4}\.\d{4}(-SNAPSHOT)?"""
info("git").asInstanceOf[Map[String, String]]("commitHash") should fullyMatch regex """^[0-9a-f]+$"""
val info: Info = fromJson[Info](responseAs[String]).get
info.application shouldBe "Conseil"
info.version should fullyMatch regex """^0\.\d{4}\.\d{4}(-SNAPSHOT)?"""
info.git.commitHash.value should fullyMatch regex """^[0-9a-f]+$"""
}
}

Expand Down
68 changes: 43 additions & 25 deletions .../scala/tech/cryptonomic/conseil/api/routes/platform/discovery/PlatformDiscoveryTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,16 @@ package tech.cryptonomic.conseil.api.routes.platform.discovery
import akka.http.scaladsl.model.{ContentTypes, StatusCodes}
import akka.http.scaladsl.testkit.ScalatestRouteTest
import org.scalamock.scalatest.MockFactory
import org.scalatest.OptionValues
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import io.circe._
import tech.cryptonomic.conseil.api.metadata.{AttributeValuesCacheConfiguration, MetadataService, UnitTransformation}
import tech.cryptonomic.conseil.common.config.Platforms.{PlatformsConfiguration, TezosConfiguration, TezosNodeConfiguration}
import tech.cryptonomic.conseil.common.config.Platforms.{
PlatformsConfiguration,
TezosConfiguration,
TezosNodeConfiguration
}
import tech.cryptonomic.conseil.common.config.Types.PlatformName
import tech.cryptonomic.conseil.common.config._
import tech.cryptonomic.conseil.common.generic.chain.PlatformDiscoveryTypes.DataType.Int
Expand All @@ -15,7 +21,12 @@ import tech.cryptonomic.conseil.common.generic.chain.PlatformDiscoveryTypes.{Att
import tech.cryptonomic.conseil.common.metadata.{EntityPath, NetworkPath, PlatformPath}
import tech.cryptonomic.conseil.common.util.JsonUtil.toListOfMaps

class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRouteTest with MockFactory {
class PlatformDiscoveryTest
extends AnyWordSpec
with Matchers
with OptionValues
with ScalatestRouteTest
with MockFactory {

"The platform discovery route" should {

Expand Down Expand Up @@ -54,7 +65,7 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
// then
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String])
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String]).get
result.head("name") shouldBe "tezos"
result.head("displayName") shouldBe "Tezos"
}
Expand All @@ -70,7 +81,7 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
// then
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String])
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String]).get
result.size shouldBe 0
}
}
Expand All @@ -86,7 +97,7 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
// then
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String])
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String]).get
result.head("displayName") shouldBe "overwritten-name"
result.head("description") shouldBe "description"
}
Expand All @@ -113,7 +124,7 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
// then
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String])
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String]).get
result.head("name") shouldBe "mainnet"
result.head("displayName") shouldBe "Mainnet"
}
Expand Down Expand Up @@ -150,10 +161,14 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
// then
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String])
result.head("name") shouldBe "entity"
result.head("displayName") shouldBe "entity-name"
result.head("count") shouldBe "1"

val parseResult = parser.parse(responseAs[String]).getOrElse(Json.arr())
parseResult.isArray shouldBe true

val headResult = parseResult.asArray.value.head.asObject.value
headResult("name").value.asString.value shouldBe "entity"
headResult("displayName").value.asString.value shouldBe "entity-name"
headResult("count").value shouldBe Json.fromInt(1)
}
}

Expand Down Expand Up @@ -203,7 +218,7 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
// then
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String])
val result: List[Map[String, String]] = toListOfMaps[String](responseAs[String]).get
result.head("name") shouldBe "attribute"
result.head("displayName") shouldBe "attribute-name"
}
Expand Down Expand Up @@ -270,20 +285,23 @@ class PlatformDiscoveryTest extends AnyWordSpec with Matchers with ScalatestRout
status shouldEqual StatusCodes.OK
contentType shouldBe ContentTypes.`application/json`

val headResult = toListOfMaps[Any](responseAs[String]).head
headResult("name") shouldBe "attribute"
headResult("displayName") shouldBe "attribute-name"
headResult("description") shouldBe "description"
headResult("placeholder") shouldBe "placeholder"
headResult("dataFormat") shouldBe "dataFormat"
headResult("scale") shouldBe 6
headResult("valueMap") shouldBe Map("0" -> "value")
headResult("dataType") shouldBe "Hash"
headResult("reference") shouldBe Map("0" -> "value")
headResult("displayPriority") shouldBe 1
headResult("displayOrder") shouldBe 1
headResult("currencySymbol") shouldBe "ꜩ"
headResult("currencySymbolCode") shouldBe 42793
val parseResult = parser.parse(responseAs[String]).getOrElse(Json.arr())
parseResult.isArray shouldBe true

val headResult: JsonObject = parseResult.asArray.value.head.asObject.value
headResult("name").value.asString.value shouldBe "attribute"
headResult("displayName").value.asString.value shouldBe "attribute-name"
headResult("description").value.asString.value shouldBe "description"
headResult("placeholder").value.asString.value shouldBe "placeholder"
headResult("dataFormat").value.asString.value shouldBe "dataFormat"
headResult("scale").value shouldBe Json.fromInt(6)
headResult("valueMap").value.asObject.value.toMap shouldBe Map("0" -> Json.fromString("value"))
headResult("dataType").value.asString.value shouldBe "Hash"
headResult("reference").value.asObject.value.toMap shouldBe Map("0" -> Json.fromString("value"))
headResult("displayPriority").value shouldBe Json.fromInt(1)
headResult("displayOrder").value shouldBe Json.fromInt(1)
headResult("currencySymbol").value.asString.value shouldBe "ꜩ"
headResult("currencySymbolCode").value shouldBe Json.fromInt(42793)
}
}

Expand Down
61 changes: 31 additions & 30 deletions conseil-common/src/main/scala/tech/cryptonomic/conseil/common/util/JsonUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package tech.cryptonomic.conseil.common.util

import com.fasterxml.jackson.core.{JsonParseException, JsonParser}
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import scala.annotation.tailrec
import io.circe._
import io.circe.parser.decode
import io.circe.syntax._
import scala.util.Try

import scala.util.matching.Regex
Expand Down Expand Up @@ -41,6 +39,8 @@ object JsonUtil {

object JsonString {

case class InvalidJsonString[T <: Throwable](cause: T) extends Throwable

// Note: instead of making it private, it might make sense to verify the input
// and return the [[JsonString]] within a wrapping effect (e.g. Option, Try, Either)
private[JsonUtil] def apply(json: String): JsonString = new JsonString(json)
Expand All @@ -50,16 +50,13 @@ object JsonUtil {
* @param s the "stringified" json
* @return a valid JsonString or a failed [[Try]] with the parsing error
*/
def wrapString(s: String): Try[JsonString] =
Try {
validate(mapper.getFactory.createParser(s))
}.map(_ => JsonString(s))

//verifies if the parser can proceed till the end
@tailrec
@throws[JsonParseException]("when content is not parseable, especially for not well-formed json")
private def validate(parser: JsonParser): Boolean =
parser.nextToken == null || validate(parser)
def fromString(s: String): Try[JsonString] =
io.circe.parser
.parse(s)
.map(_ => JsonString(s))
.left
.map(InvalidJsonString(_))
.toTry

/** A [[JsonString]] representing a json object with no attributes */
lazy val emptyObject = JsonString("{}")
Expand All @@ -71,24 +68,28 @@ object JsonUtil {
.replaceAll("""\\(u[a-zA-Z0-9]{1,4})""", "$1")
}

private val mapper = new ObjectMapper with ScalaObjectMapper
mapper
.registerModule(DefaultScalaModule)
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY)
.disable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)

def toJson[T](value: T): JsonString =
JsonString(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(value))

def toMap[V](json: String)(implicit m: Manifest[V]): Map[String, V] =
fromJson[Map[String, V]](json)
def toJson[T: Encoder](value: T): JsonString =
JsonString(value.asJson.pretty(Printer.spaces4))

def toListOfMaps[V](json: String)(implicit m: Manifest[V]): List[Map[String, V]] =
def toListOfMaps[V: Decoder](json: String): Try[List[Map[String, V]]] =
fromJson[List[Map[String, V]]](json)

def fromJson[T: Manifest](json: String): T =
mapper.readValue[T](JsonString sanitize json)
def fromJson[T: Decoder](json: String): Try[T] = {
val result = decode[T](JsonString sanitize json)

result.left.foreach {
case f @ ParsingFailure(msg, cause) =>
println(
s"Parsing failed for the following json string: $json. This is the error message $msg and the cause is ${cause.getMessage()}"
)
case f @ DecodingFailure(msg, history) =>
println(
s"Decoding to an object failed for the following json string: $json. This is the error message $msg and cursor operations so far: $history"
)
}

result.toTry
}

/** extractor object to read accountIds from a json string, based on the hash format*/
object AccountIds {
Expand Down
43 changes: 23 additions & 20 deletions conseil-common/src/test/scala/tech/cryptonomic/conseil/common/util/JsonUtilTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@ package tech.cryptonomic.conseil.common.util

import org.scalatest.Inspectors._
import JsonUtil._
import JsonUtil.JsonString.InvalidJsonString
import com.stephenn.scalatest.jsonassert.JsonMatchers
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec

case class TestObject(field: String = "")

class JsonUtilTest extends AnyWordSpec with Matchers with JsonMatchers {

"JsonUtil" should {
Expand All @@ -26,16 +25,14 @@ class JsonUtilTest extends AnyWordSpec with Matchers with JsonMatchers {
val arrays = base.mkString("[", ",", "]") :: objects.mkString("[", ",", "]") :: Nil

forAll(base ++ objects ++ arrays) { json =>
val jsonTry = JsonString.wrapString(json)
val jsonTry = JsonString.fromString(json)
jsonTry shouldBe 'success
jsonTry.get shouldBe a[JsonString]
}
}

"fail when wrapping invalid json into a JsonString" in {

import com.fasterxml.jackson.core.JsonParseException

val invalid =
List(
"{", //incomplete
Expand All @@ -45,9 +42,9 @@ class JsonUtilTest extends AnyWordSpec with Matchers with JsonMatchers {
)

forAll(invalid) { string =>
val jsonTry = JsonString.wrapString(string)
val jsonTry = JsonString.fromString(string)
jsonTry shouldBe 'failure
jsonTry.failed.get shouldBe a[JsonParseException]
jsonTry.failed.get shouldBe a[InvalidJsonString[_]]
}
}

Expand Down Expand Up @@ -81,30 +78,36 @@ class JsonUtilTest extends AnyWordSpec with Matchers with JsonMatchers {
result should matchJson("""{"key1": "value1", "key2": "value2"}""")
}

"convert a complex map to json" in {
val result = JsonUtil.toJson(Map("a" -> "b", "c" -> Map("d" -> "e", "f" -> "g"))).json
"convert a nested generic shapeless Record to json" in {
import shapeless.record.Record
/* this does the trick of encoding any Record whose fields have encoders themeselves */
import io.circe.generic.encoding.ReprObjectEncoder._

val record = Record(a = "b", c = Record(d = "e", f = "g"))
val result = JsonUtil.toJson(record).json
result should matchJson("""{"a": "b", "c": {"d": "e", "f": "g"}}""")
}

"convert a simple json to a map" in {
val result = JsonUtil.toMap[String]("""{"key1": "value1", "key2": "value2"}""")
result should be(Map("key1" -> "value1", "key2" -> "value2"))
}
"allow lenient parsing of json with duplicate object keys, actually deserialized or not" in {
case class TestObject(field: String = "")
object TestObject {
import io.circe.Decoder
import io.circe.generic.semiauto._
/* required by Circe to decode from json */
implicit val testObjectDecode: Decoder[TestObject] = deriveDecoder
}

"convert a complex json to a map" in {
val result = JsonUtil.toMap[Any]("""{"a": "b", "c": {"d": "e", "f": "g"}}""")
result should be(Map("a" -> "b", "c" -> Map("d" -> "e", "f" -> "g")))
}
val valid = """{"field": "value"}"""

"allow lenient parsing of json with duplicate object keys, actually deserialized or not" in {
JsonUtil.fromJson[TestObject](valid).get shouldBe TestObject(field = "value")

val duplicateKey = """{"field": "value", "field": "dup"}"""

JsonUtil.fromJson[TestObject](duplicateKey) shouldBe TestObject(field = "dup")
JsonUtil.fromJson[TestObject](duplicateKey).get shouldBe TestObject(field = "dup")

val duplicateInnerKey = """{"field": "value", "inner": {"field":"inner", "field": "dup", "another": 10}}"""

JsonUtil.fromJson[TestObject](duplicateInnerKey) shouldBe TestObject(field = "value")
JsonUtil.fromJson[TestObject](duplicateInnerKey).get shouldBe TestObject(field = "value")
}

}
Expand Down
6 changes: 6 additions & 0 deletions conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosJsonDecoders.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,11 @@ private[tezos] object TezosJsonDecoders {
implicit val operationDecoder: Decoder[Operation] = deriveDecoder
implicit val operationGroupDecoder: Decoder[OperationsGroup] = deriveDecoder
implicit val parametersCompatDecoder: Decoder[ParametersCompatibility] = decodeUntaggedEither
implicit val injectedOperationDecoder: Decoder[InjectedOperation] = deriveDecoder
implicit val appliedOperationBalanceDecoder: Decoder[AppliedOperationBalanceUpdates] = deriveDecoder
implicit val appliedOperationErrorDecoder: Decoder[AppliedOperationError] = deriveDecoder
implicit val appliedOperationResultDecoder: Decoder[AppliedOperationResult] = deriveDecoder
implicit val appliedOperationDecoder: Decoder[AppliedOperation] = deriveDecoder

}

Expand All @@ -284,6 +289,7 @@ private[tezos] object TezosJsonDecoders {

implicit val delegateProtocol4Decoder: Decoder[Protocol4Delegate] = deriveDecoder
implicit val accountDecoder: Decoder[Account] = deriveDecoder
implicit val managerDecoder: Decoder[ManagerKey] = deriveDecoder
}

object Rights {
Expand Down
Loading