From 68f50c34f8ab936ee64dbb2d89ababf5d3835511 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Mon, 16 Sep 2024 09:06:46 +0100 Subject: [PATCH 1/3] Add AdCryptoUtils --- README.md | 8 ++ .../gov/hmrc/crypto/json/AdCryptoUtils.scala | 91 ++++++++++++++ .../hmrc/crypto/json/AdCryptoUtilsSpec.scala | 114 ++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala create mode 100644 crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala diff --git a/README.md b/README.md index ba7cf90..c411d51 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,14 @@ val decrypter = JsonEncryption.sensitiveDecrypter[T, SensitiveT[T]](SensitiveT.a val optValue: Option[T] = decrypter.reads(encryptedValue).asOpt.map(_.decryptedValue) ``` +#### AdCryptoUtils + +Contains utilities for working with associated data. + +`AdCryptoUtils.encryptWith` will wrap a given json `Format` with AD encryption. It requires a pointer (`JSPath`) to the associated data field and takes a list of pointers to the fields to encrypt. + +See [spec](crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala) for an example. + ## Changes ### Version 8.0.0 diff --git a/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala b/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala new file mode 100644 index 0000000..ae7c173 --- /dev/null +++ b/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala @@ -0,0 +1,91 @@ +/* + * Copyright 2024 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.crypto.json + +import play.api.libs.json._ +import uk.gov.hmrc.crypto.{AdDecrypter, AdEncrypter} + +object AdCryptoUtils { + + /** Will adapt a Format (written plainly without encryption) to one applying encryption with associated data. + * It requires a JsPath to point to the associated data field, and JsPaths to point at the fields to be encrypted with it. + * + * @param associatedDataPath field to be used as the associated data. It must resolve to a single `String` field, otherwise use of the Format will result in an error. + * + * @param encryptedFieldPaths fields to be encrypted. The fields can be optional - if the field does not exist, it simply won't be encrypted. This does mean that client's tests must ensure that encryption occurs as expected to detect misspellings etc. + */ + def encryptWith[A]( + associatedDataPath : JsPath, + encryptedFieldPaths: Seq[JsPath] + )( + f: Format[A] + )(implicit + crypto: AdEncrypter with AdDecrypter + ): Format[A] = + Format( + f.preprocess(js => decryptTransform(associatedDataPath, encryptedFieldPaths)(js)), + f.transform(encryptTransform(associatedDataPath, encryptedFieldPaths)) + ) + + private def decryptTransform(associatedDataPath: JsPath, encryptedFieldPaths: Seq[JsPath])(jsValue: JsValue)(implicit crypto: AdEncrypter with AdDecrypter): JsValue = { + lazy val ad = associatedData(associatedDataPath, jsValue) + def transform(js: JsValue): JsValue = + CryptoFormats.encryptedValueFormat.reads(js) match { + case JsSuccess(ev, _) => Json.parse(crypto.decrypt(ev, ad)) + case JsError(errors) => sys.error(s"Failed to decrypt value: $errors") + } + encryptedFieldPaths.foldLeft(jsValue){ (js, encryptedFieldPath) => + if (encryptedFieldPath(jsValue).nonEmpty) + transformWithoutMerge(js, encryptedFieldPath, transform) match { + case JsSuccess(r, _) => r + case JsError(errors) => sys.error(s"Could not decrypt at $encryptedFieldPath: $errors") + } + else + js + } + } + + private def encryptTransform(associatedDataPath: JsPath, encryptedFieldPaths: Seq[JsPath])(jsValue: JsValue)(implicit crypto: AdEncrypter with AdDecrypter): JsValue = { + lazy val ad = associatedData(associatedDataPath, jsValue) + def transform(js: JsValue): JsValue = + CryptoFormats.encryptedValueFormat.writes(crypto.encrypt(js.toString, ad)) + encryptedFieldPaths.foldLeft(jsValue){ (js, encryptedFieldPath) => + if (encryptedFieldPath(jsValue).nonEmpty) + transformWithoutMerge(js, encryptedFieldPath, transform) match { + case JsSuccess(r, _) => r + case JsError(errors) => sys.error(s"Could not encrypt at $encryptedFieldPath: $errors") + } + else + js + } + } + + private def associatedData(associatedDataPath: JsPath, jsValue: JsValue) = + associatedDataPath(jsValue) match { + case List(associatedData) => associatedData.as[String] // Note only supports associatedDataPath which points to a String + case Nil => sys.error(s"No associatedData was found with $associatedDataPath") + case _ => sys.error(s"Multiple associatedData was found with $associatedDataPath") + } + + private def transformWithoutMerge(js: JsValue, path: JsPath, transformFn: JsValue => JsValue): JsResult[JsValue] = + js.transform( + path.json.update(implicitly[Reads[JsValue]].map(_ => transformFn(path(js).head))) + // Calling js.transform(path.json.update ...) actually merges the result of transform + // So we have to ensure the original (unencrypted) JsObject is removed + .composeWith(path.json.update(implicitly[Reads[JsValue]].map(_ => JsNull))) + ) +} diff --git a/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala b/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala new file mode 100644 index 0000000..6a0dc7a --- /dev/null +++ b/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2024 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.crypto.json + +import org.scalatest.OptionValues +import org.scalatest.wordspec.AnyWordSpecLike +import org.scalatest.matchers.should.Matchers +import play.api.libs.functional.syntax._ +import play.api.libs.json.{Format, Json, JsSuccess, Reads, __} +import uk.gov.hmrc.crypto.{AdDecrypter, AdEncrypter, EncryptedValue} + +class AdCryptoUtilsSpec + extends AnyWordSpecLike + with Matchers + with OptionValues { + import AdCryptoUtilsSpec._ + + "sensitiveEncrypterDecrypter" should { + "encrypt/decrypt primitives" in { + + val testEntityFormat: Format[TestEntity] = { + implicit val tsef: Format[TestSubEntity] = + ( (__ \ "aField").format[String] + ~ (__ \ "bField").format[String] + )(TestSubEntity.apply, o => (o.aField, o.bField)) + ( (__ \ "name" ).format[String] + ~ (__ \ "aString" ).format[String] + ~ (__ \ "aBoolean").format[Boolean] + ~ (__ \ "aNumber" ).format[BigDecimal] + ~ (__ \ "aObject" ).format[TestSubEntity] + )(TestEntity.apply, o => (o.name, o.aString, o.aBoolean, o.aNumber, o.aObject)) + } + + val cryptoFormat: Format[TestEntity] = + AdCryptoUtils.encryptWith[TestEntity]( + associatedDataPath = __ \ "name", + encryptedFieldPaths = Seq( + __ \ "aString", + __ \ "aBoolean", + __ \ "aNumber", + __ \ "aObject" + ) + )(testEntityFormat) + + val testEntity = TestEntity( + name = "name", + aString = "string", + aBoolean = true, + aNumber = BigDecimal(1.0), + aObject = TestSubEntity( + aField = "aField", + bField = "bField" + ) + ) + + val cryptoJson = cryptoFormat.writes(testEntity) + + // be able to read json back + Json.fromJson[TestEntity](cryptoJson)(cryptoFormat).asOpt.value shouldBe testEntity + + // not contain raw values + (cryptoJson \ "name").as[String] shouldBe "name" + cryptoJson.toString should not include "string" + cryptoJson.toString should not include "true" + cryptoJson.toString should not include "aField" + cryptoJson.toString should not include "bField" + + // be encrypted + implicit val evr: Reads[EncryptedValue] = CryptoFormats.encryptedValueFormat + (cryptoJson \ "aString" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "aBoolean").validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "aNumber" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "aObject" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + } + } +} + +object AdCryptoUtilsSpec { + implicit val crypto: AdEncrypter with AdDecrypter = { + val aesKey = { + val aesKey = new Array[Byte](32) + new java.security.SecureRandom().nextBytes(aesKey) + java.util.Base64.getEncoder.encodeToString(aesKey) + } + uk.gov.hmrc.crypto.SymmetricCryptoFactory.aesGcmAdCrypto(aesKey) + } + + case class TestEntity( + name : String, + aString : String, + aBoolean: Boolean, + aNumber : BigDecimal, + aObject : TestSubEntity + ) + + case class TestSubEntity( + aField: String, + bField: String + ) +} From 9acf98e0dd37ff4727b906aca3544a63c3189da7 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Mon, 16 Sep 2024 09:38:10 +0100 Subject: [PATCH 2/3] Bump dependencies --- project/LibDependencies.scala | 6 +++--- project/plugins.sbt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project/LibDependencies.scala b/project/LibDependencies.scala index 7ddfe58..5ffcc3b 100644 --- a/project/LibDependencies.scala +++ b/project/LibDependencies.scala @@ -11,7 +11,7 @@ object LibDependencies { "org.scalatest" %% "scalatest" % "3.2.17" % Test, "com.vladsch.flexmark" % "flexmark-all" % "0.64.8" % Test, "org.scalatestplus" %% "scalacheck-1-17" % "3.2.17.0" % Test, - "org.scalatestplus" %% "mockito-3-4" % "3.2.10.0" % Test + "org.scalatestplus" %% "mockito-4-11" % "3.2.17.0" % Test ) val cryptoJsonPlay28Compile = Seq( @@ -19,10 +19,10 @@ object LibDependencies { ) val cryptoJsonPlay29Compile = Seq( - "com.typesafe.play" %% "play-json" % "2.10.5" // version provided by Play 2.9 + "com.typesafe.play" %% "play-json" % "2.10.6" // version provided by Play 2.9 ) val cryptoJsonPlay30Compile = Seq( - "org.playframework" %% "play-json" % "3.0.3" // version provided by Play 3.0 + "org.playframework" %% "play-json" % "3.0.4" // version provided by Play 3.0 ) } diff --git a/project/plugins.sbt b/project/plugins.sbt index e8fc669..804d624 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefacts.tax.service.gov.uk/maven2") resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))(Resolver.ivyStylePatterns) -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.21.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.22.0") From 01df72eedcb5bec9efeabfca89c8f8193839bf3f Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:45:43 +0100 Subject: [PATCH 3/3] Rework transformWithoutMerge --- .../gov/hmrc/crypto/json/AdCryptoUtils.scala | 59 ++++++++------- .../hmrc/crypto/json/AdCryptoUtilsSpec.scala | 73 +++++++++++-------- 2 files changed, 73 insertions(+), 59 deletions(-) diff --git a/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala b/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala index ae7c173..0ad0d2e 100644 --- a/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala +++ b/crypto-json/src/main/scala/uk/gov/hmrc/crypto/json/AdCryptoUtils.scala @@ -26,7 +26,9 @@ object AdCryptoUtils { * * @param associatedDataPath field to be used as the associated data. It must resolve to a single `String` field, otherwise use of the Format will result in an error. * - * @param encryptedFieldPaths fields to be encrypted. The fields can be optional - if the field does not exist, it simply won't be encrypted. This does mean that client's tests must ensure that encryption occurs as expected to detect misspellings etc. + * @param encryptedFieldPaths fields to be encrypted. + * The fields can be optional - if the field does not exist, it simply won't be encrypted. This does mean that client's tests must ensure that encryption occurs as expected to detect misspellings etc. + * The field path cannot point to multiple values. */ def encryptWith[A]( associatedDataPath : JsPath, @@ -49,13 +51,10 @@ object AdCryptoUtils { case JsError(errors) => sys.error(s"Failed to decrypt value: $errors") } encryptedFieldPaths.foldLeft(jsValue){ (js, encryptedFieldPath) => - if (encryptedFieldPath(jsValue).nonEmpty) - transformWithoutMerge(js, encryptedFieldPath, transform) match { - case JsSuccess(r, _) => r - case JsError(errors) => sys.error(s"Could not decrypt at $encryptedFieldPath: $errors") - } - else - js + js.transform(updateWithoutMerge(encryptedFieldPath, transform)) match { + case JsSuccess(r, _) => r + case JsError(errors) => sys.error(s"Could not decrypt at $encryptedFieldPath: $errors") + } } } @@ -64,28 +63,34 @@ object AdCryptoUtils { def transform(js: JsValue): JsValue = CryptoFormats.encryptedValueFormat.writes(crypto.encrypt(js.toString, ad)) encryptedFieldPaths.foldLeft(jsValue){ (js, encryptedFieldPath) => - if (encryptedFieldPath(jsValue).nonEmpty) - transformWithoutMerge(js, encryptedFieldPath, transform) match { - case JsSuccess(r, _) => r - case JsError(errors) => sys.error(s"Could not encrypt at $encryptedFieldPath: $errors") - } - else - js + js.transform(updateWithoutMerge(encryptedFieldPath, transform)) match { + case JsSuccess(r, _) => r + case JsError(errors) => sys.error(s"Could not encrypt at $encryptedFieldPath: $errors") + } } } private def associatedData(associatedDataPath: JsPath, jsValue: JsValue) = - associatedDataPath(jsValue) match { - case List(associatedData) => associatedData.as[String] // Note only supports associatedDataPath which points to a String - case Nil => sys.error(s"No associatedData was found with $associatedDataPath") - case _ => sys.error(s"Multiple associatedData was found with $associatedDataPath") - } + associatedDataPath.asSingleJsResult(jsValue) + // Note only supports associatedDataPath which points to a String + .flatMap(_.validate[String]) + .fold(es => sys.error(s"Failed to look up associated data: $es"), identity) - private def transformWithoutMerge(js: JsValue, path: JsPath, transformFn: JsValue => JsValue): JsResult[JsValue] = - js.transform( - path.json.update(implicitly[Reads[JsValue]].map(_ => transformFn(path(js).head))) - // Calling js.transform(path.json.update ...) actually merges the result of transform - // So we have to ensure the original (unencrypted) JsObject is removed - .composeWith(path.json.update(implicitly[Reads[JsValue]].map(_ => JsNull))) - ) + private def updateWithoutMerge(path: JsPath, transformFn: JsValue => JsValue): Reads[JsObject] = + // not using `path.json.update(o => JsSuccess(transformFn(o)))` since this does a deep merge - keeping the unencrypted values around for JsObject + Reads[JsObject] { + case o: JsObject => + path(o) match { + case Nil => JsSuccess(o) + case List(one: JsObject) => JsSuccess( + o + .deepMerge(JsPath.createObj(path -> JsNull)) // ensure we don't merge with existing clear-text data + .deepMerge(JsPath.createObj(path -> transformFn(one))) + ) + case List(one) => JsSuccess(o.deepMerge(JsPath.createObj(path -> transformFn(one)))) + case multiple => JsError(Seq(path -> Seq(JsonValidationError("error.path.result.multiple")))) + } + case _ => + JsError(JsPath, JsonValidationError("error.expected.jsobject")) + } } diff --git a/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala b/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala index 6a0dc7a..fb196b2 100644 --- a/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala +++ b/crypto-json/src/test/scala/uk/gov/hmrc/crypto/json/AdCryptoUtilsSpec.scala @@ -37,35 +37,40 @@ class AdCryptoUtilsSpec ( (__ \ "aField").format[String] ~ (__ \ "bField").format[String] )(TestSubEntity.apply, o => (o.aField, o.bField)) - ( (__ \ "name" ).format[String] - ~ (__ \ "aString" ).format[String] - ~ (__ \ "aBoolean").format[Boolean] - ~ (__ \ "aNumber" ).format[BigDecimal] - ~ (__ \ "aObject" ).format[TestSubEntity] - )(TestEntity.apply, o => (o.name, o.aString, o.aBoolean, o.aNumber, o.aObject)) + ( (__ \ "asd" \ "name" ).format[String] + ~ (__ \ "asd" \ "aString" ).format[String] + ~ (__ \ "asd" \ "aBoolean").format[Boolean] + ~ (__ \ "asd" \ "aNumber" ).format[BigDecimal] + ~ (__ \ "asd" \ "aObject" ).format[TestSubEntity] + ~ (__ \ "asd" \ "anArray" ).format[List[String]] + )(TestEntity.apply, o => (o.name, o.aString, o.aBoolean, o.aNumber, o.aObject, o.anArray)) } val cryptoFormat: Format[TestEntity] = AdCryptoUtils.encryptWith[TestEntity]( - associatedDataPath = __ \ "name", + associatedDataPath = __ \ "asd" \ "name", encryptedFieldPaths = Seq( - __ \ "aString", - __ \ "aBoolean", - __ \ "aNumber", - __ \ "aObject" + __ \ "asd" \ "aString", + __ \ "asd" \ "aBoolean", + __ \ "asd" \ "aNumber", + __ \ "asd" \ "aObject", + __ \ "asd" \ "anArray", + __ \ "nonExisting" ) )(testEntityFormat) - val testEntity = TestEntity( - name = "name", - aString = "string", - aBoolean = true, - aNumber = BigDecimal(1.0), - aObject = TestSubEntity( - aField = "aField", - bField = "bField" - ) - ) + val testEntity = + TestEntity( + name = "name", + aString = "string", + aBoolean = true, + aNumber = BigDecimal(1.0), + aObject = TestSubEntity( + aField = "aField", + bField = "bField" + ), + anArray = List("array0", "array1") + ) val cryptoJson = cryptoFormat.writes(testEntity) @@ -73,18 +78,21 @@ class AdCryptoUtilsSpec Json.fromJson[TestEntity](cryptoJson)(cryptoFormat).asOpt.value shouldBe testEntity // not contain raw values - (cryptoJson \ "name").as[String] shouldBe "name" + (cryptoJson \ "asd" \ "name").as[String] shouldBe "name" cryptoJson.toString should not include "string" cryptoJson.toString should not include "true" cryptoJson.toString should not include "aField" - cryptoJson.toString should not include "bField" + cryptoJson.toString should not include "bField0" + cryptoJson.toString should not include "array0" + cryptoJson.toString should not include "array1" // be encrypted implicit val evr: Reads[EncryptedValue] = CryptoFormats.encryptedValueFormat - (cryptoJson \ "aString" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] - (cryptoJson \ "aBoolean").validate[EncryptedValue] shouldBe a[JsSuccess[_]] - (cryptoJson \ "aNumber" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] - (cryptoJson \ "aObject" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "asd" \ "aString" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "asd" \ "aBoolean").validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "asd" \ "aNumber" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "asd" \ "aObject" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] + (cryptoJson \ "asd" \ "anArray" ).validate[EncryptedValue] shouldBe a[JsSuccess[_]] } } } @@ -100,11 +108,12 @@ object AdCryptoUtilsSpec { } case class TestEntity( - name : String, - aString : String, - aBoolean: Boolean, - aNumber : BigDecimal, - aObject : TestSubEntity + name : String, + aString : String, + aBoolean : Boolean, + aNumber : BigDecimal, + aObject : TestSubEntity, + anArray : List[String] ) case class TestSubEntity(