Add string-interning data access [DPP-707] (#11484)

* Add new table, and party_id to party_entries
* Add write side basics (DbDto, Schema)
* Add read side basic (StorageBackendStringInterning)
* Add StorageBackend unit test
* Support for all DB backends

changelog_begin
changelog_end

ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256

@@ -1 +1 @@
-86ded6839752d76bb3e3e6cde7f4ec103dea6cee6382b6695de95e84dd74d2da
+e28b88accecead16dcd67becabf7ad230654b65f93c277e785ba77203c3db8b5

ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql

@@ -539,3 +539,8 @@ SELECT
     exercise_result_compression
 FROM participant_events_non_consuming_exercise
 ;
+
+CREATE TABLE string_interning (
+    internal_id integer PRIMARY KEY NOT NULL,
+    external_string text
+);

ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V3__Add_string_interning.sha256

@@ -0,0 +1 @@
+f87eb3e6709109d9fa34a9757cf53e3f8b84584a079442eea6e8da4f4224dc2e

ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V3__Add_string_interning.sql

@@ -0,0 +1,7 @@
+--  Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+--  SPDX-License-Identifier: Apache-2.0
+
+CREATE TABLE string_interning (
+    internal_id      NUMBER          PRIMARY KEY NOT NULL,
+    external_string  VARCHAR2(4000)
+);

ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V112__add_string_interning.sha256

@@ -0,0 +1 @@
+d807d5d30cc3c3a55ad6a7319f21cc8a77f78c0a76fbdc94cd3c6a895e432fb7

ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V112__add_string_interning.sql

@@ -0,0 +1,7 @@
+--  Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+--  SPDX-License-Identifier: Apache-2.0
+
+CREATE TABLE string_interning (
+    internal_id     INTEGER PRIMARY KEY NOT NULL,
+    external_string TEXT
+);

ledger/participant-integration-api/src/main/scala/platform/store/backend/DbDto.scala

@@ -135,4 +135,8 @@ object DbDto {
 
   final case class CommandDeduplication(deduplication_key: String) extends DbDto
 
+  final case class StringInterningDto(
+      internalId: Int,
+      externalString: String,
+  ) extends DbDto
 }

ledger/participant-integration-api/src/main/scala/platform/store/backend/StorageBackend.scala

@@ -48,6 +48,7 @@ trait StorageBackend[DB_BATCH]
     with DBLockStorageBackend
     with IntegrityStorageBackend
     with ResetStorageBackend
+    with StringInterningStorageBackend
 
 trait ResetStorageBackend {
 
@@ -352,6 +353,12 @@ trait IntegrityStorageBackend {
   def verifyIntegrity()(connection: Connection): Unit
 }
 
+trait StringInterningStorageBackend {
+  def loadStringInterningEntries(fromIdExclusive: Int, untilIdInclusive: Int)(
+      connection: Connection
+  ): Iterable[(Int, String)]
+}
+
 object StorageBackend {
   case class RawContractState(
       templateId: Option[String],

ledger/participant-integration-api/src/main/scala/platform/store/backend/StorageBackendFactory.scala

@@ -22,6 +22,7 @@ trait StorageBackendFactory {
   def createDBLockStorageBackend: DBLockStorageBackend
   def createIntegrityStorageBackend: IntegrityStorageBackend
   def createResetStorageBackend: ResetStorageBackend
+  def createStringInterningStorageBackend: StringInterningStorageBackend
 }
 
 object StorageBackendFactory {

ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Field.scala

@@ -65,9 +65,11 @@ private[backend] case class Bytea[FROM](extract: FROM => Array[Byte])
 private[backend] case class ByteaOptional[FROM](extract: FROM => Option[Array[Byte]])
     extends TrivialOptionalField[FROM, Array[Byte]]
 
+private[backend] case class Integer[FROM](extract: FROM => Int) extends TrivialField[FROM, Int]
+
 private[backend] case class IntOptional[FROM](extract: FROM => Option[Int])
     extends Field[FROM, Option[Int], java.lang.Integer] {
-  override def convert: Option[Int] => Integer = _.map(Int.box).orNull
+  override def convert: Option[Int] => java.lang.Integer = _.map(Int.box).orNull
 }
 
 private[backend] case class Bigint[FROM](extract: FROM => Long) extends TrivialField[FROM, Long]
@@ -79,7 +81,7 @@ private[backend] case class BigintOptional[FROM](extract: FROM => Option[Long])
 
 private[backend] case class SmallintOptional[FROM](extract: FROM => Option[Int])
     extends Field[FROM, Option[Int], java.lang.Integer] {
-  override def convert: Option[Int] => Integer = _.map(Int.box).orNull
+  override def convert: Option[Int] => java.lang.Integer = _.map(Int.box).orNull
 }
 
 private[backend] case class BooleanField[FROM](extract: FROM => Boolean)

ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala

@@ -52,6 +52,9 @@ private[backend] object AppendOnlySchema {
     def smallintOptional[FROM, _](extractor: FROM => Option[Int]): Field[FROM, Option[Int], _] =
       SmallintOptional(extractor)
 
+    def int[FROM, _](extractor: FROM => Int): Field[FROM, Int, _] =
+      Integer(extractor)
+
     def intOptional[FROM, _](extractor: FROM => Option[Int]): Field[FROM, Option[Int], _] =
       IntOptional(extractor)
 
@@ -227,6 +230,12 @@ private[backend] object AppendOnlySchema {
         "deduplication_key" -> fieldStrategy.string(_.deduplication_key)
       )
 
+    val stringInterningTable: Table[DbDto.StringInterningDto] =
+      fieldStrategy.insert("string_interning")(
+        "internal_id" -> fieldStrategy.int(_.internalId),
+        "external_string" -> fieldStrategy.string(_.externalString),
+      )
+
     val executes: Seq[Array[Array[_]] => Connection => Unit] = List(
       eventsDivulgence.executeUpdate,
       eventsCreate.executeUpdate,
@@ -238,6 +247,7 @@ private[backend] object AppendOnlySchema {
       partyEntries.executeUpdate,
       commandCompletions.executeUpdate,
       commandSubmissionDeletes.executeUpdate,
+      stringInterningTable.executeUpdate,
     )
 
     new Schema[DbDto] {
@@ -257,6 +267,7 @@ private[backend] object AppendOnlySchema {
           partyEntries.prepareData(collect[PartyEntry]),
           commandCompletions.prepareData(collect[CommandCompletion]),
           commandSubmissionDeletes.prepareData(collect[CommandDeduplication]),
+          stringInterningTable.prepareData(collect[StringInterningDto]),
         )
       }
 

ledger/participant-integration-api/src/main/scala/platform/store/backend/common/StringInterningStorageBackendTemplate.scala

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.platform.store.backend.common
+
+import java.sql.Connection
+
+import anorm.SqlParser.{int, str}
+import anorm.{RowParser, SqlStringInterpolation, ~}
+import com.daml.platform.store.backend.StringInterningStorageBackend
+import com.daml.platform.store.SimpleSqlAsVectorOf.SimpleSqlAsVectorOf
+
+object StringInterningStorageBackendTemplate extends StringInterningStorageBackend {
+
+  private val StringInterningEntriesParser: RowParser[(Int, String)] =
+    int("internal_id") ~ str("external_string") map { case internalId ~ externalString =>
+      (internalId, externalString)
+    }
+
+  override def loadStringInterningEntries(fromIdExclusive: Int, untilIdInclusive: Int)(
+      connection: Connection
+  ): Iterable[(Int, String)] =
+    SQL"""
+         SELECT internal_id, external_string
+         FROM string_interning
+         WHERE
+           internal_id > $fromIdExclusive
+           AND internal_id <= $untilIdInclusive
+       """.asVectorOf(StringInterningEntriesParser)(connection)
+}

ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2ResetStorageBackend.scala

@@ -22,6 +22,7 @@ object H2ResetStorageBackend extends ResetStorageBackend {
           |truncate table participant_events_consuming_exercise;
           |truncate table participant_events_non_consuming_exercise;
           |truncate table party_entries;
+          |truncate table string_interning;
           |set referential_integrity true;""".stripMargin)
       .execute()(connection)
     ()
@@ -40,6 +41,7 @@ object H2ResetStorageBackend extends ResetStorageBackend {
           |truncate table participant_events_consuming_exercise;
           |truncate table participant_events_non_consuming_exercise;
           |truncate table party_entries;
+          |truncate table string_interning;
           |set referential_integrity true;""".stripMargin)
       .execute()(connection)
     ()

ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackendFactory.scala

@@ -11,6 +11,7 @@ import com.daml.platform.store.backend.common.{
   PackageStorageBackendTemplate,
   ParameterStorageBackendTemplate,
   PartyStorageBackendTemplate,
+  StringInterningStorageBackendTemplate,
 }
 import com.daml.platform.store.backend.{
   CompletionStorageBackend,
@@ -27,6 +28,7 @@ import com.daml.platform.store.backend.{
   PartyStorageBackend,
   ResetStorageBackend,
   StorageBackendFactory,
+  StringInterningStorageBackend,
 }
 
 object H2StorageBackendFactory extends StorageBackendFactory {
@@ -68,4 +70,7 @@ object H2StorageBackendFactory extends StorageBackendFactory {
 
   override val createResetStorageBackend: ResetStorageBackend =
     H2ResetStorageBackend
+
+  override val createStringInterningStorageBackend: StringInterningStorageBackend =
+    StringInterningStorageBackendTemplate
 }

ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleResetStorageBackend.scala

@@ -21,6 +21,7 @@ object OracleResetStorageBackend extends ResetStorageBackend {
       "truncate table participant_events_consuming_exercise cascade",
       "truncate table participant_events_non_consuming_exercise cascade",
       "truncate table party_entries cascade",
+      "truncate table string_interning cascade",
     ).map(SQL(_)).foreach(_.execute()(connection))
 
   override def resetAll(connection: Connection): Unit =
@@ -36,5 +37,6 @@ object OracleResetStorageBackend extends ResetStorageBackend {
       "truncate table participant_events_consuming_exercise cascade",
       "truncate table participant_events_non_consuming_exercise cascade",
       "truncate table party_entries cascade",
+      "truncate table string_interning cascade",
     ).map(SQL(_)).foreach(_.execute()(connection))
 }

ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackendFactory.scala

@@ -12,6 +12,7 @@ import com.daml.platform.store.backend.common.{
   PackageStorageBackendTemplate,
   ParameterStorageBackendTemplate,
   PartyStorageBackendTemplate,
+  StringInterningStorageBackendTemplate,
 }
 import com.daml.platform.store.backend.{
   CompletionStorageBackend,
@@ -28,6 +29,7 @@ import com.daml.platform.store.backend.{
   PartyStorageBackend,
   ResetStorageBackend,
   StorageBackendFactory,
+  StringInterningStorageBackend,
 }
 
 object OracleStorageBackendFactory extends StorageBackendFactory {
@@ -69,4 +71,7 @@ object OracleStorageBackendFactory extends StorageBackendFactory {
 
   override val createResetStorageBackend: ResetStorageBackend =
     OracleResetStorageBackend
+
+  override val createStringInterningStorageBackend: StringInterningStorageBackend =
+    StringInterningStorageBackendTemplate
 }

ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresResetStorageBackend.scala

@@ -20,6 +20,7 @@ object PostgresResetStorageBackend extends ResetStorageBackend {
           |truncate table participant_events_consuming_exercise cascade;
           |truncate table participant_events_non_consuming_exercise cascade;
           |truncate table party_entries cascade;
+          |truncate table string_interning cascade;
           |""".stripMargin)
       .execute()(connection)
     ()
@@ -37,6 +38,7 @@ object PostgresResetStorageBackend extends ResetStorageBackend {
           |truncate table participant_events_consuming_exercise cascade;
           |truncate table participant_events_non_consuming_exercise cascade;
           |truncate table party_entries cascade;
+          |truncate table string_interning cascade;
           |""".stripMargin)
       .execute()(connection)
     ()

ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackendFactory.scala

@@ -12,6 +12,7 @@ import com.daml.platform.store.backend.common.{
   PackageStorageBackendTemplate,
   ParameterStorageBackendTemplate,
   PartyStorageBackendTemplate,
+  StringInterningStorageBackendTemplate,
 }
 import com.daml.platform.store.backend.{
   CompletionStorageBackend,
@@ -28,6 +29,7 @@ import com.daml.platform.store.backend.{
   PartyStorageBackend,
   ResetStorageBackend,
   StorageBackendFactory,
+  StringInterningStorageBackend,
 }
 
 object PostgresStorageBackendFactory extends StorageBackendFactory {
@@ -69,4 +71,7 @@ object PostgresStorageBackendFactory extends StorageBackendFactory {
 
   override val createResetStorageBackend: ResetStorageBackend =
     PostgresResetStorageBackend
+
+  override val createStringInterningStorageBackend: StringInterningStorageBackend =
+    StringInterningStorageBackendTemplate
 }

ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendSuite.scala

@@ -15,6 +15,7 @@ trait StorageBackendSuite
     with StorageBackendTestsDBLockForSuite
     with StorageBackendTestsIntegrity
     with StorageBackendTestsDeduplication
-    with StorageBackendTestsTimestamps {
+    with StorageBackendTestsTimestamps
+    with StorageBackendTestsStringInterning {
   this: AsyncFlatSpec =>
 }

ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsReset.scala

@@ -20,6 +20,8 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
     backendFactory.createPackageStorageBackend
   private val contractStorageBackend: ContractStorageBackend =
     backendFactory.createContractStorageBackend
+  private val stringInterningStorageBackend: StringInterningStorageBackend =
+    backendFactory.createStringInterningStorageBackend
   private val resetStorageBackend: ResetStorageBackend = backendFactory.createResetStorageBackend
 
   behavior of "StorageBackend (reset)"
@@ -34,13 +36,17 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
       config <- executeSql(configurationStorageBackend.ledgerConfiguration)
       packages <- executeSql(packageStorageBackend.lfPackages)
       events <- executeSql(contractStorageBackend.contractStateEvents(0, Long.MaxValue))
+      stringInterningEntries <- executeSql(
+        stringInterningStorageBackend.loadStringInterningEntries(0, 1000)
+      )
     } yield {
       identity shouldBe None
       end shouldBe None
       parties shouldBe empty
       packages shouldBe empty
       events shouldBe empty
       config shouldBe None
+      stringInterningEntries shouldBe empty
     }
   }
 
@@ -73,6 +79,7 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
       dtoExercise(offset(5), 2L, true, "#4"),
       dtoDivulgence(Some(offset(5)), 3L, "#4"),
       dtoCompletion(offset(5)),
+      DbDto.StringInterningDto(2, "2"),
     )
 
     for {
@@ -94,13 +101,17 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
       parties <- executeSql(partyStorageBackend.knownParties)
       config <- executeSql(configurationStorageBackend.ledgerConfiguration)
       packages <- executeSql(packageStorageBackend.lfPackages)
+      stringInterningEntries <- executeSql(
+        stringInterningStorageBackend.loadStringInterningEntries(0, 1000)
+      )
     } yield {
       identity shouldBe None
       end shouldBe None
       parties shouldBe empty
       packages should not be empty // Note: reset() does not delete packages
       events shouldBe empty
       config shouldBe None
+      stringInterningEntries shouldBe empty
     }
   }
 
@@ -141,13 +152,17 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
       parties <- executeSql(partyStorageBackend.knownParties)
       config <- executeSql(configurationStorageBackend.ledgerConfiguration)
       packages <- executeSql(packageStorageBackend.lfPackages)
+      stringInterningEntries <- executeSql(
+        stringInterningStorageBackend.loadStringInterningEntries(0, 1000)
+      )
     } yield {
       identity shouldBe None
       end shouldBe None
       parties shouldBe empty
       packages shouldBe empty // Note: resetAll() does delete packages
       events shouldBe empty
       config shouldBe None
+      stringInterningEntries shouldBe empty
     }
   }