Add string-interning data access [DPP-707]

* Add new table, and party_id to party_entries
* Add write side basics (DbDto, Schema)
* Add read side basic (StorageBackendStringInterning)
* Add StorageBackend unit test
* Support for all DB backends

changelog_begin
changelog_end

ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256

@@ -1 +1 @@
-86ded6839752d76bb3e3e6cde7f4ec103dea6cee6382b6695de95e84dd74d2da
+5e5e023d57d3b0b5a4ab83d9e370597fe98a1b7b4fbb114b94326d7779a8ffe7

ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql

@@ -81,6 +81,7 @@ CREATE TABLE party_entries (
     typ VARCHAR NOT NULL,
     rejection_reason VARCHAR,
     is_local BOOLEAN,
+    party_id integer NOT NULL DEFAULT 0,
 
     CONSTRAINT check_party_entry_type
         CHECK (
@@ -91,6 +92,7 @@ CREATE TABLE party_entries (
 
 CREATE INDEX idx_party_entries ON party_entries (submission_id);
 CREATE INDEX idx_party_entries_party_and_ledger_offset ON party_entries(party, ledger_offset);
+CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset);
 
 ---------------------------------------------------------------------------------------------------
 -- Submissions table
@@ -539,3 +541,8 @@ SELECT
     exercise_result_compression
 FROM participant_events_non_consuming_exercise
 ;
+
+CREATE TABLE string_interning (
+    id integer PRIMARY KEY NOT NULL,
+    s text
+);

ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V3__Add_string_interning.sha256

@@ -0,0 +1 @@
+226c3fbed157c8804d004f61996d855dd34132987a70b387cd5d38f070586f63

ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V3__Add_string_interning.sql

@@ -0,0 +1,12 @@
+--  Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+--  SPDX-License-Identifier: Apache-2.0
+
+CREATE TABLE string_interning (
+    id NUMBER          PRIMARY KEY NOT NULL,
+    s  VARCHAR2(4000)
+);
+
+ALTER TABLE party_entries
+  ADD party_id NUMBER DEFAULT 0 NOT NULL; -- needed for efficient pruning, will be filled later
+
+CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset);

ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V112__add_string_interning.sha256

@@ -0,0 +1 @@
+44d1b18342edd67f6c9be31ec8b1dffd37ac37cce7bb5a6bcf5f822aae6df51a

ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V112__add_string_interning.sql

@@ -0,0 +1,12 @@
+--  Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+--  SPDX-License-Identifier: Apache-2.0
+
+CREATE TABLE string_interning (
+    id integer PRIMARY KEY NOT NULL,
+    s text
+);
+
+ALTER TABLE party_entries
+  ADD column party_id integer NOT NULL DEFAULT 0; -- needed for efficient pruning, will be filled later
+
+CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset);

ledger/participant-integration-api/src/main/scala/platform/store/backend/DbDto.scala

@@ -135,4 +135,5 @@ object DbDto {
 
   final case class CommandDeduplication(deduplication_key: String) extends DbDto
 
+  final case class StringInterningDto(id: Int, s: String) extends DbDto
 }

ledger/participant-integration-api/src/main/scala/platform/store/backend/StorageBackend.scala

@@ -48,7 +48,8 @@ trait StorageBackend[DB_BATCH]
     with EventStorageBackend
     with DataSourceStorageBackend
     with DBLockStorageBackend
-    with IntegrityStorageBackend {
+    with IntegrityStorageBackend
+    with StringInterningStorageBackend {
 
   /** Truncates all storage backend tables, EXCEPT the packages table.
     * Does not touch other tables, like the Flyway history table.
@@ -351,6 +352,12 @@ trait IntegrityStorageBackend {
   def verifyIntegrity()(connection: Connection): Unit
 }
 
+trait StringInterningStorageBackend {
+  def loadStringInterningEntries(fromIdExclusive: Int, untilIdInclusive: Int)(
+      connection: Connection
+  ): Iterable[(Int, String)]
+}
+
 object StorageBackend {
   case class RawContractState(
       templateId: Option[String],

ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Field.scala

@@ -65,9 +65,11 @@ private[backend] case class Bytea[FROM](extract: FROM => Array[Byte])
 private[backend] case class ByteaOptional[FROM](extract: FROM => Option[Array[Byte]])
     extends TrivialOptionalField[FROM, Array[Byte]]
 
+private[backend] case class Integer[FROM](extract: FROM => Int) extends TrivialField[FROM, Int]
+
 private[backend] case class IntOptional[FROM](extract: FROM => Option[Int])
     extends Field[FROM, Option[Int], java.lang.Integer] {
-  override def convert: Option[Int] => Integer = _.map(Int.box).orNull
+  override def convert: Option[Int] => java.lang.Integer = _.map(Int.box).orNull
 }
 
 private[backend] case class Bigint[FROM](extract: FROM => Long) extends TrivialField[FROM, Long]
@@ -79,7 +81,7 @@ private[backend] case class BigintOptional[FROM](extract: FROM => Option[Long])
 
 private[backend] case class SmallintOptional[FROM](extract: FROM => Option[Int])
     extends Field[FROM, Option[Int], java.lang.Integer] {
-  override def convert: Option[Int] => Integer = _.map(Int.box).orNull
+  override def convert: Option[Int] => java.lang.Integer = _.map(Int.box).orNull
 }
 
 private[backend] case class BooleanField[FROM](extract: FROM => Boolean)

ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala

@@ -52,6 +52,9 @@ private[backend] object AppendOnlySchema {
     def smallintOptional[FROM, _](extractor: FROM => Option[Int]): Field[FROM, Option[Int], _] =
       SmallintOptional(extractor)
 
+    def int[FROM, _](extractor: FROM => Int): Field[FROM, Int, _] =
+      Integer(extractor)
+
     def intOptional[FROM, _](extractor: FROM => Option[Int]): Field[FROM, Option[Int], _] =
       IntOptional(extractor)
 
@@ -227,6 +230,12 @@ private[backend] object AppendOnlySchema {
         "deduplication_key" -> fieldStrategy.string(_.deduplication_key)
       )
 
+    val stringInterningTable: Table[DbDto.StringInterningDto] =
+      fieldStrategy.insert("string_interning")(
+        "id" -> fieldStrategy.int(_.id),
+        "s" -> fieldStrategy.string(_.s),
+      )
+
     val executes: Seq[Array[Array[_]] => Connection => Unit] = List(
       eventsDivulgence.executeUpdate,
       eventsCreate.executeUpdate,
@@ -238,6 +247,7 @@ private[backend] object AppendOnlySchema {
       partyEntries.executeUpdate,
       commandCompletions.executeUpdate,
       commandSubmissionDeletes.executeUpdate,
+      stringInterningTable.executeUpdate,
     )
 
     new Schema[DbDto] {
@@ -257,6 +267,7 @@ private[backend] object AppendOnlySchema {
           partyEntries.prepareData(collect[PartyEntry]),
           commandCompletions.prepareData(collect[CommandCompletion]),
           commandSubmissionDeletes.prepareData(collect[CommandDeduplication]),
+          stringInterningTable.prepareData(collect[StringInterningDto]),
         )
       }
 

ledger/participant-integration-api/src/main/scala/platform/store/backend/common/StringInterningStorageBackendTemplate.scala

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.platform.store.backend.common
+
+import java.sql.Connection
+
+import anorm.SqlParser.{int, str}
+import anorm.{RowParser, SqlStringInterpolation, ~}
+import com.daml.platform.store.backend.StringInterningStorageBackend
+import com.daml.platform.store.SimpleSqlAsVectorOf.SimpleSqlAsVectorOf
+
+trait StringInterningStorageBackendTemplate extends StringInterningStorageBackend {
+
+  private val StringInterningEntriesParser: RowParser[(Int, String)] =
+    int("id") ~ str("s") map { case id ~ s =>
+      (id, s)
+    }
+
+  override def loadStringInterningEntries(fromIdExclusive: Int, untilIdInclusive: Int)(
+      connection: Connection
+  ): Iterable[(Int, String)] =
+    SQL"""
+         SELECT id, s
+         FROM string_interning
+         WHERE
+           id > $fromIdExclusive
+           AND id <= $untilIdInclusive
+       """.asVectorOf(StringInterningEntriesParser)(connection)
+}

ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackend.scala

@@ -4,6 +4,7 @@
 package com.daml.platform.store.backend.h2
 
 import java.sql.Connection
+
 import anorm.{Row, SQL, SimpleSql}
 import anorm.SqlParser.get
 import com.daml.ledger.offset.Offset
@@ -29,6 +30,7 @@ import com.daml.platform.store.backend.common.{
   ParameterStorageBackendTemplate,
   PartyStorageBackendTemplate,
   QueryStrategy,
+  StringInterningStorageBackendTemplate,
 }
 import com.daml.platform.store.backend.{
   DBLockStorageBackend,
@@ -37,8 +39,8 @@ import com.daml.platform.store.backend.{
   StorageBackend,
   common,
 }
-
 import javax.sql.DataSource
+
 import scala.util.control.NonFatal
 
 private[backend] object H2StorageBackend
@@ -53,7 +55,8 @@ private[backend] object H2StorageBackend
     with ContractStorageBackendTemplate
     with CompletionStorageBackendTemplate
     with PartyStorageBackendTemplate
-    with IntegrityStorageBackendTemplate {
+    with IntegrityStorageBackendTemplate
+    with StringInterningStorageBackendTemplate {
 
   private val logger = ContextualizedLogger.get(this.getClass)
 
@@ -69,6 +72,7 @@ private[backend] object H2StorageBackend
         |truncate table participant_events_consuming_exercise;
         |truncate table participant_events_non_consuming_exercise;
         |truncate table party_entries;
+        |truncate table string_interning;
         |set referential_integrity true;""".stripMargin)
       .execute()(connection)
     ()
@@ -87,6 +91,7 @@ private[backend] object H2StorageBackend
           |truncate table participant_events_consuming_exercise;
           |truncate table participant_events_non_consuming_exercise;
           |truncate table party_entries;
+          |truncate table string_interning;
           |set referential_integrity true;""".stripMargin)
       .execute()(connection)
     ()

ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackend.scala

@@ -22,6 +22,7 @@ import com.daml.platform.store.backend.common.{
   ParameterStorageBackendTemplate,
   PartyStorageBackendTemplate,
   QueryStrategy,
+  StringInterningStorageBackendTemplate,
 }
 import com.daml.platform.store.backend.{
   DBLockStorageBackend,
@@ -30,15 +31,15 @@ import com.daml.platform.store.backend.{
   StorageBackend,
   common,
 }
-
 import java.sql.Connection
+
 import com.daml.ledger.offset.Offset
 import com.daml.lf.data.Time.Timestamp
 import com.daml.platform.store.backend.EventStorageBackend.FilterParams
 import com.daml.logging.{ContextualizedLogger, LoggingContext}
 import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation}
-
 import javax.sql.DataSource
+
 import scala.util.control.NonFatal
 
 private[backend] object OracleStorageBackend
@@ -53,7 +54,8 @@ private[backend] object OracleStorageBackend
     with ContractStorageBackendTemplate
     with CompletionStorageBackendTemplate
     with PartyStorageBackendTemplate
-    with IntegrityStorageBackendTemplate {
+    with IntegrityStorageBackendTemplate
+    with StringInterningStorageBackendTemplate {
 
   private val logger = ContextualizedLogger.get(this.getClass)
 
@@ -69,6 +71,7 @@ private[backend] object OracleStorageBackend
       "truncate table participant_events_consuming_exercise cascade",
       "truncate table participant_events_non_consuming_exercise cascade",
       "truncate table party_entries cascade",
+      "truncate table string_interning cascade",
     ).map(SQL(_)).foreach(_.execute()(connection))
 
   override def resetAll(connection: Connection): Unit =
@@ -84,6 +87,7 @@ private[backend] object OracleStorageBackend
       "truncate table participant_events_consuming_exercise cascade",
       "truncate table participant_events_non_consuming_exercise cascade",
       "truncate table party_entries cascade",
+      "truncate table string_interning cascade",
     ).map(SQL(_)).foreach(_.execute()(connection))
 
   val SQL_INSERT_COMMAND: String =

ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackend.scala

@@ -4,6 +4,7 @@
 package com.daml.platform.store.backend.postgresql
 
 import java.sql.Connection
+
 import anorm.SQL
 import anorm.SqlParser.{get, int}
 import com.daml.error.NoLogging
@@ -31,6 +32,7 @@ import com.daml.platform.store.backend.common.{
   ParameterStorageBackendTemplate,
   PartyStorageBackendTemplate,
   QueryStrategy,
+  StringInterningStorageBackendTemplate,
 }
 import com.daml.platform.store.backend.{
   DBLockStorageBackend,
@@ -39,7 +41,6 @@ import com.daml.platform.store.backend.{
   StorageBackend,
   common,
 }
-
 import javax.sql.DataSource
 import org.postgresql.ds.PGSimpleDataSource
 
@@ -55,7 +56,8 @@ private[backend] object PostgresStorageBackend
     with ContractStorageBackendTemplate
     with CompletionStorageBackendTemplate
     with PartyStorageBackendTemplate
-    with IntegrityStorageBackendTemplate {
+    with IntegrityStorageBackendTemplate
+    with StringInterningStorageBackendTemplate {
 
   private val logger: ContextualizedLogger = ContextualizedLogger.get(this.getClass)
 
@@ -100,6 +102,7 @@ private[backend] object PostgresStorageBackend
       |truncate table participant_events_consuming_exercise cascade;
       |truncate table participant_events_non_consuming_exercise cascade;
       |truncate table party_entries cascade;
+      |truncate table string_interning;
       |""".stripMargin)
       .execute()(connection)
     ()
@@ -117,6 +120,7 @@ private[backend] object PostgresStorageBackend
           |truncate table participant_events_consuming_exercise cascade;
           |truncate table participant_events_non_consuming_exercise cascade;
           |truncate table party_entries cascade;
+          |truncate table string_interning;
           |""".stripMargin)
       .execute()(connection)
     ()

ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendSuite.scala

@@ -15,6 +15,7 @@ trait StorageBackendSuite
     with StorageBackendTestsDBLockForSuite
     with StorageBackendTestsIntegrity
     with StorageBackendTestsDeduplication
-    with StorageBackendTestsTimestamps {
+    with StorageBackendTestsTimestamps
+    with StorageBackendTestsStringInterning {
   this: AsyncFlatSpec =>
 }

ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsStringInterning.scala

@@ -0,0 +1,47 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.platform.store.backend
+
+import org.scalatest.Inside
+import org.scalatest.flatspec.AsyncFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+private[backend] trait StorageBackendTestsStringInterning
+    extends Matchers
+    with Inside
+    with StorageBackendSpec {
+  this: AsyncFlatSpec =>
+
+  behavior of "StorageBackend (StringInterning)"
+
+  it should "store and load string-interning entries" in {
+    val dtos = Vector(
+      DbDto.StringInterningDto(2, "a"),
+      DbDto.StringInterningDto(3, "b"),
+      DbDto.StringInterningDto(4, "c"),
+      DbDto.StringInterningDto(5, "d"),
+    )
+
+    for {
+      interningIdsBeforeBegin <- executeSql(backend.loadStringInterningEntries(0, 5))
+      _ <- executeSql(ingest(dtos, _))
+      interningIdsFull <- executeSql(backend.loadStringInterningEntries(0, 5))
+      interningIdsOverFetch <- executeSql(backend.loadStringInterningEntries(0, 10))
+      interningIdsEmpty <- executeSql(backend.loadStringInterningEntries(5, 10))
+      interningIdsSubset <- executeSql(backend.loadStringInterningEntries(3, 10))
+    } yield {
+      val expectedFullList = List(
+        2 -> "a",
+        3 -> "b",
+        4 -> "c",
+        5 -> "d",
+      )
+      interningIdsBeforeBegin shouldBe Nil
+      interningIdsFull shouldBe expectedFullList
+      interningIdsOverFetch shouldBe expectedFullList
+      interningIdsEmpty shouldBe Nil
+      interningIdsSubset shouldBe expectedFullList.drop(2)
+    }
+  }
+}