From 72f3dd409086ea85f9a33f22bfc84eff600e3a5a Mon Sep 17 00:00:00 2001 From: Marton Nagy Date: Fri, 12 Nov 2021 20:01:02 +0100 Subject: [PATCH] Activate party interning [DPP-712] (#11663) * Add DB schema changes * Data migration scripts (PostgreSQL only) * Add StringInterning to Query strategies * Adapt queries, result-parsers * Adapt ingestion * Fix JdbcLedgerDaoBackend StringInterning handling changelog_begin changelog_end --- .../V1__Append_only_schema.sha256 | 2 +- .../V1__Append_only_schema.sql | 2 + .../V5__activate_party_interning.sha256 | 1 + .../V5__activate_party_interning.sql | 7 + ...tivate_string_interning_for_parties.sha256 | 1 + ..._activate_string_interning_for_parties.sql | 317 ++++++++++++++++++ .../scala/platform/store/Conversions.scala | 64 +++- .../CompletionStorageBackendTemplate.scala | 4 +- .../ContractStorageBackendTemplate.scala | 47 ++- .../common/EventStorageBackendTemplate.scala | 92 +++-- .../store/backend/common/QueryStrategy.scala | 2 + .../store/backend/common/Schema.scala | 51 ++- .../store/backend/h2/H2EventStrategy.scala | 14 +- .../platform/store/backend/h2/H2Field.scala | 19 ++ .../store/backend/h2/H2FunctionAliases.scala | 5 +- .../store/backend/h2/H2QueryStrategy.scala | 15 +- .../platform/store/backend/h2/H2Schema.scala | 11 + .../backend/oracle/OracleEventStrategy.scala | 67 +++- .../store/backend/oracle/OracleField.scala | 27 ++ .../backend/oracle/OracleQueryStrategy.scala | 14 +- .../store/backend/oracle/OracleSchema.scala | 10 + .../store/backend/postgresql/PGField.scala | 20 ++ .../store/backend/postgresql/PGSchema.scala | 10 + .../postgresql/PostgresEventStrategy.scala | 61 +++- .../postgresql/PostgresQueryStrategy.scala | 9 +- .../store/dao/JdbcLedgerDaoBackend.scala | 13 +- 26 files changed, 753 insertions(+), 132 deletions(-) create mode 100644 ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sha256 create mode 100644 ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sql create mode 100644 ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sha256 create mode 100644 ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sql create mode 100644 ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Field.scala diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256 b/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256 index 2308b70c55c3..cc12df2e3fc1 100644 --- a/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256 +++ b/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256 @@ -1 +1 @@ -bd9ddd16ae4ecc09923215635442c83b7894e95d23203baccde6df27cb0ce2cf +b59f4464e996e785861401df83d8b903436f3e0287fb62b884711feedae14ef7 diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql b/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql index ff8d2a22edcd..e3d43ecf6584 100644 --- a/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql +++ b/ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql @@ -82,6 +82,7 @@ CREATE TABLE party_entries ( typ VARCHAR NOT NULL, rejection_reason VARCHAR, is_local BOOLEAN, + party_id INTEGER, CONSTRAINT check_party_entry_type CHECK ( @@ -92,6 +93,7 @@ CREATE TABLE party_entries ( CREATE INDEX idx_party_entries ON party_entries (submission_id); CREATE INDEX idx_party_entries_party_and_ledger_offset ON party_entries(party, ledger_offset); +CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset); --------------------------------------------------------------------------------------------------- -- Submissions table diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sha256 b/ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sha256 new file mode 100644 index 000000000000..7d99dcb11d6f --- /dev/null +++ b/ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sha256 @@ -0,0 +1 @@ +d438a8924cfd01c9c28aae1a1bc33c3f202433748b3d3486305ced37128d7039 diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sql b/ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sql new file mode 100644 index 000000000000..384545ba1483 --- /dev/null +++ b/ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sql @@ -0,0 +1,7 @@ +-- Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +-- SPDX-License-Identifier: Apache-2.0 + +-- add party_id to party_entries +ALTER TABLE party_entries + ADD party_id NUMBER; +CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset); diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sha256 b/ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sha256 new file mode 100644 index 000000000000..fe7cc20f650f --- /dev/null +++ b/ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sha256 @@ -0,0 +1 @@ +bdb25f5fee117cb98afaca3393ac333cbd28e142814604dab24e02174adfc198 diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sql b/ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sql new file mode 100644 index 000000000000..2abf490511f4 --- /dev/null +++ b/ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sql @@ -0,0 +1,317 @@ +-- Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +-- SPDX-License-Identifier: Apache-2.0 + +SELECT 'Party interning data migration - Preparation...'; + +-- add temporary index to aid string lookups as migrating data +CREATE UNIQUE INDEX string_interning_external_string_temp_idx ON string_interning USING btree (external_string); + +-- drop participant_events view (enables column type changes) +DROP VIEW participant_events; + +-- add temporary migration function +CREATE FUNCTION internalize_string_array(prefix TEXT, string_array TEXT[]) +RETURNS INTEGER[] +LANGUAGE plpgsql +AS +$$ +DECLARE + result INTEGER[]; +BEGIN + EXECUTE + 'SELECT CASE' || + ' WHEN $2 IS NULL THEN NULL' || + ' WHEN cardinality($2) = 0 THEN ARRAY[]::INTEGER[]' || + ' ELSE (' || + ' SELECT array_agg(string_interning.internal_id)' || + ' FROM' || + ' unnest($2) AS original(string_elem),' || + ' string_interning' || + ' WHERE $1 || original.string_elem = string_interning.external_string' || + ' )' || + 'END' + INTO result + USING prefix, string_array; + ASSERT + (string_array IS NULL AND result IS NULL) OR cardinality(string_array) = cardinality(result), + 'Could not internalize some elements in the array: [' || array_to_string(string_array, ', ') || ']'; + RETURN result; +END +$$; + + +SELECT 'Party interning data migration - Migrating party_entries...'; + +-- add party_id to party_entries +ALTER TABLE party_entries + ADD COLUMN party_id INTEGER; + +-- populate internal party id-s +UPDATE party_entries + SET party_id = string_interning.internal_id +FROM string_interning +WHERE 'p|' || string_interning.external_string = party_entries.party; + +-- add index on party_id +CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset); + + + +SELECT 'Party interning data migration - Migrating participant_command_completions...'; + +ALTER TABLE participant_command_completions + ALTER COLUMN submitters TYPE INTEGER[] + USING internalize_string_array('p|', submitters); + + + +SELECT 'Party interning data migration - Migrating participant_events_divulgence...'; + +DROP INDEX participant_events_divulgence_tree_event_witnesses_idx; + +ALTER TABLE participant_events_divulgence + ALTER COLUMN submitters TYPE INTEGER[] + USING internalize_string_array('p|', submitters); + +ALTER TABLE participant_events_divulgence + ALTER COLUMN tree_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_divulgence + ALTER COLUMN tree_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', tree_event_witnesses); + + + +SELECT 'Party interning data migration - Migrating participant_events_create...'; + +DROP INDEX participant_events_create_flat_event_witnesses_idx; +DROP INDEX participant_events_create_tree_event_witnesses_idx; + +ALTER TABLE participant_events_create + ALTER COLUMN submitters TYPE INTEGER[] + USING internalize_string_array('p|', submitters); + +ALTER TABLE participant_events_create + ALTER COLUMN tree_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_create + ALTER COLUMN tree_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', tree_event_witnesses); + +ALTER TABLE participant_events_create + ALTER COLUMN flat_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_create + ALTER COLUMN flat_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', flat_event_witnesses); + +ALTER TABLE participant_events_create + ALTER COLUMN create_signatories TYPE INTEGER[] + USING internalize_string_array('p|', create_signatories); + +ALTER TABLE participant_events_create + ALTER COLUMN create_observers TYPE INTEGER[] + USING internalize_string_array('p|', create_observers); + + + +SELECT 'Party interning data migration - Migrating participant_events_consuming_exercise...'; + +DROP INDEX participant_events_consuming_exercise_flat_event_witnesses_idx; +DROP INDEX participant_events_consuming_exercise_tree_event_witnesses_idx; + +ALTER TABLE participant_events_consuming_exercise + ALTER COLUMN submitters TYPE INTEGER[] + USING internalize_string_array('p|', submitters); + +ALTER TABLE participant_events_consuming_exercise + ALTER COLUMN tree_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_consuming_exercise + ALTER COLUMN tree_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', tree_event_witnesses); + +ALTER TABLE participant_events_consuming_exercise + ALTER COLUMN flat_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_consuming_exercise + ALTER COLUMN flat_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', flat_event_witnesses); + +ALTER TABLE participant_events_consuming_exercise + ALTER COLUMN exercise_actors TYPE INTEGER[] + USING internalize_string_array('p|', exercise_actors); + + + +SELECT 'Party interning data migration - Migrating participant_events_non_consuming_exercise...'; + +DROP INDEX participant_events_non_consuming_exercise_flat_event_witnes_idx; +DROP INDEX participant_events_non_consuming_exercise_tree_event_witnes_idx; + +ALTER TABLE participant_events_non_consuming_exercise + ALTER COLUMN submitters TYPE INTEGER[] + USING internalize_string_array('p|', submitters); + +ALTER TABLE participant_events_non_consuming_exercise + ALTER COLUMN tree_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_non_consuming_exercise + ALTER COLUMN tree_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', tree_event_witnesses); + +ALTER TABLE participant_events_non_consuming_exercise + ALTER COLUMN flat_event_witnesses DROP DEFAULT; + +ALTER TABLE participant_events_non_consuming_exercise + ALTER COLUMN flat_event_witnesses TYPE INTEGER[] + USING internalize_string_array('p|', flat_event_witnesses); + +ALTER TABLE participant_events_non_consuming_exercise + ALTER COLUMN exercise_actors TYPE INTEGER[] + USING internalize_string_array('p|', exercise_actors); + + + +SELECT 'Party interning data migration - Cleanup...'; + +DROP INDEX string_interning_external_string_temp_idx; +DROP FUNCTION internalize_string_array(TEXT, TEXT[]); + +CREATE VIEW participant_events +AS +SELECT + 0::smallint as event_kind, + event_sequential_id, + NULL::text as event_offset, + NULL::text as transaction_id, + NULL::bigint as ledger_effective_time, + command_id, + workflow_id, + application_id, + submitters, + NULL::integer as node_index, + NULL::text as event_id, + contract_id, + template_id, + NULL::INTEGER[] as flat_event_witnesses, + tree_event_witnesses, + create_argument, + NULL::INTEGER[] as create_signatories, + NULL::INTEGER[] as create_observers, + NULL::text as create_agreement_text, + NULL::bytea as create_key_value, + NULL::text as create_key_hash, + NULL::text as exercise_choice, + NULL::bytea as exercise_argument, + NULL::bytea as exercise_result, + NULL::INTEGER[] as exercise_actors, + NULL::text[] as exercise_child_event_ids, + create_argument_compression, + NULL::smallint as create_key_value_compression, + NULL::smallint as exercise_argument_compression, + NULL::smallint as exercise_result_compression +FROM participant_events_divulgence +UNION ALL +SELECT + 10::smallint as event_kind, + event_sequential_id, + event_offset, + transaction_id, + ledger_effective_time, + command_id, + workflow_id, + application_id, + submitters, + node_index, + event_id, + contract_id, + template_id, + flat_event_witnesses, + tree_event_witnesses, + create_argument, + create_signatories, + create_observers, + create_agreement_text, + create_key_value, + create_key_hash, + NULL::text as exercise_choice, + NULL::bytea as exercise_argument, + NULL::bytea as exercise_result, + NULL::INTEGER[] as exercise_actors, + NULL::text[] as exercise_child_event_ids, + create_argument_compression, + create_key_value_compression, + NULL::smallint as exercise_argument_compression, + NULL::smallint as exercise_result_compression +FROM participant_events_create +UNION ALL +SELECT + 20::smallint as event_kind, + event_sequential_id, + event_offset, + transaction_id, + ledger_effective_time, + command_id, + workflow_id, + application_id, + submitters, + node_index, + event_id, + contract_id, + template_id, + flat_event_witnesses, + tree_event_witnesses, + NULL::bytea as create_argument, + NULL::INTEGER[] as create_signatories, + NULL::INTEGER[] as create_observers, + NULL::text as create_agreement_text, + create_key_value, + NULL::text as create_key_hash, + exercise_choice, + exercise_argument, + exercise_result, + exercise_actors, + exercise_child_event_ids, + NULL::smallint as create_argument_compression, + create_key_value_compression, + exercise_argument_compression, + exercise_result_compression +FROM participant_events_consuming_exercise +UNION ALL +SELECT + 25::smallint as event_kind, + event_sequential_id, + event_offset, + transaction_id, + ledger_effective_time, + command_id, + workflow_id, + application_id, + submitters, + node_index, + event_id, + contract_id, + template_id, + flat_event_witnesses, + tree_event_witnesses, + NULL::bytea as create_argument, + NULL::INTEGER[] as create_signatories, + NULL::INTEGER[] as create_observers, + NULL::text as create_agreement_text, + create_key_value, + NULL::text as create_key_hash, + exercise_choice, + exercise_argument, + exercise_result, + exercise_actors, + exercise_child_event_ids, + NULL::smallint as create_argument_compression, + create_key_value_compression, + exercise_argument_compression, + exercise_result_compression +FROM participant_events_non_consuming_exercise; + + + +SELECT 'Party interning data migration - Done'; diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/Conversions.scala b/ledger/participant-integration-api/src/main/scala/platform/store/Conversions.scala index 18f27bd2eb10..95de25d785a4 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/Conversions.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/Conversions.scala @@ -18,9 +18,11 @@ import com.daml.platform.server.api.validation.ErrorFactories import spray.json.DefaultJsonProtocol._ import spray.json._ +import scala.util.Try import java.io.BufferedReader -import java.sql.{PreparedStatement, Types} +import java.sql.{PreparedStatement, SQLNonTransientException, Types} import java.util.stream.Collectors + import scala.annotation.nowarn // TODO append-only: split this file on cleanup, and move anorm/db conversion related stuff to the right place @@ -143,7 +145,58 @@ private[platform] object Conversions { } object DefaultImplicitArrayColumn { - val default = Column.of[Array[String]] + val defaultString: Column[Array[String]] = Column.of[Array[String]] + val defaultInt: Column[Array[Int]] = Column.of[Array[Int]] + } + + object ArrayColumnToIntArray { + implicit val arrayColumnToIntArray: Column[Array[Int]] = nonNull { (value, meta) => + DefaultImplicitArrayColumn.defaultInt(value, meta) match { + case Right(value) => Right(value) + case Left(_) => + val MetaDataItem(qualified, _, _) = meta + value match { + case someArray: Array[_] => + Try( + someArray.view.map { + case i: Int => i + case null => + throw new SQLNonTransientException( + s"Cannot convert object array element null to Int" + ) + case invalid => + throw new SQLNonTransientException( + s"Cannot convert object array element (of type ${invalid.getClass.getName}) to Int" + ) + }.toArray + ).toEither.left.map(t => TypeDoesNotMatch(t.getMessage)) + + case jsonArrayString: String => + Right(jsonArrayString.parseJson.convertTo[Array[Int]]) + + case clob: java.sql.Clob => + try { + val reader = clob.getCharacterStream + val br = new BufferedReader(reader) + val jsonArrayString = br.lines.collect(Collectors.joining) + reader.close + Right(jsonArrayString.parseJson.convertTo[Array[Int]]) + } catch { + case e: Throwable => + Left( + TypeDoesNotMatch( + s"Cannot convert $value: received CLOB but failed to deserialize to " + + s"string array for column $qualified. Error message: ${e.getMessage}" + ) + ) + } + case _ => + Left( + TypeDoesNotMatch(s"Cannot convert $value: to string array for column $qualified") + ) + } + } + } } object ArrayColumnToStringArray { @@ -153,7 +206,7 @@ private[platform] object Conversions { // strategies implicit val arrayColumnToStringArray: Column[Array[String]] = nonNull { (value, meta) => - DefaultImplicitArrayColumn.default(value, meta) match { + DefaultImplicitArrayColumn.defaultString(value, meta) match { case Right(value) => Right(value) case Left(_) => val MetaDataItem(qualified, _, _) = meta @@ -252,11 +305,6 @@ private[platform] object Conversions { def contractId(columnName: String): RowParser[Value.ContractId] = SqlParser.get[Value.ContractId](columnName)(columnToContractId) - def flatEventWitnessesColumn(columnName: String): RowParser[Set[Ref.Party]] = - SqlParser - .get[Array[String]](columnName)(ArrayColumnToStringArray.arrayColumnToStringArray) - .map(_.iterator.map(Ref.Party.assertFromString).toSet) - // ContractIdString implicit val contractIdStringMetaParameter: ParameterMetaData[Ref.ContractIdString] = diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CompletionStorageBackendTemplate.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CompletionStorageBackendTemplate.scala index 6a8b9ea9e306..a3b0209c827f 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CompletionStorageBackendTemplate.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/CompletionStorageBackendTemplate.scala @@ -26,7 +26,7 @@ class CompletionStorageBackendTemplate( queryStrategy: QueryStrategy, stringInterning: StringInterning, ) extends CompletionStorageBackend { - assert(stringInterning != null) // TODO remove + private val logger: ContextualizedLogger = ContextualizedLogger.get(this.getClass) override def commandCompletions( @@ -59,7 +59,7 @@ class CompletionStorageBackendTemplate( ($startExclusive is null or completion_offset > $startExclusive) AND completion_offset <= $endInclusive AND application_id = $applicationId AND - ${queryStrategy.arrayIntersectionNonEmptyClause("submitters", parties)} + ${queryStrategy.arrayIntersectionNonEmptyClause("submitters", parties, stringInterning)} ORDER BY completion_offset ASC""" .as(completionParser.*)(connection) } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/ContractStorageBackendTemplate.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/ContractStorageBackendTemplate.scala index 1a101cc1dd05..e0bbf9b14ba4 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/ContractStorageBackendTemplate.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/ContractStorageBackendTemplate.scala @@ -5,22 +5,17 @@ package com.daml.platform.store.backend.common import java.sql.Connection -import anorm.SqlParser.{byteArray, int, long, str} +import anorm.SqlParser.{array, byteArray, int, long, str} import anorm.{ResultSetParser, Row, RowParser, SimpleSql, SqlParser, ~} import com.daml.lf.data.Ref import com.daml.lf.data.Time.Timestamp import com.daml.platform.apiserver.execution.MissingContracts -import com.daml.platform.store.Conversions.{ - contractId, - flatEventWitnessesColumn, - identifier, - offset, - timestampFromMicros, -} +import com.daml.platform.store.Conversions.{contractId, identifier, offset, timestampFromMicros} import com.daml.platform.store.SimpleSqlAsVectorOf.SimpleSqlAsVectorOf import com.daml.platform.store.appendonlydao.events.{ContractId, Key} import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} import com.daml.platform.store.backend.ContractStorageBackend +import com.daml.platform.store.backend.ContractStorageBackend.RawContractState import com.daml.platform.store.cache.LedgerEndCache import com.daml.platform.store.interfaces.LedgerDaoContractsReader.{ KeyAssigned, @@ -36,7 +31,8 @@ class ContractStorageBackendTemplate( ledgerEndCache: LedgerEndCache, stringInterning: StringInterning, ) extends ContractStorageBackend { - assert(stringInterning != null) // TODO remove + import com.daml.platform.store.Conversions.ArrayColumnToIntArray._ + override def contractKeyGlobally(key: Key)(connection: Connection): Option[ContractId] = contractKey( resultColumns = List("contract_id"), @@ -132,9 +128,12 @@ class ContractStorageBackendTemplate( resultColumns = List("contract_id", "flat_event_witnesses"), resultParser = ( contractId("contract_id") - ~ flatEventWitnessesColumn("flat_event_witnesses") + ~ array[Int]("flat_event_witnesses") ).map { case cId ~ stakeholders => - KeyAssigned(cId, stakeholders) + KeyAssigned( + cId, + stakeholders.view.map(stringInterning.party.externalize).toSet, + ) }, )( readers = None, @@ -144,13 +143,24 @@ class ContractStorageBackendTemplate( private val fullDetailsContractRowParser: RowParser[ContractStorageBackend.RawContractState] = (str("template_id").? - ~ flatEventWitnessesColumn("flat_event_witnesses") + ~ array[Int]("flat_event_witnesses") ~ byteArray("create_argument").? ~ int("create_argument_compression").? ~ int("event_kind") ~ timestampFromMicros("ledger_effective_time").?) - .map(SqlParser.flatten) - .map(ContractStorageBackend.RawContractState.tupled) + .map { + case internedTemplateId ~ flatEventWitnesses ~ createArgument ~ createArgumentCompression ~ eventKind ~ ledgerEffectiveTime => + RawContractState( + templateId = internedTemplateId, + flatEventWitnesses = flatEventWitnesses.view + .map(stringInterning.party.externalize) + .toSet, + createArgument = createArgument, + createArgumentCompression = createArgumentCompression, + eventKind = eventKind, + ledgerEffectiveTime = ledgerEffectiveTime, + ) + } override def contractState(contractId: ContractId, before: Long)( connection: Connection @@ -184,7 +194,7 @@ class ContractStorageBackendTemplate( byteArray("create_argument").? ~ int("create_argument_compression").? ~ long("event_sequential_id") ~ - flatEventWitnessesColumn("flat_event_witnesses") ~ + array[Int]("flat_event_witnesses") ~ offset("event_offset")).map { case eventKind ~ contractId ~ templateId ~ ledgerEffectiveTime ~ createKeyValue ~ createKeyCompression ~ createArgument ~ createArgumentCompression ~ eventSequentialId ~ flatEventWitnesses ~ offset => ContractStorageBackend.RawContractStateEvent( @@ -196,7 +206,9 @@ class ContractStorageBackendTemplate( createKeyCompression, createArgument, createArgumentCompression, - flatEventWitnesses, + flatEventWitnesses.view + .map(stringInterning.party.externalize) + .toSet, eventSequentialId, offset, ) @@ -315,6 +327,7 @@ class ContractStorageBackendTemplate( queryStrategy.arrayIntersectionNonEmptyClause( columnName = "tree_event_witnesses", parties = readers, + stringInterning = stringInterning, ) val coalescedColumns: String = resultColumns .map(columnName => @@ -391,6 +404,7 @@ class ContractStorageBackendTemplate( queryStrategy.arrayIntersectionNonEmptyClause( columnName = "last_contract_key_create.flat_event_witnesses", _, + stringInterning, ) ) val participantEventsFlatEventWitnessesClause = @@ -398,6 +412,7 @@ class ContractStorageBackendTemplate( queryStrategy.arrayIntersectionNonEmptyClause( columnName = "participant_events.flat_event_witnesses", _, + stringInterning, ) ) diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/EventStorageBackendTemplate.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/EventStorageBackendTemplate.scala index e454576dba2e..f3a58ed73948 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/EventStorageBackendTemplate.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/EventStorageBackendTemplate.scala @@ -38,8 +38,9 @@ abstract class EventStorageBackendTemplate( // Remove with the break-out of pruneEvents. participantAllDivulgedContractsPrunedUpToInclusive: Connection => Option[Offset], ) extends EventStorageBackend { - import com.daml.platform.store.Conversions.ArrayColumnToStringArray.arrayColumnToStringArray - assert(stringInterning != null) // TODO remove + import com.daml.platform.store.Conversions.ArrayColumnToIntArray._ + import com.daml.platform.store.Conversions.ArrayColumnToStringArray._ + private val logger: ContextualizedLogger = ContextualizedLogger.get(this.getClass) private val selectColumnsForFlatTransactions = @@ -64,7 +65,7 @@ abstract class EventStorageBackendTemplate( private type SharedRow = Offset ~ String ~ Int ~ Long ~ String ~ String ~ Timestamp ~ Identifier ~ Option[String] ~ - Option[String] ~ Array[String] + Option[String] ~ Array[Int] private val sharedRow: RowParser[SharedRow] = offset("event_offset") ~ @@ -77,25 +78,25 @@ abstract class EventStorageBackendTemplate( identifier("template_id") ~ str("command_id").? ~ str("workflow_id").? ~ - array[String]("event_witnesses") + array[Int]("event_witnesses") private type CreatedEventRow = - SharedRow ~ Array[Byte] ~ Option[Int] ~ Array[String] ~ Array[String] ~ Option[String] ~ + SharedRow ~ Array[Byte] ~ Option[Int] ~ Array[Int] ~ Array[Int] ~ Option[String] ~ Option[Array[Byte]] ~ Option[Int] private val createdEventRow: RowParser[CreatedEventRow] = sharedRow ~ byteArray("create_argument") ~ int("create_argument_compression").? ~ - array[String]("create_signatories") ~ - array[String]("create_observers") ~ + array[Int]("create_signatories") ~ + array[Int]("create_observers") ~ str("create_agreement_text").? ~ byteArray("create_key_value").? ~ int("create_key_value_compression").? private type ExercisedEventRow = SharedRow ~ Boolean ~ String ~ Array[Byte] ~ Option[Int] ~ Option[Array[Byte]] ~ Option[Int] ~ - Array[String] ~ Array[String] + Array[Int] ~ Array[String] private val exercisedEventRow: RowParser[ExercisedEventRow] = { import com.daml.platform.store.Conversions.bigDecimalColumnToBoolean @@ -106,7 +107,7 @@ abstract class EventStorageBackendTemplate( int("exercise_argument_compression").? ~ byteArray("exercise_result").? ~ int("exercise_result_compression").? ~ - array[String]("exercise_actors") ~ + array[Int]("exercise_actors") ~ array[String]("exercise_child_event_ids") } @@ -135,12 +136,18 @@ abstract class EventStorageBackendTemplate( templateId = templateId, createArgument = createArgument, createArgumentCompression = createArgumentCompression, - createSignatories = ArraySeq.unsafeWrapArray(createSignatories), - createObservers = ArraySeq.unsafeWrapArray(createObservers), + createSignatories = ArraySeq.unsafeWrapArray( + createSignatories.map(stringInterning.party.unsafe.externalize) + ), + createObservers = ArraySeq.unsafeWrapArray( + createObservers.map(stringInterning.party.unsafe.externalize) + ), createAgreementText = createAgreementText, createKeyValue = createKeyValue, createKeyValueCompression = createKeyValueCompression, - eventWitnesses = ArraySeq.unsafeWrapArray(eventWitnesses), + eventWitnesses = ArraySeq.unsafeWrapArray( + eventWitnesses.map(stringInterning.party.unsafe.externalize) + ), ), ) } @@ -162,7 +169,9 @@ abstract class EventStorageBackendTemplate( eventId = eventId, contractId = contractId, templateId = templateId, - eventWitnesses = ArraySeq.unsafeWrapArray(eventWitnesses), + eventWitnesses = ArraySeq.unsafeWrapArray( + eventWitnesses.map(stringInterning.party.unsafe.externalize) + ), ), ) } @@ -189,12 +198,18 @@ abstract class EventStorageBackendTemplate( templateId = templateId, createArgument = createArgument, createArgumentCompression = createArgumentCompression, - createSignatories = ArraySeq.unsafeWrapArray(createSignatories), - createObservers = ArraySeq.unsafeWrapArray(createObservers), + createSignatories = ArraySeq.unsafeWrapArray( + createSignatories.map(stringInterning.party.unsafe.externalize) + ), + createObservers = ArraySeq.unsafeWrapArray( + createObservers.map(stringInterning.party.unsafe.externalize) + ), createAgreementText = createAgreementText, createKeyValue = createKeyValue, createKeyValueCompression = createKeyValueCompression, - eventWitnesses = ArraySeq.unsafeWrapArray(eventWitnesses), + eventWitnesses = ArraySeq.unsafeWrapArray( + eventWitnesses.map(stringInterning.party.unsafe.externalize) + ), ), ) } @@ -222,9 +237,13 @@ abstract class EventStorageBackendTemplate( exerciseArgumentCompression = exerciseArgumentCompression, exerciseResult = exerciseResult, exerciseResultCompression = exerciseResultCompression, - exerciseActors = ArraySeq.unsafeWrapArray(exerciseActors), + exerciseActors = ArraySeq.unsafeWrapArray( + exerciseActors.map(stringInterning.party.unsafe.externalize) + ), exerciseChildEventIds = ArraySeq.unsafeWrapArray(exerciseChildEventIds), - eventWitnesses = ArraySeq.unsafeWrapArray(eventWitnesses), + eventWitnesses = ArraySeq.unsafeWrapArray( + eventWitnesses.map(stringInterning.party.unsafe.externalize) + ), ), ) } @@ -274,14 +293,14 @@ abstract class EventStorageBackendTemplate( SQL""" SELECT #$selectColumns, ${eventStrategy - .filteredEventWitnessesClause(witnessesColumn, parties)} as event_witnesses, + .filteredEventWitnessesClause(witnessesColumn, parties, stringInterning)} as event_witnesses, case when ${eventStrategy - .submittersArePartiesClause("submitters", parties)} then command_id else '' end as command_id + .submittersArePartiesClause("submitters", parties, stringInterning)} then command_id else '' end as command_id FROM participant_events #$columnPrefix $joinClause WHERE $additionalAndClause - ${eventStrategy.witnessesWhereClause(witnessesColumn, filterParams)} + ${eventStrategy.witnessesWhereClause(witnessesColumn, filterParams, stringInterning)} ORDER BY event_sequential_id ${queryStrategy.limitClause(limit)}""" .withFetchSize(fetchSizeHint) @@ -477,7 +496,7 @@ abstract class EventStorageBackendTemplate( where p.typ = 'accept' and p.ledger_offset <= c.event_offset and #${queryStrategy.isTrue("p.is_local")} - and #${queryStrategy.arrayContains("c.flat_event_witnesses", "p.party")} + and #${queryStrategy.arrayContains("c.flat_event_witnesses", "p.party_id")} ) $pruneAfterClause """ @@ -520,22 +539,22 @@ abstract class EventStorageBackendTemplate( contractId("contract_id") ~ identifier("template_id").? ~ timestampFromMicros("ledger_effective_time").? ~ - array[String]("create_signatories").? ~ - array[String]("create_observers").? ~ + array[Int]("create_signatories").? ~ + array[Int]("create_observers").? ~ str("create_agreement_text").? ~ byteArray("create_key_value").? ~ int("create_key_value_compression").? ~ byteArray("create_argument").? ~ int("create_argument_compression").? ~ - array[String]("tree_event_witnesses") ~ - array[String]("flat_event_witnesses") ~ - array[String]("submitters").? ~ + array[Int]("tree_event_witnesses") ~ + array[Int]("flat_event_witnesses") ~ + array[Int]("submitters").? ~ str("exercise_choice").? ~ byteArray("exercise_argument").? ~ int("exercise_argument_compression").? ~ byteArray("exercise_result").? ~ int("exercise_result_compression").? ~ - array[String]("exercise_actors").? ~ + array[Int]("exercise_actors").? ~ array[String]("exercise_child_event_ids").? ~ long("event_sequential_id") ~ offset("event_offset")).map { @@ -554,22 +573,24 @@ abstract class EventStorageBackendTemplate( contractId, templateId, ledgerEffectiveTime, - createSignatories, - createObservers, + createSignatories.map(_.map(stringInterning.party.unsafe.externalize)), + createObservers.map(_.map(stringInterning.party.unsafe.externalize)), createAgreementText, createKeyValue, createKeyCompression, createArgument, createArgumentCompression, - treeEventWitnesses.toSet, - flatEventWitnesses.toSet, - submitters.map(_.toSet).getOrElse(Set.empty), + treeEventWitnesses.view.map(stringInterning.party.unsafe.externalize).toSet, + flatEventWitnesses.view.map(stringInterning.party.unsafe.externalize).toSet, + submitters + .map(_.view.map(stringInterning.party.unsafe.externalize).toSet) + .getOrElse(Set.empty), exerciseChoice, exerciseArgument, exerciseArgumentCompression, exerciseResult, exerciseResultCompression, - exerciseActors, + exerciseActors.map(_.map(stringInterning.party.unsafe.externalize)), exerciseChildEventIds, eventSequentialId, offset, @@ -635,6 +656,7 @@ trait EventStrategy { def filteredEventWitnessesClause( witnessesColumnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql /** This populates the following part of the query: @@ -648,6 +670,7 @@ trait EventStrategy { def submittersArePartiesClause( submittersColumnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql /** This populates the following part of the query: @@ -661,5 +684,6 @@ trait EventStrategy { def witnessesWhereClause( witnessesColumnName: String, filterParams: FilterParams, + stringInterning: StringInterning, ): CompositeSql } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala index 6aed295945b3..cb554425ae49 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/QueryStrategy.scala @@ -5,6 +5,7 @@ package com.daml.platform.store.backend.common import com.daml.lf.data.Ref import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} +import com.daml.platform.store.interning.StringInterning trait QueryStrategy { @@ -38,6 +39,7 @@ trait QueryStrategy { def arrayIntersectionNonEmptyClause( columnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql /** Would be used in column selectors in GROUP BY situations to see whether a boolean column had true diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala index e49c9f2d14ad..46b3255b273d 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala @@ -101,10 +101,14 @@ private[backend] object AppendOnlySchema { "command_id" -> fieldStrategy.stringOptional(_ => _.command_id), "workflow_id" -> fieldStrategy.stringOptional(_ => _.workflow_id), "application_id" -> fieldStrategy.stringOptional(_ => _.application_id), - "submitters" -> fieldStrategy.stringArrayOptional(_ => _.submitters), + "submitters" -> fieldStrategy.intArrayOptional(stringInterning => + _.submitters.map(_.map(stringInterning.party.unsafe.internalize)) + ), "contract_id" -> fieldStrategy.string(_ => _.contract_id), "template_id" -> fieldStrategy.stringOptional(_ => _.template_id), - "tree_event_witnesses" -> fieldStrategy.stringArray(_ => _.tree_event_witnesses), + "tree_event_witnesses" -> fieldStrategy.intArray(stringInterning => + _.tree_event_witnesses.map(stringInterning.party.unsafe.internalize) + ), "create_argument" -> fieldStrategy.byteaOptional(_ => _.create_argument), "event_sequential_id" -> fieldStrategy.bigint(_ => _.event_sequential_id), "create_argument_compression" -> fieldStrategy.smallintOptional(_ => @@ -120,16 +124,26 @@ private[backend] object AppendOnlySchema { "command_id" -> fieldStrategy.stringOptional(_ => _.command_id), "workflow_id" -> fieldStrategy.stringOptional(_ => _.workflow_id), "application_id" -> fieldStrategy.stringOptional(_ => _.application_id), - "submitters" -> fieldStrategy.stringArrayOptional(_ => _.submitters), + "submitters" -> fieldStrategy.intArrayOptional(stringInterning => + _.submitters.map(_.map(stringInterning.party.unsafe.internalize)) + ), "node_index" -> fieldStrategy.intOptional(_ => _.node_index), "event_id" -> fieldStrategy.stringOptional(_ => _.event_id), "contract_id" -> fieldStrategy.string(_ => _.contract_id), "template_id" -> fieldStrategy.stringOptional(_ => _.template_id), - "flat_event_witnesses" -> fieldStrategy.stringArray(_ => _.flat_event_witnesses), - "tree_event_witnesses" -> fieldStrategy.stringArray(_ => _.tree_event_witnesses), + "flat_event_witnesses" -> fieldStrategy.intArray(stringInterning => + _.flat_event_witnesses.map(stringInterning.party.unsafe.internalize) + ), + "tree_event_witnesses" -> fieldStrategy.intArray(stringInterning => + _.tree_event_witnesses.map(stringInterning.party.unsafe.internalize) + ), "create_argument" -> fieldStrategy.byteaOptional(_ => _.create_argument), - "create_signatories" -> fieldStrategy.stringArrayOptional(_ => _.create_signatories), - "create_observers" -> fieldStrategy.stringArrayOptional(_ => _.create_observers), + "create_signatories" -> fieldStrategy.intArrayOptional(stringInterning => + _.create_signatories.map(_.map(stringInterning.party.unsafe.internalize)) + ), + "create_observers" -> fieldStrategy.intArrayOptional(stringInterning => + _.create_observers.map(_.map(stringInterning.party.unsafe.internalize)) + ), "create_agreement_text" -> fieldStrategy.stringOptional(_ => _.create_agreement_text), "create_key_value" -> fieldStrategy.byteaOptional(_ => _.create_key_value), "create_key_hash" -> fieldStrategy.stringOptional(_ => _.create_key_hash), @@ -153,18 +167,26 @@ private[backend] object AppendOnlySchema { "command_id" -> fieldStrategy.stringOptional(_ => _.command_id), "workflow_id" -> fieldStrategy.stringOptional(_ => _.workflow_id), "application_id" -> fieldStrategy.stringOptional(_ => _.application_id), - "submitters" -> fieldStrategy.stringArrayOptional(_ => _.submitters), + "submitters" -> fieldStrategy.intArrayOptional(stringInterning => + _.submitters.map(_.map(stringInterning.party.unsafe.internalize)) + ), "create_key_value" -> fieldStrategy.byteaOptional(_ => _.create_key_value), "exercise_choice" -> fieldStrategy.stringOptional(_ => _.exercise_choice), "exercise_argument" -> fieldStrategy.byteaOptional(_ => _.exercise_argument), "exercise_result" -> fieldStrategy.byteaOptional(_ => _.exercise_result), - "exercise_actors" -> fieldStrategy.stringArrayOptional(_ => _.exercise_actors), + "exercise_actors" -> fieldStrategy.intArrayOptional(stringInterning => + _.exercise_actors.map(_.map(stringInterning.party.unsafe.internalize)) + ), "exercise_child_event_ids" -> fieldStrategy.stringArrayOptional(_ => _.exercise_child_event_ids ), "template_id" -> fieldStrategy.stringOptional(_ => _.template_id), - "flat_event_witnesses" -> fieldStrategy.stringArray(_ => _.flat_event_witnesses), - "tree_event_witnesses" -> fieldStrategy.stringArray(_ => _.tree_event_witnesses), + "flat_event_witnesses" -> fieldStrategy.intArray(stringInterning => + _.flat_event_witnesses.map(stringInterning.party.unsafe.internalize) + ), + "tree_event_witnesses" -> fieldStrategy.intArray(stringInterning => + _.tree_event_witnesses.map(stringInterning.party.unsafe.internalize) + ), "event_sequential_id" -> fieldStrategy.bigint(_ => _.event_sequential_id), "create_key_value_compression" -> fieldStrategy.smallintOptional(_ => _.create_key_value_compression @@ -226,6 +248,9 @@ private[backend] object AppendOnlySchema { "typ" -> fieldStrategy.string(_ => _.typ), "rejection_reason" -> fieldStrategy.stringOptional(_ => _.rejection_reason), "is_local" -> fieldStrategy.booleanOptional(_ => _.is_local), + "party_id" -> fieldStrategy.intOptional(stringInterning => + _.party.map(stringInterning.party.unsafe.internalize) + ), ) val commandCompletions: Table[DbDto.CommandCompletion] = @@ -233,7 +258,9 @@ private[backend] object AppendOnlySchema { "completion_offset" -> fieldStrategy.string(_ => _.completion_offset), "record_time" -> fieldStrategy.bigint(_ => _.record_time), "application_id" -> fieldStrategy.string(_ => _.application_id), - "submitters" -> fieldStrategy.stringArray(_ => _.submitters), + "submitters" -> fieldStrategy.intArray(stringInterning => + _.submitters.map(stringInterning.party.unsafe.internalize) + ), "command_id" -> fieldStrategy.string(_ => _.command_id), "transaction_id" -> fieldStrategy.stringOptional(_ => _.transaction_id), "rejection_status_code" -> fieldStrategy.intOptional(_ => _.rejection_status_code), diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2EventStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2EventStrategy.scala index d758c18b315e..0e4aa265bdd0 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2EventStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2EventStrategy.scala @@ -7,35 +7,42 @@ import com.daml.lf.data.Ref import com.daml.platform.store.backend.EventStorageBackend.FilterParams import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} import com.daml.platform.store.backend.common.EventStrategy +import com.daml.platform.store.interning.StringInterning object H2EventStrategy extends EventStrategy { override def filteredEventWitnessesClause( witnessesColumnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql = { - val partiesArray = parties.view.map(_.toString).toArray - cSQL"array_intersection(#$witnessesColumnName, $partiesArray)" + val partiesArray: Array[java.lang.Integer] = + parties.view.map(stringInterning.party.tryInternalize).flatMap(_.toList).map(Int.box).toArray + if (partiesArray.isEmpty) cSQL"false" + else cSQL"array_intersection(#$witnessesColumnName, $partiesArray)" } override def submittersArePartiesClause( submittersColumnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql = H2QueryStrategy.arrayIntersectionNonEmptyClause( columnName = submittersColumnName, parties = parties, + stringInterning = stringInterning, ) override def witnessesWhereClause( witnessesColumnName: String, filterParams: FilterParams, + stringInterning: StringInterning, ): CompositeSql = { val wildCardClause = filterParams.wildCardParties match { case wildCardParties if wildCardParties.isEmpty => Nil case wildCardParties => - cSQL"(${H2QueryStrategy.arrayIntersectionNonEmptyClause(witnessesColumnName, wildCardParties)})" :: Nil + cSQL"(${H2QueryStrategy.arrayIntersectionNonEmptyClause(witnessesColumnName, wildCardParties, stringInterning)})" :: Nil } val partiesTemplatesClauses = filterParams.partiesAndTemplates.iterator.map { case (parties, templateIds) => @@ -43,6 +50,7 @@ object H2EventStrategy extends EventStrategy { H2QueryStrategy.arrayIntersectionNonEmptyClause( witnessesColumnName, parties, + stringInterning, ) val templateIdsArray = templateIds.view.map(_.toString).toArray cSQL"( ($clause) AND (template_id = ANY($templateIdsArray)) )" diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Field.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Field.scala new file mode 100644 index 000000000000..0eb009c4d0d8 --- /dev/null +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Field.scala @@ -0,0 +1,19 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.store.backend.h2 + +import com.daml.platform.store.backend.common.Field +import com.daml.platform.store.interning.StringInterning + +private[h2] case class IntArray[FROM](extract: StringInterning => FROM => Iterable[Int]) + extends Field[FROM, Iterable[Int], Array[java.lang.Integer]] { + override def convert: Iterable[Int] => Array[java.lang.Integer] = _.view.map(Int.box).toArray +} + +private[h2] case class IntArrayOptional[FROM]( + extract: StringInterning => FROM => Option[Iterable[Int]] +) extends Field[FROM, Option[Iterable[Int]], Array[java.lang.Integer]] { + override def convert: Option[Iterable[Int]] => Array[java.lang.Integer] = + _.map(_.view.map(Int.box).toArray).orNull +} diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2FunctionAliases.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2FunctionAliases.scala index 2769a668e990..13da82ac5995 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2FunctionAliases.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2FunctionAliases.scala @@ -6,7 +6,10 @@ package com.daml.platform.store.backend.h2 // Warning this object is accessed directly by H2Database, see "CREATE ALIAS" in H2 related FLyway scripts object H2FunctionAliases { - def arrayIntersection(a: Array[String], b: Array[String]): Array[String] = + def arrayIntersection( + a: Array[java.lang.Integer], + b: Array[java.lang.Integer], + ): Array[java.lang.Integer] = a.toSet.intersect(b.toSet).toArray } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2QueryStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2QueryStrategy.scala index 527becbbf834..f18a63445b48 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2QueryStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2QueryStrategy.scala @@ -6,19 +6,26 @@ package com.daml.platform.store.backend.h2 import com.daml.lf.data.Ref import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} import com.daml.platform.store.backend.common.QueryStrategy +import com.daml.platform.store.interning.StringInterning object H2QueryStrategy extends QueryStrategy { override def arrayIntersectionNonEmptyClause( columnName: String, parties: Set[Ref.Party], - ): CompositeSql = - if (parties.isEmpty) + stringInterning: StringInterning, + ): CompositeSql = { + val internedParties = parties.view + .map(stringInterning.party.tryInternalize) + .flatMap(_.toList) + .map(p => cSQL"array_contains(#$columnName, $p)") + .toList + if (internedParties.isEmpty) cSQL"false" else - parties.view - .map(p => cSQL"array_contains(#$columnName, '#${p.toString}')") + internedParties .mkComposite("(", " or ", ")") + } override def arrayContains(arrayColumnName: String, elementColumnName: String): String = s"array_contains($arrayColumnName, $elementColumnName)" diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Schema.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Schema.scala index b68e8ae71f07..18e25381c577 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Schema.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2Schema.scala @@ -6,9 +6,20 @@ package com.daml.platform.store.backend.h2 import com.daml.platform.store.backend.DbDto import com.daml.platform.store.backend.common.AppendOnlySchema.FieldStrategy import com.daml.platform.store.backend.common.{AppendOnlySchema, Field, Schema, Table} +import com.daml.platform.store.interning.StringInterning private[h2] object H2Schema { private val H2FieldStrategy = new FieldStrategy { + override def intArray[FROM, _]( + extractor: StringInterning => FROM => Iterable[Int] + ): Field[FROM, Iterable[Int], _] = + IntArray(extractor) + + override def intArrayOptional[FROM, _]( + extractor: StringInterning => FROM => Option[Iterable[Int]] + ): Field[FROM, Option[Iterable[Int]], _] = + IntArrayOptional(extractor) + override def insert[FROM](tableName: String)( fields: (String, Field[FROM, _, _])* ): Table[FROM] = diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleEventStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleEventStrategy.scala index d7e2de1cc860..b54b37e92d12 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleEventStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleEventStrategy.scala @@ -7,48 +7,79 @@ import com.daml.lf.data.Ref import com.daml.platform.store.backend.EventStorageBackend.FilterParams import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} import com.daml.platform.store.backend.common.EventStrategy +import com.daml.platform.store.interning.StringInterning object OracleEventStrategy extends EventStrategy { override def filteredEventWitnessesClause( witnessesColumnName: String, parties: Set[Ref.Party], - ): CompositeSql = - if (parties.size == 1) - cSQL"(json_array(${parties.head.toString}))" - else - cSQL""" + stringInterning: StringInterning, + ): CompositeSql = { + val internedParties = + parties.view.map(stringInterning.party.tryInternalize).flatMap(_.toList).toSet + internedParties.size match { + case 0 => cSQL"json_array()" + case 1 => cSQL"(json_array(${internedParties.head}))" + case _ => + cSQL""" (select json_arrayagg(value) from (select value - from json_table(#$witnessesColumnName, '$$[*]' columns (value PATH '$$')) - where value IN (${parties.map(_.toString)}))) + from json_table(#$witnessesColumnName, '$$[*]' columns (value NUMBER PATH '$$')) + where value IN ($internedParties))) """ + } + } override def submittersArePartiesClause( submittersColumnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql = - cSQL"(${OracleQueryStrategy.arrayIntersectionNonEmptyClause(submittersColumnName, parties)})" + OracleQueryStrategy.arrayIntersectionNonEmptyClause( + submittersColumnName, + parties, + stringInterning, + ) override def witnessesWhereClause( witnessesColumnName: String, filterParams: FilterParams, + stringInterning: StringInterning, ): CompositeSql = { val wildCardClause = filterParams.wildCardParties match { - case wildCardParties if wildCardParties.isEmpty => + case wildCardParties + if wildCardParties.isEmpty || + wildCardParties.view + .flatMap(party => stringInterning.party.tryInternalize(party).toList) + .isEmpty => Nil case wildCardParties => - cSQL"(${OracleQueryStrategy.arrayIntersectionNonEmptyClause(witnessesColumnName, wildCardParties)})" :: Nil + cSQL"(${OracleQueryStrategy.arrayIntersectionNonEmptyClause(witnessesColumnName, wildCardParties, stringInterning)})" :: Nil } val partiesTemplatesClauses = - filterParams.partiesAndTemplates.iterator.map { case (parties, templateIds) => - val clause = - OracleQueryStrategy.arrayIntersectionNonEmptyClause( - witnessesColumnName, - parties, + filterParams.partiesAndTemplates.iterator + .map { case (parties, templateIds) => + ( + parties.flatMap(s => stringInterning.party.tryInternalize(s).toList), + templateIds, ) - cSQL"( ($clause) AND (template_id IN (${templateIds.map(_.toString)})) )" - }.toList - (wildCardClause ::: partiesTemplatesClauses).mkComposite("(", " OR ", ")") + } + .filterNot(_._1.isEmpty) + .filterNot(_._2.isEmpty) + .map { case (parties, templateIds) => + val clause = + OracleQueryStrategy.arrayIntersectionNonEmptyClause( + witnessesColumnName, + parties.map(stringInterning.party.externalize), + stringInterning, + ) + cSQL"( ($clause) AND (template_id IN (${templateIds.map(_.toString)})) )" + } + .toList + wildCardClause ::: partiesTemplatesClauses match { + case Nil => cSQL"1 = 0" + case allClauses => allClauses.mkComposite("(", " OR ", ")") + } } } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleField.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleField.scala index cf4cf02c2cd5..0295f38493e3 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleField.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleField.scala @@ -36,3 +36,30 @@ private[oracle] case class OracleStringArrayOptional[FROM]( preparedStatement.setObject(index, value) } } + +private[oracle] case class OracleIntArray[FROM]( + extract: StringInterning => FROM => Iterable[Int] +) extends Field[FROM, Iterable[Int], String] { + override def convert: Iterable[Int] => String = _.toList.toJson.compactPrint + override def prepareDataTemplate( + preparedStatement: PreparedStatement, + index: Int, + value: String, + ): Unit = { + preparedStatement.setObject(index, value) + } +} + +private[oracle] case class OracleIntArrayOptional[FROM]( + extract: StringInterning => FROM => Option[Iterable[Int]] +) extends Field[FROM, Option[Iterable[Int]], String] { + override def convert: Option[Iterable[Int]] => String = + _.map(_.toList.toJson.compactPrint).getOrElse("[]") + override def prepareDataTemplate( + preparedStatement: PreparedStatement, + index: Int, + value: String, + ): Unit = { + preparedStatement.setObject(index, value) + } +} diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleQueryStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleQueryStrategy.scala index 6975d2d5b567..cef94d741fce 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleQueryStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleQueryStrategy.scala @@ -6,15 +6,21 @@ package com.daml.platform.store.backend.oracle import com.daml.lf.data.Ref import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} import com.daml.platform.store.backend.common.QueryStrategy +import com.daml.platform.store.interning.StringInterning object OracleQueryStrategy extends QueryStrategy { override def arrayIntersectionNonEmptyClause( columnName: String, parties: Set[Ref.Party], - ): CompositeSql = - cSQL"EXISTS (SELECT 1 FROM JSON_TABLE(#$columnName, '$$[*]' columns (value PATH '$$')) WHERE value IN (${parties - .map(_.toString)}))" + stringInterning: StringInterning, + ): CompositeSql = { + val internedParties = + parties.view.map(stringInterning.party.tryInternalize).flatMap(_.toList).toSet + if (internedParties.isEmpty) cSQL"1 = 0" + else + cSQL"(EXISTS (SELECT 1 FROM JSON_TABLE(#$columnName, '$$[*]' columns (value NUMBER PATH '$$')) WHERE value IN ($internedParties)))" + } override def columnEqualityBoolean(column: String, value: String): String = s"""case when ($column = $value) then 1 else 0 end""" @@ -22,7 +28,7 @@ object OracleQueryStrategy extends QueryStrategy { override def booleanOrAggregationFunction: String = "max" override def arrayContains(arrayColumnName: String, elementColumnName: String): String = - s"EXISTS (SELECT 1 FROM JSON_TABLE($arrayColumnName, '$$[*]' columns (value PATH '$$')) WHERE value = $elementColumnName)" + s"EXISTS (SELECT 1 FROM JSON_TABLE($arrayColumnName, '$$[*]' columns (value NUMBER PATH '$$')) WHERE value = $elementColumnName)" override def isTrue(booleanColumnName: String): String = s"$booleanColumnName = 1" } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleSchema.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleSchema.scala index 3937ae4d815d..944093589b59 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleSchema.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleSchema.scala @@ -20,6 +20,16 @@ private[oracle] object OracleSchema { ): Field[FROM, Option[Iterable[String]], _] = OracleStringArrayOptional(extractor) + override def intArray[FROM, _]( + extractor: StringInterning => FROM => Iterable[Int] + ): Field[FROM, Iterable[Int], _] = + OracleIntArray(extractor) + + override def intArrayOptional[FROM, _]( + extractor: StringInterning => FROM => Option[Iterable[Int]] + ): Field[FROM, Option[Iterable[Int]], _] = + OracleIntArrayOptional(extractor) + override def insert[FROM](tableName: String)( fields: (String, Field[FROM, _, _])* ): Table[FROM] = diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGField.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGField.scala index b654cc022832..9ab6e6139e29 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGField.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGField.scala @@ -31,6 +31,26 @@ private[postgresql] case class PGStringArrayOptional[FROM]( override def convert: Option[Iterable[String]] => String = _.map(convertBase).orNull } +private[postgresql] trait PGIntArrayBase[FROM, TO] extends Field[FROM, TO, String] { + override def selectFieldExpression(inputFieldName: String): String = + s"string_to_array($inputFieldName, '|')::integer[]" // TODO interning consider doing some hex magic here to compress the transport data more + + protected def convertBase: Iterable[Int] => String = { in => + in.mkString("|") + } +} + +private[postgresql] case class PGIntArray[FROM](extract: StringInterning => FROM => Iterable[Int]) + extends PGIntArrayBase[FROM, Iterable[Int]] { + override def convert: Iterable[Int] => String = convertBase +} + +private[postgresql] case class PGIntArrayOptional[FROM]( + extract: StringInterning => FROM => Option[Iterable[Int]] +) extends PGIntArrayBase[FROM, Option[Iterable[Int]]] { + override def convert: Option[Iterable[Int]] => String = _.map(convertBase).orNull +} + private[postgresql] case class PGSmallintOptional[FROM]( extract: StringInterning => FROM => Option[Int] ) extends Field[FROM, Option[Int], java.lang.Integer] { diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGSchema.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGSchema.scala index ecb3e0ee0cb5..6d7f1658c687 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGSchema.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PGSchema.scala @@ -20,6 +20,16 @@ private[postgresql] object PGSchema { ): Field[FROM, Option[Iterable[String]], _] = PGStringArrayOptional(extractor) + override def intArray[FROM, _]( + extractor: StringInterning => FROM => Iterable[Int] + ): Field[FROM, Iterable[Int], _] = + PGIntArray(extractor) + + override def intArrayOptional[FROM, _]( + extractor: StringInterning => FROM => Option[Iterable[Int]] + ): Field[FROM, Option[Iterable[Int]], _] = + PGIntArrayOptional(extractor) + override def smallintOptional[FROM, _]( extractor: StringInterning => FROM => Option[Int] ): Field[FROM, Option[Int], _] = diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresEventStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresEventStrategy.scala index 6aefd1a53743..82be018df596 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresEventStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresEventStrategy.scala @@ -7,45 +7,72 @@ import com.daml.platform.store.appendonlydao.events.Party import com.daml.platform.store.backend.EventStorageBackend.FilterParams import com.daml.platform.store.backend.common.ComposableQuery.{CompositeSql, SqlStringInterpolation} import com.daml.platform.store.backend.common.EventStrategy +import com.daml.platform.store.interning.StringInterning object PostgresEventStrategy extends EventStrategy { override def filteredEventWitnessesClause( witnessesColumnName: String, parties: Set[Party], - ): CompositeSql = - if (parties.size == 1) - cSQL"array[${parties.head.toString}]::text[]" - else { - val partiesArray: Array[String] = parties.view.map(_.toString).toArray - cSQL"array(select unnest(#$witnessesColumnName) intersect select unnest($partiesArray::text[]))" - } + stringInterning: StringInterning, + ): CompositeSql = { + val internedParties: Array[java.lang.Integer] = parties.view + .flatMap(party => stringInterning.party.tryInternalize(party).map(Int.box).toList) + .toArray + if (internedParties.length == 1) + cSQL"array[${internedParties.head}]::integer[]" + else + cSQL"array(select unnest(#$witnessesColumnName) intersect select unnest($internedParties::integer[]))" + } override def submittersArePartiesClause( submittersColumnName: String, parties: Set[Party], + stringInterning: StringInterning, ): CompositeSql = { - val partiesArray = parties.view.map(_.toString).toArray - cSQL"(#$submittersColumnName::text[] && $partiesArray::text[])" + val partiesArray: Array[java.lang.Integer] = parties.view + .flatMap(party => stringInterning.party.tryInternalize(party).map(Int.box).toList) + .toArray + cSQL"(#$submittersColumnName::integer[] && $partiesArray::integer[])" } override def witnessesWhereClause( witnessesColumnName: String, filterParams: FilterParams, + stringInterning: StringInterning, ): CompositeSql = { val wildCardClause = filterParams.wildCardParties match { case wildCardParties if wildCardParties.isEmpty => Nil case wildCardParties => - val partiesArray = wildCardParties.view.map(_.toString).toArray - cSQL"(#$witnessesColumnName::text[] && $partiesArray::text[])" :: Nil + val partiesArray: Array[java.lang.Integer] = wildCardParties.view + .flatMap(party => stringInterning.party.tryInternalize(party).map(Int.box).toList) + .toArray + if (partiesArray.isEmpty) + Nil + else + cSQL"(#$witnessesColumnName::integer[] && $partiesArray::integer[])" :: Nil } val partiesTemplatesClauses = - filterParams.partiesAndTemplates.iterator.map { case (parties, templateIds) => - val partiesArray = parties.view.map(_.toString).toArray - val templateIdsArray = templateIds.view.map(_.toString).toArray - cSQL"( (#$witnessesColumnName::text[] && $partiesArray::text[]) AND (template_id = ANY($templateIdsArray::text[])) )" - }.toList - (wildCardClause ::: partiesTemplatesClauses).mkComposite("(", " OR ", ")") + filterParams.partiesAndTemplates.iterator + .map { case (parties, templateIds) => + ( + parties.flatMap(s => stringInterning.party.tryInternalize(s).toList), + templateIds, + ) + } + .filterNot(_._1.isEmpty) + .filterNot(_._2.isEmpty) + .map { case (parties, templateIds) => + val partiesArray: Array[java.lang.Integer] = parties.view.map(Int.box).toArray + val templateIdsArray = templateIds.view.map(_.toString).toArray + cSQL"( (#$witnessesColumnName::integer[] && $partiesArray::integer[]) AND (template_id = ANY($templateIdsArray::text[])) )" + } + .toList + + wildCardClause ::: partiesTemplatesClauses match { + case Nil => cSQL"false" + case allClauses => allClauses.mkComposite("(", " OR ", ")") + } } } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresQueryStrategy.scala b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresQueryStrategy.scala index 10e187dcf10f..b0ff786770eb 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresQueryStrategy.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresQueryStrategy.scala @@ -6,16 +6,21 @@ package com.daml.platform.store.backend.postgresql import com.daml.lf.data.Ref import com.daml.platform.store.backend.common.ComposableQuery.CompositeSql import com.daml.platform.store.backend.common.QueryStrategy +import com.daml.platform.store.interning.StringInterning object PostgresQueryStrategy extends QueryStrategy { override def arrayIntersectionNonEmptyClause( columnName: String, parties: Set[Ref.Party], + stringInterning: StringInterning, ): CompositeSql = { import com.daml.platform.store.backend.common.ComposableQuery.SqlStringInterpolation - val partiesArray: Array[String] = parties.map(_.toString).toArray - cSQL"#$columnName::text[] && $partiesArray::text[]" + val partiesArray: Array[java.lang.Integer] = + parties + .flatMap(party => stringInterning.party.tryInternalize(party).map(Int.box).toList) + .toArray + cSQL"#$columnName::int[] && $partiesArray::int[]" } override def arrayContains(arrayColumnName: String, elementColumnName: String): String = diff --git a/ledger/participant-integration-api/src/test/lib/scala/platform/store/dao/JdbcLedgerDaoBackend.scala b/ledger/participant-integration-api/src/test/lib/scala/platform/store/dao/JdbcLedgerDaoBackend.scala index 41ac5443db17..6b3633af3af4 100644 --- a/ledger/participant-integration-api/src/test/lib/scala/platform/store/dao/JdbcLedgerDaoBackend.scala +++ b/ledger/participant-integration-api/src/test/lib/scala/platform/store/dao/JdbcLedgerDaoBackend.scala @@ -29,7 +29,7 @@ import com.daml.platform.store.interning.StringInterningView import com.daml.platform.store.{DbType, FlywayMigrations, LfValueTranslationCache} import org.scalatest.AsyncTestSuite -import scala.concurrent.Await +import scala.concurrent.{Await, Future} import scala.concurrent.duration.DurationInt object JdbcLedgerDaoBackend { @@ -71,15 +71,6 @@ private[dao] trait JdbcLedgerDaoBackend extends AkkaBeforeAndAfterAll { metrics = metrics, ) .map { dbDispatcher => - val stringInterningStorageBackend = - storageBackendFactory.createStringInterningStorageBackend - val stringInterningView = new StringInterningView( - loadPrefixedEntries = (fromExclusive, toInclusive) => - implicit loggingContext => - dbDispatcher.executeSql(metrics.daml.index.db.loadStringInterningEntries) { - stringInterningStorageBackend.loadStringInterningEntries(fromExclusive, toInclusive) - } - ) JdbcLedgerDao.write( dbDispatcher = dbDispatcher, sequentialWriteDao = SequentialWriteDao( @@ -109,6 +100,7 @@ private[dao] trait JdbcLedgerDaoBackend extends AkkaBeforeAndAfterAll { protected final var ledgerDao: LedgerDao = _ protected var ledgerEndCache: MutableLedgerEndCache = _ + protected var stringInterningView: StringInterningView = _ // `dbDispatcher` and `ledgerDao` depend on the `postgresFixture` which is in turn initialized `beforeAll` private var resource: Resource[LedgerDao] = _ @@ -121,6 +113,7 @@ private[dao] trait JdbcLedgerDaoBackend extends AkkaBeforeAndAfterAll { // We use the dispatcher here because the default Scalatest execution context is too slow. implicit val resourceContext: ResourceContext = ResourceContext(system.dispatcher) ledgerEndCache = MutableLedgerEndCache() + stringInterningView = new StringInterningView((_, _) => _ => Future.successful(Nil)) resource = newLoggingContext { implicit loggingContext => for { _ <- Resource.fromFuture(