Activate party interning [DPP-712] (#11663)

* Add DB schema changes
* Data migration scripts (PostgreSQL only)
* Add StringInterning to Query strategies
* Adapt queries, result-parsers
* Adapt ingestion
* Fix JdbcLedgerDaoBackend StringInterning handling

changelog_begin
changelog_end

ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256

@@ -1 +1 @@
-bd9ddd16ae4ecc09923215635442c83b7894e95d23203baccde6df27cb0ce2cf
+b59f4464e996e785861401df83d8b903436f3e0287fb62b884711feedae14ef7

ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql

@@ -82,6 +82,7 @@ CREATE TABLE party_entries (
     typ VARCHAR NOT NULL,
     rejection_reason VARCHAR,
     is_local BOOLEAN,
+    party_id INTEGER,
 
     CONSTRAINT check_party_entry_type
         CHECK (
@@ -92,6 +93,7 @@ CREATE TABLE party_entries (
 
 CREATE INDEX idx_party_entries ON party_entries (submission_id);
 CREATE INDEX idx_party_entries_party_and_ledger_offset ON party_entries(party, ledger_offset);
+CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset);
 
 ---------------------------------------------------------------------------------------------------
 -- Submissions table

ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sha256

@@ -0,0 +1 @@
+d438a8924cfd01c9c28aae1a1bc33c3f202433748b3d3486305ced37128d7039

ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V5__activate_party_interning.sql

@@ -0,0 +1,7 @@
+--  Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+--  SPDX-License-Identifier: Apache-2.0
+
+-- add party_id to party_entries
+ALTER TABLE party_entries
+  ADD party_id NUMBER;
+CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset);

ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sha256

@@ -0,0 +1 @@
+bdb25f5fee117cb98afaca3393ac333cbd28e142814604dab24e02174adfc198

ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V114__activate_string_interning_for_parties.sql

@@ -0,0 +1,317 @@
+--  Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+--  SPDX-License-Identifier: Apache-2.0
+
+SELECT 'Party interning data migration - Preparation...';
+
+-- add temporary index to aid string lookups as migrating data
+CREATE UNIQUE INDEX string_interning_external_string_temp_idx ON string_interning USING btree (external_string);
+
+-- drop participant_events view (enables column type changes)
+DROP VIEW participant_events;
+
+-- add temporary migration function
+CREATE FUNCTION internalize_string_array(prefix TEXT, string_array TEXT[])
+RETURNS INTEGER[]
+LANGUAGE plpgsql
+AS
+$$
+DECLARE
+   result INTEGER[];
+BEGIN
+   EXECUTE
+   'SELECT CASE' ||
+   '  WHEN $2 IS NULL THEN NULL' ||
+   '  WHEN cardinality($2) = 0 THEN ARRAY[]::INTEGER[]' ||
+   '  ELSE (' ||
+   '    SELECT array_agg(string_interning.internal_id)' ||
+   '      FROM' ||
+   '        unnest($2) AS original(string_elem),' ||
+   '        string_interning' ||
+   '      WHERE $1 || original.string_elem = string_interning.external_string' ||
+   '  )' ||
+   'END'
+   INTO result
+   USING prefix, string_array;
+   ASSERT
+     (string_array IS NULL AND result IS NULL) OR cardinality(string_array) = cardinality(result),
+     'Could not internalize some elements in the array: [' || array_to_string(string_array, ', ') || ']';
+   RETURN result;
+END
+$$;
+
+
+SELECT 'Party interning data migration - Migrating party_entries...';
+
+-- add party_id to party_entries
+ALTER TABLE party_entries
+  ADD COLUMN party_id INTEGER;
+
+-- populate internal party id-s
+UPDATE party_entries
+  SET party_id = string_interning.internal_id
+FROM string_interning
+WHERE 'p|' || string_interning.external_string = party_entries.party;
+
+-- add index on party_id
+CREATE INDEX idx_party_entries_party_id_and_ledger_offset ON party_entries(party_id, ledger_offset);
+
+
+
+SELECT 'Party interning data migration - Migrating participant_command_completions...';
+
+ALTER TABLE participant_command_completions
+  ALTER COLUMN submitters TYPE INTEGER[]
+  USING internalize_string_array('p|', submitters);
+
+
+
+SELECT 'Party interning data migration - Migrating participant_events_divulgence...';
+
+DROP INDEX participant_events_divulgence_tree_event_witnesses_idx;
+
+ALTER TABLE participant_events_divulgence
+  ALTER COLUMN submitters TYPE INTEGER[]
+  USING internalize_string_array('p|', submitters);
+
+ALTER TABLE participant_events_divulgence
+  ALTER COLUMN tree_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_divulgence
+  ALTER COLUMN tree_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', tree_event_witnesses);
+
+
+
+SELECT 'Party interning data migration - Migrating participant_events_create...';
+
+DROP INDEX participant_events_create_flat_event_witnesses_idx;
+DROP INDEX participant_events_create_tree_event_witnesses_idx;
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN submitters TYPE INTEGER[]
+  USING internalize_string_array('p|', submitters);
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN tree_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN tree_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', tree_event_witnesses);
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN flat_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN flat_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', flat_event_witnesses);
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN create_signatories TYPE INTEGER[]
+  USING internalize_string_array('p|', create_signatories);
+
+ALTER TABLE participant_events_create
+  ALTER COLUMN create_observers TYPE INTEGER[]
+  USING internalize_string_array('p|', create_observers);
+
+
+
+SELECT 'Party interning data migration - Migrating participant_events_consuming_exercise...';
+
+DROP INDEX participant_events_consuming_exercise_flat_event_witnesses_idx;
+DROP INDEX participant_events_consuming_exercise_tree_event_witnesses_idx;
+
+ALTER TABLE participant_events_consuming_exercise
+  ALTER COLUMN submitters TYPE INTEGER[]
+  USING internalize_string_array('p|', submitters);
+
+ALTER TABLE participant_events_consuming_exercise
+  ALTER COLUMN tree_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_consuming_exercise
+  ALTER COLUMN tree_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', tree_event_witnesses);
+
+ALTER TABLE participant_events_consuming_exercise
+  ALTER COLUMN flat_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_consuming_exercise
+  ALTER COLUMN flat_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', flat_event_witnesses);
+
+ALTER TABLE participant_events_consuming_exercise
+  ALTER COLUMN exercise_actors TYPE INTEGER[]
+  USING internalize_string_array('p|', exercise_actors);
+
+
+
+SELECT 'Party interning data migration - Migrating participant_events_non_consuming_exercise...';
+
+DROP INDEX participant_events_non_consuming_exercise_flat_event_witnes_idx;
+DROP INDEX participant_events_non_consuming_exercise_tree_event_witnes_idx;
+
+ALTER TABLE participant_events_non_consuming_exercise
+  ALTER COLUMN submitters TYPE INTEGER[]
+  USING internalize_string_array('p|', submitters);
+
+ALTER TABLE participant_events_non_consuming_exercise
+  ALTER COLUMN tree_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_non_consuming_exercise
+  ALTER COLUMN tree_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', tree_event_witnesses);
+
+ALTER TABLE participant_events_non_consuming_exercise
+  ALTER COLUMN flat_event_witnesses DROP DEFAULT;
+
+ALTER TABLE participant_events_non_consuming_exercise
+  ALTER COLUMN flat_event_witnesses TYPE INTEGER[]
+  USING internalize_string_array('p|', flat_event_witnesses);
+
+ALTER TABLE participant_events_non_consuming_exercise
+  ALTER COLUMN exercise_actors TYPE INTEGER[]
+  USING internalize_string_array('p|', exercise_actors);
+
+
+
+SELECT 'Party interning data migration - Cleanup...';
+
+DROP INDEX string_interning_external_string_temp_idx;
+DROP FUNCTION internalize_string_array(TEXT, TEXT[]);
+
+CREATE VIEW participant_events
+AS
+SELECT
+    0::smallint as event_kind,
+    event_sequential_id,
+    NULL::text as event_offset,
+    NULL::text as transaction_id,
+    NULL::bigint as ledger_effective_time,
+    command_id,
+    workflow_id,
+    application_id,
+    submitters,
+    NULL::integer as node_index,
+    NULL::text as event_id,
+    contract_id,
+    template_id,
+    NULL::INTEGER[] as flat_event_witnesses,
+    tree_event_witnesses,
+    create_argument,
+    NULL::INTEGER[] as create_signatories,
+    NULL::INTEGER[] as create_observers,
+    NULL::text as create_agreement_text,
+    NULL::bytea as create_key_value,
+    NULL::text as create_key_hash,
+    NULL::text as exercise_choice,
+    NULL::bytea as exercise_argument,
+    NULL::bytea as exercise_result,
+    NULL::INTEGER[] as exercise_actors,
+    NULL::text[] as exercise_child_event_ids,
+    create_argument_compression,
+    NULL::smallint as create_key_value_compression,
+    NULL::smallint as exercise_argument_compression,
+    NULL::smallint as exercise_result_compression
+FROM participant_events_divulgence
+UNION ALL
+SELECT
+    10::smallint as event_kind,
+    event_sequential_id,
+    event_offset,
+    transaction_id,
+    ledger_effective_time,
+    command_id,
+    workflow_id,
+    application_id,
+    submitters,
+    node_index,
+    event_id,
+    contract_id,
+    template_id,
+    flat_event_witnesses,
+    tree_event_witnesses,
+    create_argument,
+    create_signatories,
+    create_observers,
+    create_agreement_text,
+    create_key_value,
+    create_key_hash,
+    NULL::text as exercise_choice,
+    NULL::bytea as exercise_argument,
+    NULL::bytea as exercise_result,
+    NULL::INTEGER[] as exercise_actors,
+    NULL::text[] as exercise_child_event_ids,
+    create_argument_compression,
+    create_key_value_compression,
+    NULL::smallint as exercise_argument_compression,
+    NULL::smallint as exercise_result_compression
+FROM participant_events_create
+UNION ALL
+SELECT
+    20::smallint as event_kind,
+    event_sequential_id,
+    event_offset,
+    transaction_id,
+    ledger_effective_time,
+    command_id,
+    workflow_id,
+    application_id,
+    submitters,
+    node_index,
+    event_id,
+    contract_id,
+    template_id,
+    flat_event_witnesses,
+    tree_event_witnesses,
+    NULL::bytea as create_argument,
+    NULL::INTEGER[] as create_signatories,
+    NULL::INTEGER[] as create_observers,
+    NULL::text as create_agreement_text,
+    create_key_value,
+    NULL::text as create_key_hash,
+    exercise_choice,
+    exercise_argument,
+    exercise_result,
+    exercise_actors,
+    exercise_child_event_ids,
+    NULL::smallint as create_argument_compression,
+    create_key_value_compression,
+    exercise_argument_compression,
+    exercise_result_compression
+FROM participant_events_consuming_exercise
+UNION ALL
+SELECT
+    25::smallint as event_kind,
+    event_sequential_id,
+    event_offset,
+    transaction_id,
+    ledger_effective_time,
+    command_id,
+    workflow_id,
+    application_id,
+    submitters,
+    node_index,
+    event_id,
+    contract_id,
+    template_id,
+    flat_event_witnesses,
+    tree_event_witnesses,
+    NULL::bytea as create_argument,
+    NULL::INTEGER[] as create_signatories,
+    NULL::INTEGER[] as create_observers,
+    NULL::text as create_agreement_text,
+    create_key_value,
+    NULL::text as create_key_hash,
+    exercise_choice,
+    exercise_argument,
+    exercise_result,
+    exercise_actors,
+    exercise_child_event_ids,
+    NULL::smallint as create_argument_compression,
+    create_key_value_compression,
+    exercise_argument_compression,
+    exercise_result_compression
+FROM participant_events_non_consuming_exercise;
+
+
+
+SELECT 'Party interning data migration - Done';