deploy(clickhouse): Drop Beacon_p2p_attestation signature (#273)

* deploy(clickhouse): Drop Beacon_p2p_attestation signature * deploy(clickhouse): Drop Beacon_p2p_attestation signature * deploy(clickhouse): Drop Beacon_p2p_attestation signature * deploy(clickhouse): Drop Beacon_p2p_attestation signature
ethpandaops · Jan 16, 2024 · ae207b3 · ae207b3
1 parent 7d9507e
commit ae207b3
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 4 deletions.
diff --git a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml
@@ -593,7 +593,6 @@ transforms:
         .attesting_validator_index = .meta.client.additional_data.attesting_validator.index
         .attesting_validator_committee_index = .meta.client.additional_data.attesting_validator.committee_index
       }
-      .signature = .data.signature
       .aggregation_bits = .data.aggregation_bits
       .beacon_block_root = .data.data.beacon_block_root
       .epoch = .meta.client.additional_data.epoch.number
@@ -1521,7 +1520,6 @@ transforms:
         .attesting_validator_index = .meta.client.additional_data.attesting_validator.index
         .attesting_validator_committee_index = .meta.client.additional_data.attesting_validator.committee_index
       }
-      .signature = .data.signature
       .aggregation_bits = .data.aggregation_bits
       .beacon_block_root = .data.data.beacon_block_root
       .epoch = .meta.client.additional_data.epoch.number
@@ -1693,7 +1691,7 @@ transforms:
         log(., level: "error", rate_limit_secs: 60)
       }
       .target_root = .data.data.target.root
-      key, err = .block_root + .block_slot + .position_in_block + .signature + .beacon_block_root + .slot + .committee_index + .source_root + .target_root
+      key, err = .block_root + .block_slot + .position_in_block + .beacon_block_root + .slot + .committee_index + .source_root + .target_root
       if err != null {
         .error = err
         .error_description = "failed to generate unique key"

diff --git a/deploy/migrations/clickhouse/020_imports.up.sql b/deploy/migrations/clickhouse/020_imports.up.sql
@@ -42,7 +42,7 @@ PARTITION BY toStartOfMonth(timestamp)
 ORDER BY (timestamp, unique_key, chain_id);
 
 ALTER TABLE default.mempool_dumpster_transaction_local ON CLUSTER '{cluster}'
-MODIFY COMMENT 'Contains transactions from mempool dumpster dataset. Following the parquet schema with some additions; https://github.com/flashbots/mempool-dumpster?tab=readme-ov-file#schema-of-output-files',
+MODIFY COMMENT 'Contains transactions from mempool dumpster dataset. Following the parquet schema with some additions: https://github.com/flashbots/mempool-dumpster?tab=readme-ov-file#schema-of-output-files',
 COMMENT COLUMN unique_key 'Unique key for the row, this is outside the source data and used for deduplication',
 COMMENT COLUMN updated_date_time 'When this row was last updated, this is outside the source data and used for deduplication',
 COMMENT COLUMN timestamp 'Timestamp of the transaction',

diff --git a/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.down.sql b/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.down.sql
@@ -0,0 +1,11 @@
+ALTER TABLE default.beacon_api_eth_v1_events_attestation_local ON CLUSTER '{cluster}'
+ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index;
+
+ALTER TABLE default.beacon_api_eth_v1_events_attestation ON CLUSTER '{cluster}'
+ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index;
+
+ALTER TABLE default.beacon_p2p_attestation_local ON CLUSTER '{cluster}'
+ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index;
+
+ALTER TABLE default.beacon_p2p_attestation ON CLUSTER '{cluster}'
+ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index;
diff --git a/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.up.sql b/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.up.sql
@@ -0,0 +1,12 @@
+ALTER TABLE default.beacon_api_eth_v1_events_attestation_local ON CLUSTER '{cluster}'
+DROP COLUMN signature;
+
+ALTER TABLE default.beacon_api_eth_v1_events_attestation ON CLUSTER '{cluster}'
+DROP COLUMN signature;
+
+
+ALTER TABLE default.beacon_p2p_attestation_local ON CLUSTER '{cluster}'
+DROP COLUMN signature;
+
+ALTER TABLE default.beacon_p2p_attestation ON CLUSTER '{cluster}'
+DROP COLUMN signature;