diff --git a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml index 9c2a4605..0fb56fb7 100644 --- a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml +++ b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml @@ -593,7 +593,6 @@ transforms: .attesting_validator_index = .meta.client.additional_data.attesting_validator.index .attesting_validator_committee_index = .meta.client.additional_data.attesting_validator.committee_index } - .signature = .data.signature .aggregation_bits = .data.aggregation_bits .beacon_block_root = .data.data.beacon_block_root .epoch = .meta.client.additional_data.epoch.number @@ -1521,7 +1520,6 @@ transforms: .attesting_validator_index = .meta.client.additional_data.attesting_validator.index .attesting_validator_committee_index = .meta.client.additional_data.attesting_validator.committee_index } - .signature = .data.signature .aggregation_bits = .data.aggregation_bits .beacon_block_root = .data.data.beacon_block_root .epoch = .meta.client.additional_data.epoch.number @@ -1693,7 +1691,7 @@ transforms: log(., level: "error", rate_limit_secs: 60) } .target_root = .data.data.target.root - key, err = .block_root + .block_slot + .position_in_block + .signature + .beacon_block_root + .slot + .committee_index + .source_root + .target_root + key, err = .block_root + .block_slot + .position_in_block + .beacon_block_root + .slot + .committee_index + .source_root + .target_root if err != null { .error = err .error_description = "failed to generate unique key" diff --git a/deploy/migrations/clickhouse/020_imports.up.sql b/deploy/migrations/clickhouse/020_imports.up.sql index 3db2f773..6727a99e 100644 --- a/deploy/migrations/clickhouse/020_imports.up.sql +++ b/deploy/migrations/clickhouse/020_imports.up.sql @@ -42,7 +42,7 @@ PARTITION BY toStartOfMonth(timestamp) ORDER BY (timestamp, unique_key, chain_id); ALTER TABLE default.mempool_dumpster_transaction_local ON CLUSTER '{cluster}' -MODIFY COMMENT 'Contains transactions from mempool dumpster dataset. Following the parquet schema with some additions; https://github.com/flashbots/mempool-dumpster?tab=readme-ov-file#schema-of-output-files', +MODIFY COMMENT 'Contains transactions from mempool dumpster dataset. Following the parquet schema with some additions: https://github.com/flashbots/mempool-dumpster?tab=readme-ov-file#schema-of-output-files', COMMENT COLUMN unique_key 'Unique key for the row, this is outside the source data and used for deduplication', COMMENT COLUMN updated_date_time 'When this row was last updated, this is outside the source data and used for deduplication', COMMENT COLUMN timestamp 'Timestamp of the transaction', diff --git a/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.down.sql b/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.down.sql new file mode 100644 index 00000000..79e942cc --- /dev/null +++ b/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.down.sql @@ -0,0 +1,11 @@ +ALTER TABLE default.beacon_api_eth_v1_events_attestation_local ON CLUSTER '{cluster}' +ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index; + +ALTER TABLE default.beacon_api_eth_v1_events_attestation ON CLUSTER '{cluster}' +ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index; + +ALTER TABLE default.beacon_p2p_attestation_local ON CLUSTER '{cluster}' +ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index; + +ALTER TABLE default.beacon_p2p_attestation ON CLUSTER '{cluster}' +ADD COLUMN signature String Codec(ZSTD(1)) AFTER attesting_validator_committee_index; diff --git a/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.up.sql b/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.up.sql new file mode 100644 index 00000000..dd5c80c9 --- /dev/null +++ b/deploy/migrations/clickhouse/025_rm_beacon_attestation_signature.up.sql @@ -0,0 +1,12 @@ +ALTER TABLE default.beacon_api_eth_v1_events_attestation_local ON CLUSTER '{cluster}' +DROP COLUMN signature; + +ALTER TABLE default.beacon_api_eth_v1_events_attestation ON CLUSTER '{cluster}' +DROP COLUMN signature; + + +ALTER TABLE default.beacon_p2p_attestation_local ON CLUSTER '{cluster}' +DROP COLUMN signature; + +ALTER TABLE default.beacon_p2p_attestation ON CLUSTER '{cluster}' +DROP COLUMN signature;