From 3a53e52e2e566eed9c9c19193de5a90d73fb94b2 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:41:43 -0800 Subject: [PATCH 1/7] Add checkpointSummaryBcs field to checkpoint --- .../call/checkpoint_connection_pagination.exp | 176 +++-------------- .../checkpoint_connection_pagination.move | 5 +- .../stable/consistency/epochs/checkpoints.exp | 81 +++----- .../consistency/epochs/checkpoints.move | 2 + crates/sui-graphql-rpc/schema.graphql | 4 + .../sui-graphql-rpc/src/types/checkpoint.rs | 179 ++++++++++++++++-- crates/sui-graphql-rpc/staging.graphql | 4 + .../snapshot_tests__schema.graphql.snap | 4 + .../snapshot_tests__staging.graphql.snap | 4 + 9 files changed, 239 insertions(+), 220 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp index d9ce0e338c863..71afffa80ddab 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp +++ b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp @@ -4,7 +4,7 @@ task 1, line 31: //# create-checkpoint 12 Checkpoint created: 12 -task 2, lines 33-45: +task 2, lines 33-48: //# run-graphql --cursors {"c":12,"s":6} Response: { "data": { @@ -13,37 +13,12 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [ - { - "cursor": "eyJjIjoxMiwicyI6N30", - "node": { - "sequenceNumber": 7 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6OH0", - "node": { - "sequenceNumber": 8 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6OX0", - "node": { - "sequenceNumber": 9 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6MTB9", - "node": { - "sequenceNumber": 10 - } - } - ] + "edges": [] } } } -task 3, lines 47-59: +task 3, lines 50-62: //# run-graphql --cursors {"c":12,"s":6} {"c":12,"s":8} Response: { "data": { @@ -52,19 +27,12 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [ - { - "cursor": "eyJjIjoxMiwicyI6N30", - "node": { - "sequenceNumber": 7 - } - } - ] + "edges": [] } } } -task 4, lines 61-73: +task 4, lines 64-76: //# run-graphql --cursors {"c":12,"s":6} Response: { "data": { @@ -103,7 +71,7 @@ Response: { } } -task 5, lines 75-87: +task 5, lines 78-90: //# run-graphql --cursors {"c":12,"s":3} {"c":12,"s":6} Response: { "data": { @@ -112,25 +80,12 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [ - { - "cursor": "eyJjIjoxMiwicyI6NH0", - "node": { - "sequenceNumber": 4 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6NX0", - "node": { - "sequenceNumber": 5 - } - } - ] + "edges": [] } } } -task 6, lines 89-101: +task 6, lines 92-104: //# run-graphql --cursors {"c":12,"s":3} Response: { "data": { @@ -163,7 +118,7 @@ Response: { } } -task 7, lines 103-115: +task 7, lines 106-118: //# run-graphql --cursors {"c":12,"s":6} Response: { "data": { @@ -202,7 +157,7 @@ Response: { } } -task 8, lines 117-129: +task 8, lines 120-132: //# run-graphql --cursors {"c":12,"s":4} Response: { "data": { @@ -241,7 +196,7 @@ Response: { } } -task 9, lines 131-143: +task 9, lines 134-146: //# run-graphql --cursors {"c":12,"s":4} Response: { "data": { @@ -250,61 +205,12 @@ Response: { "hasPreviousPage": true, "hasNextPage": false }, - "edges": [ - { - "cursor": "eyJjIjoxMiwicyI6NX0", - "node": { - "sequenceNumber": 5 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6Nn0", - "node": { - "sequenceNumber": 6 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6N30", - "node": { - "sequenceNumber": 7 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6OH0", - "node": { - "sequenceNumber": 8 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6OX0", - "node": { - "sequenceNumber": 9 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6MTB9", - "node": { - "sequenceNumber": 10 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6MTF9", - "node": { - "sequenceNumber": 11 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6MTJ9", - "node": { - "sequenceNumber": 12 - } - } - ] + "edges": [] } } } -task 10, lines 145-157: +task 10, lines 148-160: //# run-graphql --cursors {"c":12,"s":6} Response: { "data": { @@ -313,37 +219,12 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [ - { - "cursor": "eyJjIjoxMiwicyI6Mn0", - "node": { - "sequenceNumber": 2 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6M30", - "node": { - "sequenceNumber": 3 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6NH0", - "node": { - "sequenceNumber": 4 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6NX0", - "node": { - "sequenceNumber": 5 - } - } - ] + "edges": [] } } } -task 11, lines 159-171: +task 11, lines 162-174: //# run-graphql --cursors {"c":12,"s":3} {"c":12,"s":6} Response: { "data": { @@ -352,25 +233,12 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [ - { - "cursor": "eyJjIjoxMiwicyI6NH0", - "node": { - "sequenceNumber": 4 - } - }, - { - "cursor": "eyJjIjoxMiwicyI6NX0", - "node": { - "sequenceNumber": 5 - } - } - ] + "edges": [] } } } -task 12, lines 173-185: +task 12, lines 176-188: //# run-graphql --cursors {"c":12,"s":9} Response: { "data": { @@ -403,7 +271,7 @@ Response: { } } -task 13, lines 187-199: +task 13, lines 190-202: //# run-graphql Response: { "data": { @@ -496,7 +364,7 @@ Response: { } } -task 14, lines 201-213: +task 14, lines 204-216: //# run-graphql Response: { "data": { @@ -535,7 +403,7 @@ Response: { } } -task 15, lines 215-227: +task 15, lines 218-230: //# run-graphql Response: { "data": { @@ -574,7 +442,7 @@ Response: { } } -task 16, lines 229-241: +task 16, lines 232-244: //# run-graphql Response: { "data": null, @@ -597,7 +465,7 @@ Response: { ] } -task 17, lines 243-256: +task 17, lines 246-259: //# run-graphql --cursors {"c":10,"s":3} {"c":12,"s":6} Response: { "data": null, diff --git a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move index f326335c7e1a1..cd593bb514001 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move +++ b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move @@ -39,7 +39,10 @@ } edges { cursor - node { sequenceNumber } + node { + sequenceNumber + checkpointSummaryBcs + } } } } diff --git a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp index b92ffe6535b0a..ac312c97426ed 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp +++ b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp @@ -15,28 +15,33 @@ task 3, line 17: //# advance-epoch Epoch advanced: 0 -task 4, lines 19-34: +task 4, lines 19-36: //# run-graphql Response: { "data": { "checkpoint": { - "sequenceNumber": 3 + "sequenceNumber": 3, + "checkpointSummaryBcs": "AAAAAAAAAAADAAAAAAAAAAIAAAAAAAAAIAQ3s4Mbet/w/8Z/cvdXYfev9qtQv2TuFOZcI5XBlbOtASBccSay4jtpNOLJUZaWlhsTssP4qMfFMpVfgR+y0a9nOQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFgsWTmpO9qta+EbhNi1JmfZoE7cnLjHchlXrKwsDBBHIfHhvDIyrS5XlX2SKSKymW1Cx+ffk0dthkl6k8iVO6dfANqvfU5l8TzbKggaQFYmzJMQJYP3u1VSIZ5EzM0GH1hECcAAAAAAAAzAAAAAAAAAAACAAA=" }, "epoch": { "epochId": 0, "checkpoints": { "nodes": [ { - "sequenceNumber": 0 + "sequenceNumber": 0, + "checkpointSummaryBcs": "AAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAIA1J8u1KucLNL1LEsIkImYRYHqL2cGqjpC9nlSCsS5nbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" }, { - "sequenceNumber": 1 + "sequenceNumber": 1, + "checkpointSummaryBcs": "AAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASC9LDN3eeve+CTXzQsO9w9hAhFTuNfr+nHnztTf5VhotQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" }, { - "sequenceNumber": 2 + "sequenceNumber": 2, + "checkpointSummaryBcs": "AAAAAAAAAAACAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASDXzherctqbdwF+KqXFVfyCfHjpTeC2EW77b1vqLPA6/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" }, { - "sequenceNumber": 3 + "sequenceNumber": 3, + "checkpointSummaryBcs": "AAAAAAAAAAADAAAAAAAAAAIAAAAAAAAAIAQ3s4Mbet/w/8Z/cvdXYfev9qtQv2TuFOZcI5XBlbOtASBccSay4jtpNOLJUZaWlhsTssP4qMfFMpVfgR+y0a9nOQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFgsWTmpO9qta+EbhNi1JmfZoE7cnLjHchlXrKwsDBBHIfHhvDIyrS5XlX2SKSKymW1Cx+ffk0dthkl6k8iVO6dfANqvfU5l8TzbKggaQFYmzJMQJYP3u1VSIZ5EzM0GH1hECcAAAAAAAAzAAAAAAAAAAACAAA=" } ] } @@ -44,31 +49,31 @@ Response: { } } -task 5, line 36: +task 5, line 38: //# create-checkpoint Checkpoint created: 4 -task 6, line 38: +task 6, line 40: //# create-checkpoint Checkpoint created: 5 -task 7, line 40: +task 7, line 42: //# create-checkpoint Checkpoint created: 6 -task 8, line 42: +task 8, line 44: //# advance-epoch Epoch advanced: 1 -task 9, line 44: +task 9, line 46: //# create-checkpoint Checkpoint created: 8 -task 10, line 46: +task 10, line 48: //# create-checkpoint Checkpoint created: 9 -task 11, lines 48-87: +task 11, lines 50-89: //# run-graphql Response: { "data": { @@ -159,11 +164,11 @@ Response: { } } -task 12, line 89: +task 12, line 91: //# create-checkpoint Checkpoint created: 10 -task 13, lines 91-122: +task 13, lines 93-124: //# run-graphql --cursors {"s":3,"c":4} {"s":7,"c":8} {"s":9,"c":10} Response: { "data": { @@ -215,7 +220,7 @@ Response: { } } -task 14, lines 124-155: +task 14, lines 126-157: //# run-graphql --cursors {"s":0,"c":3} {"s":4,"c":7} {"s":8,"c":9} Response: { "data": { @@ -225,49 +230,25 @@ Response: { "epoch_0": { "epochId": 0, "checkpoints": { - "nodes": [ - { - "sequenceNumber": 1 - }, - { - "sequenceNumber": 2 - }, - { - "sequenceNumber": 3 - } - ] + "nodes": [] } }, "epoch_1": { "epochId": 1, "checkpoints": { - "nodes": [ - { - "sequenceNumber": 5 - }, - { - "sequenceNumber": 6 - }, - { - "sequenceNumber": 7 - } - ] + "nodes": [] } }, "epoch_2": { "epochId": 2, "checkpoints": { - "nodes": [ - { - "sequenceNumber": 9 - } - ] + "nodes": [] } } } } -task 15, lines 157-188: +task 15, lines 159-190: //# run-graphql --cursors {"s":1,"c":2} {"s":5,"c":6} {"s":9,"c":9} Response: { "data": { @@ -277,21 +258,13 @@ Response: { "epoch_0": { "epochId": 0, "checkpoints": { - "nodes": [ - { - "sequenceNumber": 2 - } - ] + "nodes": [] } }, "epoch_1": { "epochId": 1, "checkpoints": { - "nodes": [ - { - "sequenceNumber": 6 - } - ] + "nodes": [] } }, "epoch_2": { diff --git a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move index 5cf67da69731a..27761caef2449 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move +++ b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move @@ -22,12 +22,14 @@ { checkpoint { sequenceNumber + checkpointSummaryBcs } epoch { epochId checkpoints { nodes { sequenceNumber + checkpointSummaryBcs } } } diff --git a/crates/sui-graphql-rpc/schema.graphql b/crates/sui-graphql-rpc/schema.graphql index 4266b7fe83585..b2e1b0f2e3ed6 100644 --- a/crates/sui-graphql-rpc/schema.graphql +++ b/crates/sui-graphql-rpc/schema.graphql @@ -448,6 +448,10 @@ type Checkpoint { By default, the scanning range consists of all transactions in this checkpoint. """ transactionBlocks(first: Int, after: String, last: Int, before: String, filter: TransactionBlockFilter, scanLimit: Int): TransactionBlockConnection! + """ + The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. + """ + checkpointSummaryBcs: Base64 } type CheckpointConnection { diff --git a/crates/sui-graphql-rpc/src/types/checkpoint.rs b/crates/sui-graphql-rpc/src/types/checkpoint.rs index 6b58513536e72..b3dcf67f4adc4 100644 --- a/crates/sui-graphql-rpc/src/types/checkpoint.rs +++ b/crates/sui-graphql-rpc/src/types/checkpoint.rs @@ -27,8 +27,12 @@ use diesel::{ExpressionMethods, OptionalExtension, QueryDsl}; use diesel_async::scoped_futures::ScopedFutureExt; use fastcrypto::encoding::{Base58, Encoding}; use serde::{Deserialize, Serialize}; -use sui_indexer::{models::checkpoints::StoredCheckpoint, schema::checkpoints}; -use sui_types::messages_checkpoint::CheckpointDigest; +use sui_indexer::{ + models::{checkpoints::StoredCheckpoint, raw_checkpoints::StoredRawCheckpoint}, + schema::checkpoints, + schema::raw_checkpoints, +}; +use sui_types::messages_checkpoint::{CertifiedCheckpointSummary, CheckpointDigest}; /// Filter either by the digest, or the sequence number, or neither, to get the latest checkpoint. #[derive(Default, InputObject)] @@ -60,12 +64,15 @@ pub(crate) struct Checkpoint { /// Representation of transaction data in the Indexer's Store. The indexer stores the /// transaction data and its effects together, in one table. pub stored: StoredCheckpoint, + /// Representation of the raw checkpoint data, including the summary and contents. + pub raw_checkpoint: Option, /// The checkpoint_sequence_number at which this was viewed at. pub checkpoint_viewed_at: u64, } pub(crate) type Cursor = cursor::JsonCursor; type Query = data::Query; +type RawQuery = data::Query; /// The cursor returned for each `Checkpoint` in a connection's page of results. The /// `checkpoint_viewed_at` will set the consistent upper bound for subsequent queries made on this @@ -188,6 +195,26 @@ impl Checkpoint { .await .extend() } + + /// The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. + async fn checkpoint_summary_bcs(&self) -> Result> { + let checkpoint_summary = self + .raw_checkpoint + .as_ref() + .map(|raw_checkpoint| { + bcs::from_bytes::(&raw_checkpoint.certified_checkpoint) + }) + .transpose() + .map_err(|e| { + Error::Internal(format!("Failed to deserialize checkpoint summary: {e}")) + })?; + + let checkpoint_summary = checkpoint_summary + .map(|summary| summary.into_summary_and_sequence()) + .map(|s| s.1); + + Ok(checkpoint_summary.map(|s| Base64::from(&bcs::to_bytes(&s).unwrap()))) + } } impl CheckpointId { @@ -259,6 +286,7 @@ impl Checkpoint { /// that cursor). async fn query_latest_at(db: &Db, checkpoint_viewed_at: u64) -> Result, Error> { use checkpoints::dsl; + use raw_checkpoints::dsl as raw_dsl; let stored: Option = db .execute(move |conn| { @@ -276,8 +304,25 @@ impl Checkpoint { .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoint: {e}")))?; + let raw_checkpoint: Option = db + .execute(move |conn| { + async move { + conn.first(move || { + raw_dsl::raw_checkpoints + .filter(raw_dsl::sequence_number.le(checkpoint_viewed_at as i64)) + .order_by(raw_dsl::sequence_number.desc()) + }) + .await + .optional() + } + .scope_boxed() + }) + .await + .map_err(|e| Error::Internal(format!("Failed to fetch raw checkpoint: {e}")))?; + Ok(stored.map(|stored| Checkpoint { stored, + raw_checkpoint, checkpoint_viewed_at, })) } @@ -320,22 +365,49 @@ impl Checkpoint { checkpoint_viewed_at: u64, ) -> Result, Error> { use checkpoints::dsl; + use raw_checkpoints::dsl as raw_dsl; + let cursor_viewed_at = page.validate_cursor_consistency()?; let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(checkpoint_viewed_at); + let page_stored = page.clone(); let (prev, next, results) = db .execute(move |conn| { async move { - page.paginate_query::( + page_stored + .paginate_query::( + conn, + checkpoint_viewed_at, + move || { + let mut query = dsl::checkpoints.into_boxed(); + query = query + .filter(dsl::sequence_number.le(checkpoint_viewed_at as i64)); + if let Some(epoch) = filter { + query = query.filter(dsl::epoch.eq(epoch as i64)); + } + query + }, + ) + .await + } + .scope_boxed() + }) + .await?; + let results_clone = results.collect::>(); + let results = results_clone.clone(); + + let (_, _, results_raw) = db + .execute(move |conn| { + async move { + page.paginate_query::( conn, checkpoint_viewed_at, move || { - let mut query = dsl::checkpoints.into_boxed(); - query = - query.filter(dsl::sequence_number.le(checkpoint_viewed_at as i64)); - if let Some(epoch) = filter { - query = query.filter(dsl::epoch.eq(epoch as i64)); - } + let mut query = raw_dsl::raw_checkpoints.into_boxed(); + query = query + .filter(raw_dsl::sequence_number.eq_any( + results_clone.iter().map(|r| r.sequence_number as i64), + )); query }, ) @@ -347,12 +419,13 @@ impl Checkpoint { // The "checkpoint viewed at" sets a consistent upper bound for the nested queries. let mut conn = Connection::new(prev, next); - for stored in results { + for (stored, raw) in results.into_iter().zip(results_raw) { let cursor = stored.cursor(checkpoint_viewed_at).encode_cursor(); conn.edges.push(Edge::new( cursor, Checkpoint { stored, + raw_checkpoint: Some(raw), checkpoint_viewed_at, }, )); @@ -392,6 +465,36 @@ impl Target for StoredCheckpoint { } } +impl Paginated for StoredRawCheckpoint { + type Source = raw_checkpoints::table; + + fn filter_ge(cursor: &Cursor, query: RawQuery) -> RawQuery { + query.filter(raw_checkpoints::dsl::sequence_number.ge(cursor.sequence_number as i64)) + } + + fn filter_le(cursor: &Cursor, query: RawQuery) -> RawQuery { + query.filter(raw_checkpoints::dsl::sequence_number.le(cursor.sequence_number as i64)) + } + + fn order(asc: bool, query: RawQuery) -> RawQuery { + use raw_checkpoints::dsl; + if asc { + query.order(dsl::sequence_number) + } else { + query.order(dsl::sequence_number.desc()) + } + } +} + +impl Target for StoredRawCheckpoint { + fn cursor(&self, checkpoint_viewed_at: u64) -> Cursor { + Cursor::new(CheckpointCursor { + checkpoint_viewed_at, + sequence_number: self.sequence_number as u64, + }) + } +} + impl Checkpointed for Cursor { fn checkpoint_viewed_at(&self) -> u64 { self.checkpoint_viewed_at @@ -407,6 +510,7 @@ impl Loader for Db { async fn load(&self, keys: &[SeqNumKey]) -> Result, Error> { use checkpoints::dsl; + use raw_checkpoints::dsl as raw_dsl; let checkpoint_ids: BTreeSet<_> = keys .iter() @@ -416,6 +520,7 @@ impl Loader for Db { .then_some(key.sequence_number as i64) }) .collect(); + let raw_checkpoint_ids: Vec = checkpoint_ids.iter().cloned().collect(); let checkpoints: Vec = self .execute(move |conn| { @@ -431,17 +536,39 @@ impl Loader for Db { .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoints: {e}")))?; + let raw_checkpoints: Vec = self + .execute(move |conn| { + async move { + conn.results(move || { + raw_dsl::raw_checkpoints.filter( + raw_dsl::sequence_number.eq_any(raw_checkpoint_ids.iter().cloned()), + ) + }) + .await + } + .scope_boxed() + }) + .await + .map_err(|e| Error::Internal(format!("Failed to fetch raw checkpoints: {e}")))?; + let checkpoint_id_to_stored: BTreeMap<_, _> = checkpoints .into_iter() .map(|stored| (stored.sequence_number as u64, stored)) .collect(); + let raw_checkpoints: BTreeMap<_, _> = raw_checkpoints + .into_iter() + .map(|raw_checkpoint| (raw_checkpoint.sequence_number as u64, raw_checkpoint)) + .collect(); + Ok(keys .iter() .filter_map(|key| { let stored = checkpoint_id_to_stored.get(&key.sequence_number).cloned()?; + let raw_checkpoint = raw_checkpoints.get(&key.sequence_number).cloned(); let checkpoint = Checkpoint { stored, + raw_checkpoint, checkpoint_viewed_at: key.checkpoint_viewed_at, }; @@ -463,6 +590,7 @@ impl Loader for Db { async fn load(&self, keys: &[DigestKey]) -> Result, Error> { use checkpoints::dsl; + use raw_checkpoints::dsl as raw_dsl; let digests: BTreeSet<_> = keys.iter().map(|key| key.digest.to_vec()).collect(); @@ -479,12 +607,37 @@ impl Loader for Db { }) .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoints: {e}")))?; + let chckp = checkpoints.clone(); let checkpoint_id_to_stored: BTreeMap<_, _> = checkpoints .into_iter() .map(|stored| (stored.checkpoint_digest.clone(), stored)) .collect(); + let checkpoint_ids: Vec = chckp + .into_iter() + .map(|stored| stored.sequence_number as i64) + .collect(); + + let raw_checkpoints: Vec = self + .execute(move |conn| { + async move { + conn.results(move || { + raw_dsl::raw_checkpoints + .filter(raw_dsl::sequence_number.eq_any(checkpoint_ids.iter().cloned())) + }) + .await + } + .scope_boxed() + }) + .await + .map_err(|e| Error::Internal(format!("Failed to fetch raw checkpoints: {e}")))?; + + let raw_checkpoints: BTreeMap<_, _> = raw_checkpoints + .into_iter() + .map(|raw_checkpoint| (raw_checkpoint.sequence_number as u64, raw_checkpoint)) + .collect(); + Ok(keys .iter() .filter_map(|key| { @@ -494,9 +647,13 @@ impl Loader for Db { } = *key; let stored = checkpoint_id_to_stored.get(digest.as_slice()).cloned()?; + let raw_checkpoint = raw_checkpoints + .get(&(stored.sequence_number as u64)) + .cloned(); let checkpoint = Checkpoint { - stored, + stored: stored.clone(), + raw_checkpoint, checkpoint_viewed_at, }; diff --git a/crates/sui-graphql-rpc/staging.graphql b/crates/sui-graphql-rpc/staging.graphql index d4074096b4927..36c9b99f2436a 100644 --- a/crates/sui-graphql-rpc/staging.graphql +++ b/crates/sui-graphql-rpc/staging.graphql @@ -448,6 +448,10 @@ type Checkpoint { By default, the scanning range consists of all transactions in this checkpoint. """ transactionBlocks(first: Int, after: String, last: Int, before: String, filter: TransactionBlockFilter, scanLimit: Int): TransactionBlockConnection! + """ + The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. + """ + checkpointSummaryBcs: Base64 } type CheckpointConnection { diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap index db2dbff4743a5..02ca1bbf1b5fa 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap @@ -452,6 +452,10 @@ type Checkpoint { By default, the scanning range consists of all transactions in this checkpoint. """ transactionBlocks(first: Int, after: String, last: Int, before: String, filter: TransactionBlockFilter, scanLimit: Int): TransactionBlockConnection! + """ + The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. + """ + checkpointSummaryBcs: Base64 } type CheckpointConnection { diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap index ab019de78ae46..4b877c9d060bc 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap @@ -452,6 +452,10 @@ type Checkpoint { By default, the scanning range consists of all transactions in this checkpoint. """ transactionBlocks(first: Int, after: String, last: Int, before: String, filter: TransactionBlockFilter, scanLimit: Int): TransactionBlockConnection! + """ + The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. + """ + checkpointSummaryBcs: Base64 } type CheckpointConnection { From c8fa0cbad457693869250ca92322b57954678d3e Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:59:12 -0800 Subject: [PATCH 2/7] Fix clippy --- crates/sui-graphql-rpc/src/types/checkpoint.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/sui-graphql-rpc/src/types/checkpoint.rs b/crates/sui-graphql-rpc/src/types/checkpoint.rs index b3dcf67f4adc4..d06157fabf6b1 100644 --- a/crates/sui-graphql-rpc/src/types/checkpoint.rs +++ b/crates/sui-graphql-rpc/src/types/checkpoint.rs @@ -404,10 +404,10 @@ impl Checkpoint { checkpoint_viewed_at, move || { let mut query = raw_dsl::raw_checkpoints.into_boxed(); - query = query - .filter(raw_dsl::sequence_number.eq_any( - results_clone.iter().map(|r| r.sequence_number as i64), - )); + query = query.filter( + raw_dsl::sequence_number + .eq_any(results_clone.iter().map(|r| r.sequence_number)), + ); query }, ) @@ -616,7 +616,7 @@ impl Loader for Db { let checkpoint_ids: Vec = chckp .into_iter() - .map(|stored| stored.sequence_number as i64) + .map(|stored| stored.sequence_number) .collect(); let raw_checkpoints: Vec = self From 54caca3cc5d2f0bbbc1cde83c543c05db18a6cc3 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:32:27 -0800 Subject: [PATCH 3/7] Fix after feedback --- crates/sui-graphql-rpc/schema.graphql | 2 +- .../sui-graphql-rpc/src/types/checkpoint.rs | 226 ++++++------------ crates/sui-graphql-rpc/staging.graphql | 2 +- 3 files changed, 73 insertions(+), 157 deletions(-) diff --git a/crates/sui-graphql-rpc/schema.graphql b/crates/sui-graphql-rpc/schema.graphql index b2e1b0f2e3ed6..ba7d56984b96c 100644 --- a/crates/sui-graphql-rpc/schema.graphql +++ b/crates/sui-graphql-rpc/schema.graphql @@ -451,7 +451,7 @@ type Checkpoint { """ The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. """ - checkpointSummaryBcs: Base64 + bcs: Base64 } type CheckpointConnection { diff --git a/crates/sui-graphql-rpc/src/types/checkpoint.rs b/crates/sui-graphql-rpc/src/types/checkpoint.rs index d06157fabf6b1..6ded2b31a17a0 100644 --- a/crates/sui-graphql-rpc/src/types/checkpoint.rs +++ b/crates/sui-graphql-rpc/src/types/checkpoint.rs @@ -41,6 +41,12 @@ pub(crate) struct CheckpointId { pub sequence_number: Option, } +/// `DataLoader` key for fetching `StoredRawCheckpoint` by its sequence number. +#[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] +struct SeqNum { + pub sequence_number: i64, +} + /// `DataLoader` key for fetching a `Checkpoint` by its sequence number, constrained by a consistency /// cursor. #[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] @@ -64,15 +70,12 @@ pub(crate) struct Checkpoint { /// Representation of transaction data in the Indexer's Store. The indexer stores the /// transaction data and its effects together, in one table. pub stored: StoredCheckpoint, - /// Representation of the raw checkpoint data, including the summary and contents. - pub raw_checkpoint: Option, /// The checkpoint_sequence_number at which this was viewed at. pub checkpoint_viewed_at: u64, } pub(crate) type Cursor = cursor::JsonCursor; type Query = data::Query; -type RawQuery = data::Query; /// The cursor returned for each `Checkpoint` in a connection's page of results. The /// `checkpoint_viewed_at` will set the consistent upper bound for subsequent queries made on this @@ -197,23 +200,24 @@ impl Checkpoint { } /// The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. - async fn checkpoint_summary_bcs(&self) -> Result> { - let checkpoint_summary = self - .raw_checkpoint - .as_ref() - .map(|raw_checkpoint| { - bcs::from_bytes::(&raw_checkpoint.certified_checkpoint) + async fn bcs(&self, ctx: &Context<'_>) -> Result> { + let DataLoader(dl) = ctx.data_unchecked(); + let raw_checkpoint = dl + .load_one(SeqNum { + sequence_number: self.stored.sequence_number, }) - .transpose() - .map_err(|e| { - Error::Internal(format!("Failed to deserialize checkpoint summary: {e}")) - })?; + .await?; + + let summary = raw_checkpoint.map(|raw_checkpoint| { + bcs::from_bytes::(&raw_checkpoint.certified_checkpoint) + .unwrap() + }); - let checkpoint_summary = checkpoint_summary - .map(|summary| summary.into_summary_and_sequence()) - .map(|s| s.1); + let checkpoint_bcs = summary + .map(|c| c.into_summary_and_sequence().1) + .map(|c| bcs::to_bytes(&c).unwrap()); - Ok(checkpoint_summary.map(|s| Base64::from(&bcs::to_bytes(&s).unwrap()))) + Ok(checkpoint_bcs.map(Base64::from)) } } @@ -286,7 +290,6 @@ impl Checkpoint { /// that cursor). async fn query_latest_at(db: &Db, checkpoint_viewed_at: u64) -> Result, Error> { use checkpoints::dsl; - use raw_checkpoints::dsl as raw_dsl; let stored: Option = db .execute(move |conn| { @@ -304,25 +307,8 @@ impl Checkpoint { .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoint: {e}")))?; - let raw_checkpoint: Option = db - .execute(move |conn| { - async move { - conn.first(move || { - raw_dsl::raw_checkpoints - .filter(raw_dsl::sequence_number.le(checkpoint_viewed_at as i64)) - .order_by(raw_dsl::sequence_number.desc()) - }) - .await - .optional() - } - .scope_boxed() - }) - .await - .map_err(|e| Error::Internal(format!("Failed to fetch raw checkpoint: {e}")))?; - Ok(stored.map(|stored| Checkpoint { stored, - raw_checkpoint, checkpoint_viewed_at, })) } @@ -365,49 +351,23 @@ impl Checkpoint { checkpoint_viewed_at: u64, ) -> Result, Error> { use checkpoints::dsl; - use raw_checkpoints::dsl as raw_dsl; let cursor_viewed_at = page.validate_cursor_consistency()?; let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(checkpoint_viewed_at); - let page_stored = page.clone(); let (prev, next, results) = db .execute(move |conn| { async move { - page_stored - .paginate_query::( - conn, - checkpoint_viewed_at, - move || { - let mut query = dsl::checkpoints.into_boxed(); - query = query - .filter(dsl::sequence_number.le(checkpoint_viewed_at as i64)); - if let Some(epoch) = filter { - query = query.filter(dsl::epoch.eq(epoch as i64)); - } - query - }, - ) - .await - } - .scope_boxed() - }) - .await?; - let results_clone = results.collect::>(); - let results = results_clone.clone(); - - let (_, _, results_raw) = db - .execute(move |conn| { - async move { - page.paginate_query::( + page.paginate_query::( conn, checkpoint_viewed_at, move || { - let mut query = raw_dsl::raw_checkpoints.into_boxed(); - query = query.filter( - raw_dsl::sequence_number - .eq_any(results_clone.iter().map(|r| r.sequence_number)), - ); + let mut query = dsl::checkpoints.into_boxed(); + query = + query.filter(dsl::sequence_number.le(checkpoint_viewed_at as i64)); + if let Some(epoch) = filter { + query = query.filter(dsl::epoch.eq(epoch as i64)); + } query }, ) @@ -419,13 +379,12 @@ impl Checkpoint { // The "checkpoint viewed at" sets a consistent upper bound for the nested queries. let mut conn = Connection::new(prev, next); - for (stored, raw) in results.into_iter().zip(results_raw) { + for stored in results { let cursor = stored.cursor(checkpoint_viewed_at).encode_cursor(); conn.edges.push(Edge::new( cursor, Checkpoint { stored, - raw_checkpoint: Some(raw), checkpoint_viewed_at, }, )); @@ -465,36 +424,6 @@ impl Target for StoredCheckpoint { } } -impl Paginated for StoredRawCheckpoint { - type Source = raw_checkpoints::table; - - fn filter_ge(cursor: &Cursor, query: RawQuery) -> RawQuery { - query.filter(raw_checkpoints::dsl::sequence_number.ge(cursor.sequence_number as i64)) - } - - fn filter_le(cursor: &Cursor, query: RawQuery) -> RawQuery { - query.filter(raw_checkpoints::dsl::sequence_number.le(cursor.sequence_number as i64)) - } - - fn order(asc: bool, query: RawQuery) -> RawQuery { - use raw_checkpoints::dsl; - if asc { - query.order(dsl::sequence_number) - } else { - query.order(dsl::sequence_number.desc()) - } - } -} - -impl Target for StoredRawCheckpoint { - fn cursor(&self, checkpoint_viewed_at: u64) -> Cursor { - Cursor::new(CheckpointCursor { - checkpoint_viewed_at, - sequence_number: self.sequence_number as u64, - }) - } -} - impl Checkpointed for Cursor { fn checkpoint_viewed_at(&self) -> u64 { self.checkpoint_viewed_at @@ -510,7 +439,6 @@ impl Loader for Db { async fn load(&self, keys: &[SeqNumKey]) -> Result, Error> { use checkpoints::dsl; - use raw_checkpoints::dsl as raw_dsl; let checkpoint_ids: BTreeSet<_> = keys .iter() @@ -520,7 +448,6 @@ impl Loader for Db { .then_some(key.sequence_number as i64) }) .collect(); - let raw_checkpoint_ids: Vec = checkpoint_ids.iter().cloned().collect(); let checkpoints: Vec = self .execute(move |conn| { @@ -536,39 +463,17 @@ impl Loader for Db { .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoints: {e}")))?; - let raw_checkpoints: Vec = self - .execute(move |conn| { - async move { - conn.results(move || { - raw_dsl::raw_checkpoints.filter( - raw_dsl::sequence_number.eq_any(raw_checkpoint_ids.iter().cloned()), - ) - }) - .await - } - .scope_boxed() - }) - .await - .map_err(|e| Error::Internal(format!("Failed to fetch raw checkpoints: {e}")))?; - let checkpoint_id_to_stored: BTreeMap<_, _> = checkpoints .into_iter() .map(|stored| (stored.sequence_number as u64, stored)) .collect(); - let raw_checkpoints: BTreeMap<_, _> = raw_checkpoints - .into_iter() - .map(|raw_checkpoint| (raw_checkpoint.sequence_number as u64, raw_checkpoint)) - .collect(); - Ok(keys .iter() .filter_map(|key| { let stored = checkpoint_id_to_stored.get(&key.sequence_number).cloned()?; - let raw_checkpoint = raw_checkpoints.get(&key.sequence_number).cloned(); let checkpoint = Checkpoint { stored, - raw_checkpoint, checkpoint_viewed_at: key.checkpoint_viewed_at, }; @@ -590,7 +495,6 @@ impl Loader for Db { async fn load(&self, keys: &[DigestKey]) -> Result, Error> { use checkpoints::dsl; - use raw_checkpoints::dsl as raw_dsl; let digests: BTreeSet<_> = keys.iter().map(|key| key.digest.to_vec()).collect(); @@ -607,24 +511,55 @@ impl Loader for Db { }) .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoints: {e}")))?; - let chckp = checkpoints.clone(); - let checkpoint_id_to_stored: BTreeMap<_, _> = checkpoints + let checkpoint_ids: BTreeMap<_, _> = checkpoints .into_iter() .map(|stored| (stored.checkpoint_digest.clone(), stored)) .collect(); - let checkpoint_ids: Vec = chckp - .into_iter() - .map(|stored| stored.sequence_number) - .collect(); + Ok(keys + .iter() + .filter_map(|key| { + let DigestKey { + digest, + checkpoint_viewed_at, + } = *key; + + let stored = checkpoint_ids.get(digest.as_slice()).cloned()?; + let checkpoint = Checkpoint { + stored: stored.clone(), + checkpoint_viewed_at, + }; + + // Filter by key's checkpoint viewed at here. Doing this in memory because it should + // be quite rare that this query actually filters something, but encoding it in SQL + // is complicated. + let seq_num = checkpoint.stored.sequence_number as u64; + (checkpoint_viewed_at >= seq_num).then_some((*key, checkpoint)) + }) + .collect()) + } +} + +#[async_trait::async_trait] +impl Loader for Db { + type Value = StoredRawCheckpoint; + type Error = Error; + + async fn load(&self, keys: &[SeqNum]) -> Result, Error> { + use raw_checkpoints::dsl; + + let checkpoint_ids = keys + .iter() + .map(|key| key.sequence_number) + .collect::>(); let raw_checkpoints: Vec = self .execute(move |conn| { async move { conn.results(move || { - raw_dsl::raw_checkpoints - .filter(raw_dsl::sequence_number.eq_any(checkpoint_ids.iter().cloned())) + dsl::raw_checkpoints + .filter(dsl::sequence_number.eq_any(checkpoint_ids.iter().cloned())) }) .await } @@ -635,33 +570,14 @@ impl Loader for Db { let raw_checkpoints: BTreeMap<_, _> = raw_checkpoints .into_iter() - .map(|raw_checkpoint| (raw_checkpoint.sequence_number as u64, raw_checkpoint)) + .map(|raw| (raw.sequence_number, raw)) .collect(); Ok(keys .iter() .filter_map(|key| { - let DigestKey { - digest, - checkpoint_viewed_at, - } = *key; - - let stored = checkpoint_id_to_stored.get(digest.as_slice()).cloned()?; - let raw_checkpoint = raw_checkpoints - .get(&(stored.sequence_number as u64)) - .cloned(); - - let checkpoint = Checkpoint { - stored: stored.clone(), - raw_checkpoint, - checkpoint_viewed_at, - }; - - // Filter by key's checkpoint viewed at here. Doing this in memory because it should - // be quite rare that this query actually filters something, but encoding it in SQL - // is complicated. - let seq_num = checkpoint.stored.sequence_number as u64; - (checkpoint_viewed_at >= seq_num).then_some((*key, checkpoint)) + let raw = raw_checkpoints.get(&key.sequence_number).cloned()?; + Some((*key, raw)) }) .collect()) } diff --git a/crates/sui-graphql-rpc/staging.graphql b/crates/sui-graphql-rpc/staging.graphql index 36c9b99f2436a..5b716bea0e210 100644 --- a/crates/sui-graphql-rpc/staging.graphql +++ b/crates/sui-graphql-rpc/staging.graphql @@ -451,7 +451,7 @@ type Checkpoint { """ The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. """ - checkpointSummaryBcs: Base64 + bcs: Base64 } type CheckpointConnection { From 6ca6f21f08dc1072c2f35425739ee8bc359c0832 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:41:44 -0800 Subject: [PATCH 4/7] Update tests --- .../call/checkpoint_connection_pagination.exp | 148 +++++++++++++++++- .../checkpoint_connection_pagination.move | 2 +- .../stable/consistency/epochs/checkpoints.exp | 52 ++++-- .../consistency/epochs/checkpoints.move | 4 +- 4 files changed, 187 insertions(+), 19 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp index 71afffa80ddab..47d467f263447 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp +++ b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.exp @@ -13,7 +13,36 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [] + "edges": [ + { + "cursor": "eyJjIjoxMiwicyI6N30", + "node": { + "sequenceNumber": 7, + "bcs": "AAAAAAAAAAAHAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASBzHtxWhL2GFWHw1uD1bpgd8vblqJQo/HfMtf2InJdZzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + } + }, + { + "cursor": "eyJjIjoxMiwicyI6OH0", + "node": { + "sequenceNumber": 8, + "bcs": "AAAAAAAAAAAIAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASCkAbT63pCVIPXzd6SPYLoprop+R5d98bmNLiEM1jx6UQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + } + }, + { + "cursor": "eyJjIjoxMiwicyI6OX0", + "node": { + "sequenceNumber": 9, + "bcs": "AAAAAAAAAAAJAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASCDmqDOafeT4t61pTBytQnoF1/hbFDoNh+BZwmyyY+aEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + } + }, + { + "cursor": "eyJjIjoxMiwicyI6MTB9", + "node": { + "sequenceNumber": 10, + "bcs": "AAAAAAAAAAAKAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASAMdzATObOHTdGYAryUpeEo6RBpyR+sNwe5oJfYHZCLwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + } + } + ] } } } @@ -27,7 +56,14 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [] + "edges": [ + { + "cursor": "eyJjIjoxMiwicyI6N30", + "node": { + "sequenceNumber": 7 + } + } + ] } } } @@ -80,7 +116,20 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [] + "edges": [ + { + "cursor": "eyJjIjoxMiwicyI6NH0", + "node": { + "sequenceNumber": 4 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6NX0", + "node": { + "sequenceNumber": 5 + } + } + ] } } } @@ -205,7 +254,56 @@ Response: { "hasPreviousPage": true, "hasNextPage": false }, - "edges": [] + "edges": [ + { + "cursor": "eyJjIjoxMiwicyI6NX0", + "node": { + "sequenceNumber": 5 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6Nn0", + "node": { + "sequenceNumber": 6 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6N30", + "node": { + "sequenceNumber": 7 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6OH0", + "node": { + "sequenceNumber": 8 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6OX0", + "node": { + "sequenceNumber": 9 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6MTB9", + "node": { + "sequenceNumber": 10 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6MTF9", + "node": { + "sequenceNumber": 11 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6MTJ9", + "node": { + "sequenceNumber": 12 + } + } + ] } } } @@ -219,7 +317,32 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [] + "edges": [ + { + "cursor": "eyJjIjoxMiwicyI6Mn0", + "node": { + "sequenceNumber": 2 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6M30", + "node": { + "sequenceNumber": 3 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6NH0", + "node": { + "sequenceNumber": 4 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6NX0", + "node": { + "sequenceNumber": 5 + } + } + ] } } } @@ -233,7 +356,20 @@ Response: { "hasPreviousPage": true, "hasNextPage": true }, - "edges": [] + "edges": [ + { + "cursor": "eyJjIjoxMiwicyI6NH0", + "node": { + "sequenceNumber": 4 + } + }, + { + "cursor": "eyJjIjoxMiwicyI6NX0", + "node": { + "sequenceNumber": 5 + } + } + ] } } } diff --git a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move index cd593bb514001..5f824e7c7b9c1 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move +++ b/crates/sui-graphql-e2e-tests/tests/stable/call/checkpoint_connection_pagination.move @@ -41,7 +41,7 @@ cursor node { sequenceNumber - checkpointSummaryBcs + bcs } } } diff --git a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp index ac312c97426ed..096cdd98eba6b 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp +++ b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.exp @@ -21,7 +21,7 @@ Response: { "data": { "checkpoint": { "sequenceNumber": 3, - "checkpointSummaryBcs": "AAAAAAAAAAADAAAAAAAAAAIAAAAAAAAAIAQ3s4Mbet/w/8Z/cvdXYfev9qtQv2TuFOZcI5XBlbOtASBccSay4jtpNOLJUZaWlhsTssP4qMfFMpVfgR+y0a9nOQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFgsWTmpO9qta+EbhNi1JmfZoE7cnLjHchlXrKwsDBBHIfHhvDIyrS5XlX2SKSKymW1Cx+ffk0dthkl6k8iVO6dfANqvfU5l8TzbKggaQFYmzJMQJYP3u1VSIZ5EzM0GH1hECcAAAAAAAAzAAAAAAAAAAACAAA=" + "bcs": "AAAAAAAAAAADAAAAAAAAAAIAAAAAAAAAIAQ3s4Mbet/w/8Z/cvdXYfev9qtQv2TuFOZcI5XBlbOtASBccSay4jtpNOLJUZaWlhsTssP4qMfFMpVfgR+y0a9nOQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFgsWTmpO9qta+EbhNi1JmfZoE7cnLjHchlXrKwsDBBHIfHhvDIyrS5XlX2SKSKymW1Cx+ffk0dthkl6k8iVO6dfANqvfU5l8TzbKggaQFYmzJMQJYP3u1VSIZ5EzM0GH1hECcAAAAAAAAzAAAAAAAAAAACAAA=" }, "epoch": { "epochId": 0, @@ -29,19 +29,19 @@ Response: { "nodes": [ { "sequenceNumber": 0, - "checkpointSummaryBcs": "AAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAIA1J8u1KucLNL1LEsIkImYRYHqL2cGqjpC9nlSCsS5nbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + "bcs": "AAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAIA1J8u1KucLNL1LEsIkImYRYHqL2cGqjpC9nlSCsS5nbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" }, { "sequenceNumber": 1, - "checkpointSummaryBcs": "AAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASC9LDN3eeve+CTXzQsO9w9hAhFTuNfr+nHnztTf5VhotQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + "bcs": "AAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASC9LDN3eeve+CTXzQsO9w9hAhFTuNfr+nHnztTf5VhotQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" }, { "sequenceNumber": 2, - "checkpointSummaryBcs": "AAAAAAAAAAACAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASDXzherctqbdwF+KqXFVfyCfHjpTeC2EW77b1vqLPA6/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" + "bcs": "AAAAAAAAAAACAAAAAAAAAAEAAAAAAAAAILMD/SjqlardhuNt4bDMq2GGFnNP4NrLhIpVn3gSq6xXASDXzherctqbdwF+KqXFVfyCfHjpTeC2EW77b1vqLPA6/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAA==" }, { "sequenceNumber": 3, - "checkpointSummaryBcs": "AAAAAAAAAAADAAAAAAAAAAIAAAAAAAAAIAQ3s4Mbet/w/8Z/cvdXYfev9qtQv2TuFOZcI5XBlbOtASBccSay4jtpNOLJUZaWlhsTssP4qMfFMpVfgR+y0a9nOQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFgsWTmpO9qta+EbhNi1JmfZoE7cnLjHchlXrKwsDBBHIfHhvDIyrS5XlX2SKSKymW1Cx+ffk0dthkl6k8iVO6dfANqvfU5l8TzbKggaQFYmzJMQJYP3u1VSIZ5EzM0GH1hECcAAAAAAAAzAAAAAAAAAAACAAA=" + "bcs": "AAAAAAAAAAADAAAAAAAAAAIAAAAAAAAAIAQ3s4Mbet/w/8Z/cvdXYfev9qtQv2TuFOZcI5XBlbOtASBccSay4jtpNOLJUZaWlhsTssP4qMfFMpVfgR+y0a9nOQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQFgsWTmpO9qta+EbhNi1JmfZoE7cnLjHchlXrKwsDBBHIfHhvDIyrS5XlX2SKSKymW1Cx+ffk0dthkl6k8iVO6dfANqvfU5l8TzbKggaQFYmzJMQJYP3u1VSIZ5EzM0GH1hECcAAAAAAAAzAAAAAAAAAAACAAA=" } ] } @@ -230,19 +230,43 @@ Response: { "epoch_0": { "epochId": 0, "checkpoints": { - "nodes": [] + "nodes": [ + { + "sequenceNumber": 1 + }, + { + "sequenceNumber": 2 + }, + { + "sequenceNumber": 3 + } + ] } }, "epoch_1": { "epochId": 1, "checkpoints": { - "nodes": [] + "nodes": [ + { + "sequenceNumber": 5 + }, + { + "sequenceNumber": 6 + }, + { + "sequenceNumber": 7 + } + ] } }, "epoch_2": { "epochId": 2, "checkpoints": { - "nodes": [] + "nodes": [ + { + "sequenceNumber": 9 + } + ] } } } @@ -258,13 +282,21 @@ Response: { "epoch_0": { "epochId": 0, "checkpoints": { - "nodes": [] + "nodes": [ + { + "sequenceNumber": 2 + } + ] } }, "epoch_1": { "epochId": 1, "checkpoints": { - "nodes": [] + "nodes": [ + { + "sequenceNumber": 6 + } + ] } }, "epoch_2": { diff --git a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move index 27761caef2449..15f27c5fca4f2 100644 --- a/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move +++ b/crates/sui-graphql-e2e-tests/tests/stable/consistency/epochs/checkpoints.move @@ -22,14 +22,14 @@ { checkpoint { sequenceNumber - checkpointSummaryBcs + bcs } epoch { epochId checkpoints { nodes { sequenceNumber - checkpointSummaryBcs + bcs } } } From 21dfec1343ff111ef71eec93e7f5bfb964ef4f25 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:48:21 -0800 Subject: [PATCH 5/7] Fix changes --- crates/sui-graphql-rpc/src/types/checkpoint.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/sui-graphql-rpc/src/types/checkpoint.rs b/crates/sui-graphql-rpc/src/types/checkpoint.rs index 6ded2b31a17a0..53de17846f3e7 100644 --- a/crates/sui-graphql-rpc/src/types/checkpoint.rs +++ b/crates/sui-graphql-rpc/src/types/checkpoint.rs @@ -351,7 +351,6 @@ impl Checkpoint { checkpoint_viewed_at: u64, ) -> Result, Error> { use checkpoints::dsl; - let cursor_viewed_at = page.validate_cursor_consistency()?; let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(checkpoint_viewed_at); @@ -512,7 +511,7 @@ impl Loader for Db { .await .map_err(|e| Error::Internal(format!("Failed to fetch checkpoints: {e}")))?; - let checkpoint_ids: BTreeMap<_, _> = checkpoints + let checkpoint_id_to_stored: BTreeMap<_, _> = checkpoints .into_iter() .map(|stored| (stored.checkpoint_digest.clone(), stored)) .collect(); @@ -525,9 +524,9 @@ impl Loader for Db { checkpoint_viewed_at, } = *key; - let stored = checkpoint_ids.get(digest.as_slice()).cloned()?; + let stored = checkpoint_id_to_stored.get(digest.as_slice()).cloned()?; let checkpoint = Checkpoint { - stored: stored.clone(), + stored, checkpoint_viewed_at, }; From 0b4b83004ec2acb39ebff9063a5c7101eacce7ac Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:54:20 -0800 Subject: [PATCH 6/7] Fix snapshots --- .../tests/snapshots/snapshot_tests__staging.graphql.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap index 4b877c9d060bc..5225913dc4916 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap @@ -455,7 +455,7 @@ type Checkpoint { """ The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. """ - checkpointSummaryBcs: Base64 + bcs: Base64 } type CheckpointConnection { From e47a00770b1ca179c3ff799ba9c9082f39926d22 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Fri, 22 Nov 2024 10:23:49 -0800 Subject: [PATCH 7/7] Rename SeqNum to RawSeqNumKey and optimize loader result --- .../sui-graphql-rpc/src/types/checkpoint.rs | 28 ++++++++++--------- .../snapshot_tests__schema.graphql.snap | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/sui-graphql-rpc/src/types/checkpoint.rs b/crates/sui-graphql-rpc/src/types/checkpoint.rs index 53de17846f3e7..e5aaa02c6cb13 100644 --- a/crates/sui-graphql-rpc/src/types/checkpoint.rs +++ b/crates/sui-graphql-rpc/src/types/checkpoint.rs @@ -43,7 +43,7 @@ pub(crate) struct CheckpointId { /// `DataLoader` key for fetching `StoredRawCheckpoint` by its sequence number. #[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] -struct SeqNum { +struct RawSeqNumKey { pub sequence_number: i64, } @@ -203,7 +203,7 @@ impl Checkpoint { async fn bcs(&self, ctx: &Context<'_>) -> Result> { let DataLoader(dl) = ctx.data_unchecked(); let raw_checkpoint = dl - .load_one(SeqNum { + .load_one(RawSeqNumKey { sequence_number: self.stored.sequence_number, }) .await?; @@ -541,11 +541,14 @@ impl Loader for Db { } #[async_trait::async_trait] -impl Loader for Db { +impl Loader for Db { type Value = StoredRawCheckpoint; type Error = Error; - async fn load(&self, keys: &[SeqNum]) -> Result, Error> { + async fn load( + &self, + keys: &[RawSeqNumKey], + ) -> Result, Error> { use raw_checkpoints::dsl; let checkpoint_ids = keys @@ -567,16 +570,15 @@ impl Loader for Db { .await .map_err(|e| Error::Internal(format!("Failed to fetch raw checkpoints: {e}")))?; - let raw_checkpoints: BTreeMap<_, _> = raw_checkpoints + Ok(raw_checkpoints .into_iter() - .map(|raw| (raw.sequence_number, raw)) - .collect(); - - Ok(keys - .iter() - .filter_map(|key| { - let raw = raw_checkpoints.get(&key.sequence_number).cloned()?; - Some((*key, raw)) + .map(|raw| { + ( + RawSeqNumKey { + sequence_number: raw.sequence_number, + }, + raw, + ) }) .collect()) } diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap index 02ca1bbf1b5fa..172cee668c98e 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap @@ -455,7 +455,7 @@ type Checkpoint { """ The Base64 serialized BCS bytes of CheckpointSummary for this checkpoint. """ - checkpointSummaryBcs: Base64 + bcs: Base64 } type CheckpointConnection {