From c33bd2b2c1ab28b5c2a23592f019010e4216ec99 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 26 Nov 2024 16:47:31 -0500 Subject: [PATCH 01/34] all historic volume --- crates/sui-deepbook-indexer/src/server.rs | 30 ++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 8e7cc411d24f6..a995886d91d93 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -29,6 +29,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL: &str = pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; +pub const GET_ALL_HISTORICAL_VOLUME_PATH: &str = "/get_all_historical_volume"; pub fn run_server(socket_address: SocketAddr, state: PgDeepbookPersistent) -> JoinHandle<()> { tokio::spawn(async move { @@ -42,6 +43,7 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { .route("/", get(health_check)) .route(GET_POOLS_PATH, get(get_pools)) .route(GET_HISTORICAL_VOLUME_PATH, get(get_historical_volume)) + .route(GET_ALL_HISTORICAL_VOLUME_PATH, get(get_all_historical_volume)) .route( GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL, get(get_historical_volume_by_balance_manager_id_with_interval), @@ -78,7 +80,6 @@ async fn health_check() -> StatusCode { } /// Get all pools stored in database -#[debug_handler] async fn get_pools( State(state): State, ) -> Result>, DeepBookError> { @@ -145,6 +146,33 @@ async fn get_historical_volume( Ok(Json(volume_by_pool)) } +async fn get_all_historical_volume( + Query(params): Query>, + State(state): State, +) -> Result>, DeepBookError> { + // Step 1: Clone `state.pool` into a separate variable to ensure it lives long enough + let pool = state.pool.clone(); + + // Step 2: Use the cloned pool to get a connection + let connection = &mut pool.get().await?; + + // Step 3: Fetch all pool IDs + let pools: Vec = schema::pools::table + .select(Pools::as_select()) + .load(connection) + .await?; + + // Extract all pool IDs + let pool_ids: String = pools + .into_iter() + .map(|pool| pool.pool_id) + .collect::>() + .join(","); + + // Step 4: Call `get_historical_volume` with the pool IDs and query parameters + get_historical_volume(Path(pool_ids), Query(params), State(state)).await +} + async fn get_historical_volume_by_balance_manager_id( Path((pool_ids, balance_manager_id)): Path<(String, String)>, Query(params): Query>, From 94123b0a09c9b36b07768632653c28a9029a3cb9 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 09:40:48 -0500 Subject: [PATCH 02/34] normalize pool addresses --- crates/sui-deepbook-indexer/src/server.rs | 30 ++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index a995886d91d93..e82a90967b39b 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -143,7 +143,7 @@ async fn get_historical_volume( *volume_by_pool.entry(pool_id).or_insert(0) += volume as u64; } - Ok(Json(volume_by_pool)) + Ok(Json(normalize_pool_addresses(volume_by_pool))) } async fn get_all_historical_volume( @@ -338,3 +338,31 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( Ok(Json(metrics_by_interval)) } + +/// Helper function to normalize pool addresses +fn normalize_pool_addresses( + raw_response: HashMap, +) -> HashMap { + let pool_map = HashMap::from([ + ("0xb663828d6217467c8a1838a03793da896cbe745b150ebd57d82f814ca579fc22", "DEEP_SUI"), + ("0xf948981b806057580f91622417534f491da5f61aeaf33d0ed8e69fd5691c95ce", "DEEP_USDC"), + ("0xe05dafb5133bcffb8d59f4e12465dc0e9faeaa05e3e342a08fe135800e3e4407", "SUI_USDC"), + ("0x1109352b9112717bd2a7c3eb9a416fff1ba6951760f5bdd5424cf5e4e5b3e65c", "BWETH_USDC"), + ("0xa0b9ebefb38c963fd115f52d71fa64501b79d1adcb5270563f92ce0442376545", "WUSDC_USDC"), + ("0x4e2ca3988246e1d50b9bf209abb9c1cbfec65bd95afdacc620a36c67bdb8452f", "WUSDT_USDC"), + ("0x27c4fdb3b846aa3ae4a65ef5127a309aa3c1f466671471a806d8912a18b253e8", "NS_SUI"), + ("0x0c0fdd4008740d81a8a7d4281322aee71a1b62c449eb5b142656753d89ebc060", "NS_USDC"), + ("0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", "TYPUS_SUI") + ]); + + raw_response + .into_iter() + .map(|(address, volume)| { + let pool_name = pool_map + .get(address.as_str()) + .unwrap_or(&"Unknown Pool") + .to_string(); + (pool_name, volume) + }) + .collect() +} From 9988900ab1174e05fdbfc30388e0c18eff590b92 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 10:30:39 -0500 Subject: [PATCH 03/34] example bid-ask endpoint --- crates/sui-deepbook-indexer/src/server.rs | 124 ++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index e82a90967b39b..9b20687af3986 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -14,6 +14,7 @@ use axum::{ routing::get, Json, Router, }; +use anyhow::anyhow; use diesel::dsl::sql; use diesel::BoolExpressionMethods; use diesel::QueryDsl; @@ -23,6 +24,13 @@ use std::time::{SystemTime, UNIX_EPOCH}; use std::{collections::HashMap, net::SocketAddr}; use tokio::{net::TcpListener, task::JoinHandle}; +use sui_json_rpc_types::{SuiObjectData, SuiObjectDataOptions, SuiObjectResponse}; +use sui_sdk::SuiClientBuilder; +use std::str::FromStr; +use sui_types::{ + base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}, digests::ObjectDigest, programmable_transaction_builder::ProgrammableTransactionBuilder, transaction::{Argument, CallArg, Command, ObjectArg, ProgrammableMoveCall, TransactionKind}, type_input::TypeInput, TypeTag +}; + pub const GET_POOLS_PATH: &str = "/get_pools"; pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL: &str = "/get_historical_volume_by_balance_manager_id_with_interval/:pool_ids/:balance_manager_id"; @@ -30,6 +38,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; pub const GET_ALL_HISTORICAL_VOLUME_PATH: &str = "/get_all_historical_volume"; +pub const TESTING_PATH: &str = "/testing"; pub fn run_server(socket_address: SocketAddr, state: PgDeepbookPersistent) -> JoinHandle<()> { tokio::spawn(async move { @@ -52,6 +61,7 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID, get(get_historical_volume_by_balance_manager_id), ) + .route(TESTING_PATH, get(testing)) .with_state(state) } @@ -339,6 +349,115 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( Ok(Json(metrics_by_interval)) } +async fn testing() -> Result>>, DeepBookError> { + let sui_client = SuiClientBuilder::default().build_testnet().await?; + let mut ptb = ProgrammableTransactionBuilder::new(); + + let pool_address = ObjectID::from_hex_literal( + "0x2decc59a6f05c5800e5c8a1135f9d133d1746f562bf56673e6e81ef4f7ccd3b7", + )?; + // get the latest pool object version + let pool_object: SuiObjectResponse = sui_client + .read_api() + .get_object_with_options(pool_address, SuiObjectDataOptions::full_content()) + .await?; + let pool_data: &SuiObjectData = pool_object + .data + .as_ref() + .ok_or(anyhow!("Missing data in pool object response"))?; + + let pool_object_ref: ObjectRef = ( + pool_data.object_id.clone(), + SequenceNumber::from(pool_data.version), + ObjectDigest::from(pool_data.digest.clone()), + ); + + // mark pool_object_ref as the first input. Later used as Argument::Input(0) + let pool_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(pool_object_ref)); + ptb.input(pool_input)?; + + // mark ticks_from_mid as the second input. Later used as Argument::Input(1) + let ticks_from_mid = 10u64; + let input_argument = CallArg::Pure(bcs::to_bytes(&ticks_from_mid).unwrap()); + ptb.input(input_argument)?; + + // Convert the sui_clock_object_id string to ObjectID + let sui_clock_object_id = ObjectID::from_hex_literal( + "0x0000000000000000000000000000000000000000000000000000000000000006", + )?; + // get the latest clock object version + let sui_clock_object: SuiObjectResponse = sui_client + .read_api() + .get_object_with_options(sui_clock_object_id, SuiObjectDataOptions::full_content()) + .await?; + let clock_data: &SuiObjectData = sui_clock_object + .data + .as_ref() + .ok_or(anyhow!("Missing data in clock object response"))?; + + let sui_clock_object_ref: ObjectRef = ( + clock_data.object_id.clone(), + SequenceNumber::from(clock_data.version), + ObjectDigest::from(clock_data.digest.clone()), + ); + + // mark sui_clock_object_ref as the third input. Later used as Argument::Input(2) + let clock_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(sui_clock_object_ref)); + ptb.input(clock_input)?; + + // Correctly use TypeTag for base_coin_type and quote_coin_type + let base_coin_type = parse_type_input("0x36dbef866a1d62bf7328989a10fb2f07d769f4ee587c0de4a0a256e57e0a58a8::deep::DEEP")?; + let quote_coin_type = parse_type_input("0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI")?; + + // Add the Move call to the PTB + let pkg_id = "0xc89b2bd6172c077aec6e8d7ba201e99c32f9770cdae7be6dac9d95132fff8e8e"; + let package = ObjectID::from_hex_literal(pkg_id).map_err(|e| anyhow!(e))?; + let module = "pool".to_string(); + let function = "get_level2_ticks_from_mid".to_string(); + + ptb.command(Command::MoveCall(Box::new(ProgrammableMoveCall { + package, + module, + function, + type_arguments: vec![base_coin_type, quote_coin_type], + arguments: vec![ + Argument::Input(0), // pool.address + Argument::Input(1), // tickFromMid + Argument::Input(2), // SUI_CLOCK_OBJECT_ID + ], + }))); + + let builder = ptb.finish(); + let tx = TransactionKind::ProgrammableTransaction(builder); + // use the read_api() to get the dev_inspect_transaction_block function. + // this does not require you to input any gas coins. + let result = sui_client + .read_api() + .dev_inspect_transaction_block(SuiAddress::default(), tx, None, None, None) + .await?; + + // parse the results. + let binding = result.results.unwrap(); + + let bid_prices = &binding.get(0).unwrap().return_values.get(0).unwrap().0; + let bid_parsed_prices: Vec = bcs::from_bytes(&bid_prices).unwrap(); + let bid_quantities = &binding.get(0).unwrap().return_values.get(1).unwrap().0; + let bid_parsed_quantities: Vec = bcs::from_bytes(&bid_quantities).unwrap(); + + let ask_prices = &binding.get(0).unwrap().return_values.get(2).unwrap().0; + let ask_parsed_prices: Vec = bcs::from_bytes(&ask_prices).unwrap(); + let ask_quantities = &binding.get(0).unwrap().return_values.get(3).unwrap().0; + let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); + + let mut result = HashMap::new(); + result.insert("bid_parsed_prices".to_string(), bid_parsed_prices); + result.insert("bid_parsed_quantities".to_string(), bid_parsed_quantities); + result.insert("ask_parsed_prices".to_string(), ask_parsed_prices); + result.insert("ask_parsed_quantities".to_string(), ask_parsed_quantities); + + Ok(Json(result)) +} + /// Helper function to normalize pool addresses fn normalize_pool_addresses( raw_response: HashMap, @@ -366,3 +485,8 @@ fn normalize_pool_addresses( }) .collect() } + +fn parse_type_input(type_str: &str) -> Result { + let type_tag = TypeTag::from_str(type_str)?; + Ok(TypeInput::from(type_tag)) +} From fd3099ef82221f84287da1916d6e6503650e846f Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 10:38:44 -0500 Subject: [PATCH 04/34] level2 endpoint --- crates/sui-deepbook-indexer/src/server.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 9b20687af3986..c570bb3830be6 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -8,7 +8,6 @@ use crate::{ sui_deepbook_indexer::PgDeepbookPersistent, }; use axum::{ - debug_handler, extract::{Path, Query, State}, http::StatusCode, routing::get, @@ -38,7 +37,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; pub const GET_ALL_HISTORICAL_VOLUME_PATH: &str = "/get_all_historical_volume"; -pub const TESTING_PATH: &str = "/testing"; +pub const LEVEL2_PATH: &str = "/get_level2_ticks_from_mid/:pool_id"; pub fn run_server(socket_address: SocketAddr, state: PgDeepbookPersistent) -> JoinHandle<()> { tokio::spawn(async move { @@ -61,7 +60,7 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID, get(get_historical_volume_by_balance_manager_id), ) - .route(TESTING_PATH, get(testing)) + .route(LEVEL2_PATH, get(get_level2_ticks_from_mid)) .with_state(state) } @@ -349,12 +348,14 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( Ok(Json(metrics_by_interval)) } -async fn testing() -> Result>>, DeepBookError> { +async fn get_level2_ticks_from_mid( + Path(pool_id): Path, +) -> Result>>, DeepBookError> { let sui_client = SuiClientBuilder::default().build_testnet().await?; let mut ptb = ProgrammableTransactionBuilder::new(); let pool_address = ObjectID::from_hex_literal( - "0x2decc59a6f05c5800e5c8a1135f9d133d1746f562bf56673e6e81ef4f7ccd3b7", + &pool_id, )?; // get the latest pool object version let pool_object: SuiObjectResponse = sui_client From b5b042489fe146179b51b667af05212e735fe94b Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 10:47:17 -0500 Subject: [PATCH 05/34] mainnet packages --- crates/sui-deepbook-indexer/src/server.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index c570bb3830be6..97b1002c46061 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -30,6 +30,7 @@ use sui_types::{ base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}, digests::ObjectDigest, programmable_transaction_builder::ProgrammableTransactionBuilder, transaction::{Argument, CallArg, Command, ObjectArg, ProgrammableMoveCall, TransactionKind}, type_input::TypeInput, TypeTag }; +pub const SUI_MAINNET_URL: &str = "https://fullnode.mainnet.sui.io:443"; pub const GET_POOLS_PATH: &str = "/get_pools"; pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL: &str = "/get_historical_volume_by_balance_manager_id_with_interval/:pool_ids/:balance_manager_id"; @@ -351,7 +352,7 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( async fn get_level2_ticks_from_mid( Path(pool_id): Path, ) -> Result>>, DeepBookError> { - let sui_client = SuiClientBuilder::default().build_testnet().await?; + let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); let pool_address = ObjectID::from_hex_literal( @@ -407,11 +408,11 @@ async fn get_level2_ticks_from_mid( ptb.input(clock_input)?; // Correctly use TypeTag for base_coin_type and quote_coin_type - let base_coin_type = parse_type_input("0x36dbef866a1d62bf7328989a10fb2f07d769f4ee587c0de4a0a256e57e0a58a8::deep::DEEP")?; + let base_coin_type = parse_type_input("0xdeeb7a4662eec9f2f3def03fb937a663dddaa2e215b8078a284d026b7946c270::deep::DEEP")?; let quote_coin_type = parse_type_input("0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI")?; // Add the Move call to the PTB - let pkg_id = "0xc89b2bd6172c077aec6e8d7ba201e99c32f9770cdae7be6dac9d95132fff8e8e"; + let pkg_id = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; // Mainnet package let package = ObjectID::from_hex_literal(pkg_id).map_err(|e| anyhow!(e))?; let module = "pool".to_string(); let function = "get_level2_ticks_from_mid".to_string(); From 6a37fd9d9912dc330e2c113f039c9df16ed6999b Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 11:12:22 -0500 Subject: [PATCH 06/34] get all mainnet pools without scaling --- crates/sui-deepbook-indexer/src/server.rs | 78 ++++++++++++++++++----- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 97b1002c46061..4bb1a883072bf 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -407,9 +407,22 @@ async fn get_level2_ticks_from_mid( let clock_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(sui_clock_object_ref)); ptb.input(clock_input)?; - // Correctly use TypeTag for base_coin_type and quote_coin_type - let base_coin_type = parse_type_input("0xdeeb7a4662eec9f2f3def03fb937a663dddaa2e215b8078a284d026b7946c270::deep::DEEP")?; - let quote_coin_type = parse_type_input("0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI")?; + let pool_name_map = get_pool_name_mapping(); + let pool_name = pool_name_map + .get(&pool_id) + .ok_or_else(|| anyhow!("Pool ID not found"))?; + let (base_asset, quote_asset) = parse_pool_name(pool_name)?; + + let asset_type_map = get_asset_type_mapping(); + let base_coin_type = asset_type_map + .get(&base_asset) + .ok_or_else(|| anyhow!("Base asset type not found"))?; + let quote_coin_type = asset_type_map + .get("e_asset) + .ok_or_else(|| anyhow!("Quote asset type not found"))?; + + let base_coin_type = parse_type_input(base_coin_type)?; + let quote_coin_type = parse_type_input(quote_coin_type)?; // Add the Move call to the PTB let pkg_id = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; // Mainnet package @@ -464,30 +477,63 @@ async fn get_level2_ticks_from_mid( fn normalize_pool_addresses( raw_response: HashMap, ) -> HashMap { - let pool_map = HashMap::from([ - ("0xb663828d6217467c8a1838a03793da896cbe745b150ebd57d82f814ca579fc22", "DEEP_SUI"), - ("0xf948981b806057580f91622417534f491da5f61aeaf33d0ed8e69fd5691c95ce", "DEEP_USDC"), - ("0xe05dafb5133bcffb8d59f4e12465dc0e9faeaa05e3e342a08fe135800e3e4407", "SUI_USDC"), - ("0x1109352b9112717bd2a7c3eb9a416fff1ba6951760f5bdd5424cf5e4e5b3e65c", "BWETH_USDC"), - ("0xa0b9ebefb38c963fd115f52d71fa64501b79d1adcb5270563f92ce0442376545", "WUSDC_USDC"), - ("0x4e2ca3988246e1d50b9bf209abb9c1cbfec65bd95afdacc620a36c67bdb8452f", "WUSDT_USDC"), - ("0x27c4fdb3b846aa3ae4a65ef5127a309aa3c1f466671471a806d8912a18b253e8", "NS_SUI"), - ("0x0c0fdd4008740d81a8a7d4281322aee71a1b62c449eb5b142656753d89ebc060", "NS_USDC"), - ("0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", "TYPUS_SUI") - ]); + let pool_map = get_pool_name_mapping(); raw_response .into_iter() .map(|(address, volume)| { let pool_name = pool_map - .get(address.as_str()) - .unwrap_or(&"Unknown Pool") + .get(&address) + .unwrap_or(&"Unknown Pool".to_string()) .to_string(); (pool_name, volume) }) .collect() } +/// This function can return what's in the pool table when stable +fn get_pool_name_mapping() -> HashMap { + [ + ("0xb663828d6217467c8a1838a03793da896cbe745b150ebd57d82f814ca579fc22", "DEEP_SUI"), + ("0xf948981b806057580f91622417534f491da5f61aeaf33d0ed8e69fd5691c95ce", "DEEP_USDC"), + ("0xe05dafb5133bcffb8d59f4e12465dc0e9faeaa05e3e342a08fe135800e3e4407", "SUI_USDC"), + ("0x1109352b9112717bd2a7c3eb9a416fff1ba6951760f5bdd5424cf5e4e5b3e65c", "BWETH_USDC"), + ("0xa0b9ebefb38c963fd115f52d71fa64501b79d1adcb5270563f92ce0442376545", "WUSDC_USDC"), + ("0x4e2ca3988246e1d50b9bf209abb9c1cbfec65bd95afdacc620a36c67bdb8452f", "WUSDT_USDC"), + ("0x27c4fdb3b846aa3ae4a65ef5127a309aa3c1f466671471a806d8912a18b253e8", "NS_SUI"), + ("0x0c0fdd4008740d81a8a7d4281322aee71a1b62c449eb5b142656753d89ebc060", "NS_USDC"), + ("0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", "TYPUS_SUI"), + ] + .iter() + .map(|&(id, name)| (id.to_string(), name.to_string())) + .collect() +} + +/// This function can return what's in the pool table when stable +fn get_asset_type_mapping() -> HashMap { + [ + ("SUI", "0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI"), + ("DEEP", "0xdeeb7a4662eec9f2f3def03fb937a663dddaa2e215b8078a284d026b7946c270::deep::DEEP"), + ("USDC", "0xdba34672e30cb065b1f93e3ab55318768fd6fef66c15942c9f7cb846e2f900e7::usdc::USDC"), + ("BETH", "0xd0e89b2af5e4910726fbcd8b8dd37bb79b29e5f83f7491bca830e94f7f226d29::eth::ETH"), + ("WUSDC", "0x5d4b302506645c37ff133b98c4b50a5ae14841659738d6d733d59d0d217a93bf::coin::COIN"), + ("WUSDT", "0xc060006111016b8a020ad5b33834984a437aaa7d3c74c18e09a95d48aceab08c::coin::COIN"), + ("NS", "0x5145494a5f5100e645e4b0aa950fa6b68f614e8c59e17bc5ded3495123a79178::ns::NS"), + ("TYPUS", "0xf82dc05634970553615eef6112a1ac4fb7bf10272bf6cbe0f80ef44a6c489385::typus::TYPUS"), + ] + .iter() + .map(|&(name, type_str)| (name.to_string(), type_str.to_string())) + .collect() +} + +fn parse_pool_name(pool_name: &str) -> Result<(String, String), anyhow::Error> { + let parts: Vec<&str> = pool_name.split('_').collect(); + if parts.len() != 2 { + return Err(anyhow::anyhow!("Invalid pool name format")); + } + Ok((parts[0].to_string(), parts[1].to_string())) +} + fn parse_type_input(type_str: &str) -> Result { let type_tag = TypeTag::from_str(type_str)?; Ok(TypeInput::from(type_tag)) From 3b8e2a49fce3dfc0a53655aa3f407f94dc78f367 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 11:28:37 -0500 Subject: [PATCH 07/34] basic without division factor --- crates/sui-deepbook-indexer/src/server.rs | 95 ++++++++++++++++++----- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 4bb1a883072bf..412318c04d112 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -413,13 +413,13 @@ async fn get_level2_ticks_from_mid( .ok_or_else(|| anyhow!("Pool ID not found"))?; let (base_asset, quote_asset) = parse_pool_name(pool_name)?; - let asset_type_map = get_asset_type_mapping(); - let base_coin_type = asset_type_map + let asset_info_map = get_asset_info_mapping(); + let (base_coin_type, base_decimals) = asset_info_map .get(&base_asset) - .ok_or_else(|| anyhow!("Base asset type not found"))?; - let quote_coin_type = asset_type_map + .ok_or_else(|| anyhow!("Base asset info not found"))?; + let (quote_coin_type, quote_decimals) = asset_info_map .get("e_asset) - .ok_or_else(|| anyhow!("Quote asset type not found"))?; + .ok_or_else(|| anyhow!("Quote asset info not found"))?; let base_coin_type = parse_type_input(base_coin_type)?; let quote_coin_type = parse_type_input(quote_coin_type)?; @@ -465,10 +465,19 @@ async fn get_level2_ticks_from_mid( let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); let mut result = HashMap::new(); - result.insert("bid_parsed_prices".to_string(), bid_parsed_prices); - result.insert("bid_parsed_quantities".to_string(), bid_parsed_quantities); - result.insert("ask_parsed_prices".to_string(), ask_parsed_prices); - result.insert("ask_parsed_quantities".to_string(), ask_parsed_quantities); + result.insert("bid_parsed_prices".to_string(), bid_parsed_prices.into_iter().map(|quantity| quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap())) + .collect()); + result.insert("bid_parsed_quantities".to_string(), bid_parsed_quantities + .into_iter() + .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) + .collect()); + result.insert("ask_parsed_prices".to_string(), ask_parsed_prices.into_iter() + .map(|quantity| quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap())) + .collect()); + result.insert("ask_parsed_quantities".to_string(), ask_parsed_quantities + .into_iter() + .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) + .collect()); Ok(Json(result)) } @@ -510,19 +519,67 @@ fn get_pool_name_mapping() -> HashMap { } /// This function can return what's in the pool table when stable -fn get_asset_type_mapping() -> HashMap { +fn get_asset_info_mapping() -> HashMap { [ - ("SUI", "0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI"), - ("DEEP", "0xdeeb7a4662eec9f2f3def03fb937a663dddaa2e215b8078a284d026b7946c270::deep::DEEP"), - ("USDC", "0xdba34672e30cb065b1f93e3ab55318768fd6fef66c15942c9f7cb846e2f900e7::usdc::USDC"), - ("BETH", "0xd0e89b2af5e4910726fbcd8b8dd37bb79b29e5f83f7491bca830e94f7f226d29::eth::ETH"), - ("WUSDC", "0x5d4b302506645c37ff133b98c4b50a5ae14841659738d6d733d59d0d217a93bf::coin::COIN"), - ("WUSDT", "0xc060006111016b8a020ad5b33834984a437aaa7d3c74c18e09a95d48aceab08c::coin::COIN"), - ("NS", "0x5145494a5f5100e645e4b0aa950fa6b68f614e8c59e17bc5ded3495123a79178::ns::NS"), - ("TYPUS", "0xf82dc05634970553615eef6112a1ac4fb7bf10272bf6cbe0f80ef44a6c489385::typus::TYPUS"), + ( + "SUI", + ( + "0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI", + 9, + ), + ), + ( + "DEEP", + ( + "0xdeeb7a4662eec9f2f3def03fb937a663dddaa2e215b8078a284d026b7946c270::deep::DEEP", + 6, + ), + ), + ( + "USDC", + ( + "0xdba34672e30cb065b1f93e3ab55318768fd6fef66c15942c9f7cb846e2f900e7::usdc::USDC", + 6, + ), + ), + ( + "BETH", + ( + "0xd0e89b2af5e4910726fbcd8b8dd37bb79b29e5f83f7491bca830e94f7f226d29::eth::ETH", + 8, + ), + ), + ( + "WUSDC", + ( + "0x5d4b302506645c37ff133b98c4b50a5ae14841659738d6d733d59d0d217a93bf::coin::COIN", + 6, + ), + ), + ( + "WUSDT", + ( + "0xc060006111016b8a020ad5b33834984a437aaa7d3c74c18e09a95d48aceab08c::coin::COIN", + 6, + ), + ), + ( + "NS", + ( + "0x5145494a5f5100e645e4b0aa950fa6b68f614e8c59e17bc5ded3495123a79178::ns::NS", + 6, + ), + ), + ( + "TYPUS", + ( + "0xf82dc05634970553615eef6112a1ac4fb7bf10272bf6cbe0f80ef44a6c489385::typus::TYPUS", + 9, + ), + ), ] .iter() - .map(|&(name, type_str)| (name.to_string(), type_str.to_string())) + .map(|&(name, (type_str, decimals))| (name.to_string(), (type_str.to_string(), decimals))) .collect() } From 440598ecf85c0c59be0b97657aa6b4df9749cc76 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 27 Nov 2024 11:29:06 -0500 Subject: [PATCH 08/34] cargo fmt --- crates/sui-deepbook-indexer/src/server.rs | 114 ++++++++++++++++------ 1 file changed, 82 insertions(+), 32 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 412318c04d112..af29dd3a6de3b 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -7,13 +7,13 @@ use crate::{ schema::{self}, sui_deepbook_indexer::PgDeepbookPersistent, }; +use anyhow::anyhow; use axum::{ extract::{Path, Query, State}, http::StatusCode, routing::get, Json, Router, }; -use anyhow::anyhow; use diesel::dsl::sql; use diesel::BoolExpressionMethods; use diesel::QueryDsl; @@ -23,11 +23,16 @@ use std::time::{SystemTime, UNIX_EPOCH}; use std::{collections::HashMap, net::SocketAddr}; use tokio::{net::TcpListener, task::JoinHandle}; +use std::str::FromStr; use sui_json_rpc_types::{SuiObjectData, SuiObjectDataOptions, SuiObjectResponse}; use sui_sdk::SuiClientBuilder; -use std::str::FromStr; use sui_types::{ - base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}, digests::ObjectDigest, programmable_transaction_builder::ProgrammableTransactionBuilder, transaction::{Argument, CallArg, Command, ObjectArg, ProgrammableMoveCall, TransactionKind}, type_input::TypeInput, TypeTag + base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}, + digests::ObjectDigest, + programmable_transaction_builder::ProgrammableTransactionBuilder, + transaction::{Argument, CallArg, Command, ObjectArg, ProgrammableMoveCall, TransactionKind}, + type_input::TypeInput, + TypeTag, }; pub const SUI_MAINNET_URL: &str = "https://fullnode.mainnet.sui.io:443"; @@ -52,7 +57,10 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { .route("/", get(health_check)) .route(GET_POOLS_PATH, get(get_pools)) .route(GET_HISTORICAL_VOLUME_PATH, get(get_historical_volume)) - .route(GET_ALL_HISTORICAL_VOLUME_PATH, get(get_all_historical_volume)) + .route( + GET_ALL_HISTORICAL_VOLUME_PATH, + get(get_all_historical_volume), + ) .route( GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL, get(get_historical_volume_by_balance_manager_id_with_interval), @@ -355,9 +363,7 @@ async fn get_level2_ticks_from_mid( let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); - let pool_address = ObjectID::from_hex_literal( - &pool_id, - )?; + let pool_address = ObjectID::from_hex_literal(&pool_id)?; // get the latest pool object version let pool_object: SuiObjectResponse = sui_client .read_api() @@ -465,27 +471,44 @@ async fn get_level2_ticks_from_mid( let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); let mut result = HashMap::new(); - result.insert("bid_parsed_prices".to_string(), bid_parsed_prices.into_iter().map(|quantity| quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap())) - .collect()); - result.insert("bid_parsed_quantities".to_string(), bid_parsed_quantities - .into_iter() - .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) - .collect()); - result.insert("ask_parsed_prices".to_string(), ask_parsed_prices.into_iter() - .map(|quantity| quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap())) - .collect()); - result.insert("ask_parsed_quantities".to_string(), ask_parsed_quantities - .into_iter() - .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) - .collect()); + result.insert( + "bid_parsed_prices".to_string(), + bid_parsed_prices + .into_iter() + .map(|quantity| { + quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()) + }) + .collect(), + ); + result.insert( + "bid_parsed_quantities".to_string(), + bid_parsed_quantities + .into_iter() + .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) + .collect(), + ); + result.insert( + "ask_parsed_prices".to_string(), + ask_parsed_prices + .into_iter() + .map(|quantity| { + quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()) + }) + .collect(), + ); + result.insert( + "ask_parsed_quantities".to_string(), + ask_parsed_quantities + .into_iter() + .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) + .collect(), + ); Ok(Json(result)) } /// Helper function to normalize pool addresses -fn normalize_pool_addresses( - raw_response: HashMap, -) -> HashMap { +fn normalize_pool_addresses(raw_response: HashMap) -> HashMap { let pool_map = get_pool_name_mapping(); raw_response @@ -503,15 +526,42 @@ fn normalize_pool_addresses( /// This function can return what's in the pool table when stable fn get_pool_name_mapping() -> HashMap { [ - ("0xb663828d6217467c8a1838a03793da896cbe745b150ebd57d82f814ca579fc22", "DEEP_SUI"), - ("0xf948981b806057580f91622417534f491da5f61aeaf33d0ed8e69fd5691c95ce", "DEEP_USDC"), - ("0xe05dafb5133bcffb8d59f4e12465dc0e9faeaa05e3e342a08fe135800e3e4407", "SUI_USDC"), - ("0x1109352b9112717bd2a7c3eb9a416fff1ba6951760f5bdd5424cf5e4e5b3e65c", "BWETH_USDC"), - ("0xa0b9ebefb38c963fd115f52d71fa64501b79d1adcb5270563f92ce0442376545", "WUSDC_USDC"), - ("0x4e2ca3988246e1d50b9bf209abb9c1cbfec65bd95afdacc620a36c67bdb8452f", "WUSDT_USDC"), - ("0x27c4fdb3b846aa3ae4a65ef5127a309aa3c1f466671471a806d8912a18b253e8", "NS_SUI"), - ("0x0c0fdd4008740d81a8a7d4281322aee71a1b62c449eb5b142656753d89ebc060", "NS_USDC"), - ("0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", "TYPUS_SUI"), + ( + "0xb663828d6217467c8a1838a03793da896cbe745b150ebd57d82f814ca579fc22", + "DEEP_SUI", + ), + ( + "0xf948981b806057580f91622417534f491da5f61aeaf33d0ed8e69fd5691c95ce", + "DEEP_USDC", + ), + ( + "0xe05dafb5133bcffb8d59f4e12465dc0e9faeaa05e3e342a08fe135800e3e4407", + "SUI_USDC", + ), + ( + "0x1109352b9112717bd2a7c3eb9a416fff1ba6951760f5bdd5424cf5e4e5b3e65c", + "BWETH_USDC", + ), + ( + "0xa0b9ebefb38c963fd115f52d71fa64501b79d1adcb5270563f92ce0442376545", + "WUSDC_USDC", + ), + ( + "0x4e2ca3988246e1d50b9bf209abb9c1cbfec65bd95afdacc620a36c67bdb8452f", + "WUSDT_USDC", + ), + ( + "0x27c4fdb3b846aa3ae4a65ef5127a309aa3c1f466671471a806d8912a18b253e8", + "NS_SUI", + ), + ( + "0x0c0fdd4008740d81a8a7d4281322aee71a1b62c449eb5b142656753d89ebc060", + "NS_USDC", + ), + ( + "0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", + "TYPUS_SUI", + ), ] .iter() .map(|&(id, name)| (id.to_string(), name.to_string())) From e05b806ddf9be32ed02e0c8ab2dad3c021afa2b7 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 17:01:12 -0500 Subject: [PATCH 09/34] accurate data --- crates/sui-deepbook-indexer/src/server.rs | 26 +++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index af29dd3a6de3b..4f6d10dd17af6 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -359,7 +359,7 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( async fn get_level2_ticks_from_mid( Path(pool_id): Path, -) -> Result>>, DeepBookError> { +) -> Result>>, DeepBookError> { let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); @@ -476,32 +476,40 @@ async fn get_level2_ticks_from_mid( bid_parsed_prices .into_iter() .map(|quantity| { - quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()) + let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + quantity as f64 / factor as f64 }) - .collect(), + .collect::>(), ); result.insert( "bid_parsed_quantities".to_string(), bid_parsed_quantities .into_iter() - .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) - .collect(), + .map(|quantity| { + let factor = 10u64.pow((*base_decimals).try_into().unwrap()); + quantity as f64 / factor as f64 + }) + .collect::>(), ); result.insert( "ask_parsed_prices".to_string(), ask_parsed_prices .into_iter() .map(|quantity| { - quantity / 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()) + let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + quantity as f64 / factor as f64 }) - .collect(), + .collect::>(), ); result.insert( "ask_parsed_quantities".to_string(), ask_parsed_quantities .into_iter() - .map(|quantity| quantity / 10u64.pow((*base_decimals).try_into().unwrap())) - .collect(), + .map(|quantity| { + let factor = 10u64.pow((*base_decimals).try_into().unwrap()); + quantity as f64 / factor as f64 + }) + .collect::>(), ); Ok(Json(result)) From e42ec710028e154dcfa3b89d6a0118c4e2d0c8ac Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 17:08:14 -0500 Subject: [PATCH 10/34] timestamp added --- Cargo.lock | 1 + crates/sui-deepbook-indexer/Cargo.toml | 1 + crates/sui-deepbook-indexer/src/server.rs | 73 +++++++++++++---------- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 188711a11d150..27ca9bbcbf530 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13594,6 +13594,7 @@ dependencies = [ "mysten-metrics", "prometheus", "serde", + "serde_json", "serde_yaml 0.8.26", "sui-config", "sui-data-ingestion-core", diff --git a/crates/sui-deepbook-indexer/Cargo.toml b/crates/sui-deepbook-indexer/Cargo.toml index 035ad9e8fc739..24908cc5aa05f 100644 --- a/crates/sui-deepbook-indexer/Cargo.toml +++ b/crates/sui-deepbook-indexer/Cargo.toml @@ -33,6 +33,7 @@ sui-indexer-builder.workspace = true tempfile.workspace = true axum.workspace = true bigdecimal = { version = "0.4.5" } +serde_json = { version = "1.0", features = ["preserve_order"] } [dev-dependencies] hex-literal = "0.3.4" diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 4f6d10dd17af6..b3b0cffdda84a 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -22,6 +22,7 @@ use diesel_async::RunQueryDsl; use std::time::{SystemTime, UNIX_EPOCH}; use std::{collections::HashMap, net::SocketAddr}; use tokio::{net::TcpListener, task::JoinHandle}; +use serde_json::Value; use std::str::FromStr; use sui_json_rpc_types::{SuiObjectData, SuiObjectDataOptions, SuiObjectResponse}; @@ -359,7 +360,7 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( async fn get_level2_ticks_from_mid( Path(pool_id): Path, -) -> Result>>, DeepBookError> { +) -> Result>, DeepBookError> { let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); @@ -471,47 +472,55 @@ async fn get_level2_ticks_from_mid( let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); let mut result = HashMap::new(); - result.insert( - "bid_parsed_prices".to_string(), - bid_parsed_prices - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); - quantity as f64 / factor as f64 - }) - .collect::>(), - ); + // Insert bid parsed quantities result.insert( "bid_parsed_quantities".to_string(), - bid_parsed_quantities - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((*base_decimals).try_into().unwrap()); - quantity as f64 / factor as f64 - }) - .collect::>(), + Value::Array( + bid_parsed_quantities + .into_iter() + .map(|quantity| { + let factor = 10u64.pow((*base_decimals).try_into().unwrap()); + Value::from(quantity as f64 / factor as f64) + }) + .collect(), + ), ); + + // Insert ask parsed prices result.insert( "ask_parsed_prices".to_string(), - ask_parsed_prices - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); - quantity as f64 / factor as f64 - }) - .collect::>(), + Value::Array( + ask_parsed_prices + .into_iter() + .map(|quantity| { + let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + Value::from(quantity as f64 / factor as f64) + }) + .collect(), + ), ); + + // Insert ask parsed quantities result.insert( "ask_parsed_quantities".to_string(), - ask_parsed_quantities - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((*base_decimals).try_into().unwrap()); - quantity as f64 / factor as f64 - }) - .collect::>(), + Value::Array( + ask_parsed_quantities + .into_iter() + .map(|quantity| { + let factor = 10u64.pow((*base_decimals).try_into().unwrap()); + Value::from(quantity as f64 / factor as f64) + }) + .collect(), + ), ); + // Insert timestamp + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + result.insert("timestamp".to_string(), Value::from(timestamp)); + Ok(Json(result)) } From 4c4d85785b3efbb832ae3f31c1053c74318944e8 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 17:10:55 -0500 Subject: [PATCH 11/34] bid q --- crates/sui-deepbook-indexer/src/server.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index b3b0cffdda84a..afe69aad4f769 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -472,6 +472,20 @@ async fn get_level2_ticks_from_mid( let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); let mut result = HashMap::new(); + // Insert bid parsed prices + result.insert( + "bid_parsed_prices".to_string(), + Value::Array( + bid_parsed_prices + .into_iter() + .map(|quantity| { + let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + Value::from(quantity as f64 / factor as f64) + }) + .collect(), + ), + ); + // Insert bid parsed quantities result.insert( "bid_parsed_quantities".to_string(), From b6f2a15b45a75f65456ace22ee52be11bd5a93f9 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 17:15:13 -0500 Subject: [PATCH 12/34] sorted --- crates/sui-deepbook-indexer/src/server.rs | 87 ++++++++--------------- 1 file changed, 31 insertions(+), 56 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index afe69aad4f769..37c161a51c449 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -472,68 +472,43 @@ async fn get_level2_ticks_from_mid( let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); let mut result = HashMap::new(); - // Insert bid parsed prices - result.insert( - "bid_parsed_prices".to_string(), - Value::Array( - bid_parsed_prices - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); - Value::from(quantity as f64 / factor as f64) - }) - .collect(), - ), - ); - - // Insert bid parsed quantities - result.insert( - "bid_parsed_quantities".to_string(), - Value::Array( - bid_parsed_quantities - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((*base_decimals).try_into().unwrap()); - Value::from(quantity as f64 / factor as f64) - }) - .collect(), - ), - ); - - // Insert ask parsed prices - result.insert( - "ask_parsed_prices".to_string(), - Value::Array( - ask_parsed_prices - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); - Value::from(quantity as f64 / factor as f64) - }) - .collect(), - ), - ); - - // Insert ask parsed quantities - result.insert( - "ask_parsed_quantities".to_string(), - Value::Array( - ask_parsed_quantities - .into_iter() - .map(|quantity| { - let factor = 10u64.pow((*base_decimals).try_into().unwrap()); - Value::from(quantity as f64 / factor as f64) - }) - .collect(), - ), - ); // Insert timestamp let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_millis() as i64; - result.insert("timestamp".to_string(), Value::from(timestamp)); + result.insert("timestamp".to_string(), Value::from(timestamp.to_string())); + + // Generate bids + let bids: Vec = bid_parsed_prices + .into_iter() + .zip(bid_parsed_quantities.into_iter()) + .map(|(price, quantity)| { + let price_factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); + Value::Array(vec![ + Value::from((price as f64 / price_factor as f64).to_string()), + Value::from((quantity as f64 / quantity_factor as f64).to_string()), + ]) + }) + .collect(); + result.insert("bids".to_string(), Value::Array(bids)); + + // Generate asks + let asks: Vec = ask_parsed_prices + .into_iter() + .zip(ask_parsed_quantities.into_iter()) + .map(|(price, quantity)| { + let price_factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); + Value::Array(vec![ + Value::from((price as f64 / price_factor as f64).to_string()), + Value::from((quantity as f64 / quantity_factor as f64).to_string()), + ]) + }) + .collect(); + result.insert("asks".to_string(), Value::Array(asks)); Ok(Json(result)) } From abbc610ee380db5e6e45f5ffa151176d1498e776 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 17:23:22 -0500 Subject: [PATCH 13/34] using pool name --- crates/sui-deepbook-indexer/src/server.rs | 44 ++++++++++------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 37c161a51c449..3d5d20504bad5 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -359,13 +359,20 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( } async fn get_level2_ticks_from_mid( - Path(pool_id): Path, + Path(pool_name): Path, ) -> Result>, DeepBookError> { let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); - let pool_address = ObjectID::from_hex_literal(&pool_id)?; - // get the latest pool object version + let pool_name_map = get_pool_name_mapping(); + let pool_id = pool_name_map + .iter() + .find(|(_, name)| name == &&pool_name) + .map(|(address, _)| address) + .ok_or_else(|| anyhow!("Pool name '{}' not found", pool_name))?; + + let pool_address = ObjectID::from_hex_literal(pool_id)?; + let pool_object: SuiObjectResponse = sui_client .read_api() .get_object_with_options(pool_address, SuiObjectDataOptions::full_content()) @@ -373,28 +380,23 @@ async fn get_level2_ticks_from_mid( let pool_data: &SuiObjectData = pool_object .data .as_ref() - .ok_or(anyhow!("Missing data in pool object response"))?; - + .ok_or(anyhow!("Missing data in pool object response for '{}'", pool_name))?; let pool_object_ref: ObjectRef = ( pool_data.object_id.clone(), SequenceNumber::from(pool_data.version), ObjectDigest::from(pool_data.digest.clone()), ); - // mark pool_object_ref as the first input. Later used as Argument::Input(0) let pool_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(pool_object_ref)); ptb.input(pool_input)?; - // mark ticks_from_mid as the second input. Later used as Argument::Input(1) let ticks_from_mid = 10u64; let input_argument = CallArg::Pure(bcs::to_bytes(&ticks_from_mid).unwrap()); ptb.input(input_argument)?; - // Convert the sui_clock_object_id string to ObjectID let sui_clock_object_id = ObjectID::from_hex_literal( "0x0000000000000000000000000000000000000000000000000000000000000006", )?; - // get the latest clock object version let sui_clock_object: SuiObjectResponse = sui_client .read_api() .get_object_with_options(sui_clock_object_id, SuiObjectDataOptions::full_content()) @@ -410,15 +412,13 @@ async fn get_level2_ticks_from_mid( ObjectDigest::from(clock_data.digest.clone()), ); - // mark sui_clock_object_ref as the third input. Later used as Argument::Input(2) let clock_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(sui_clock_object_ref)); ptb.input(clock_input)?; - let pool_name_map = get_pool_name_mapping(); - let pool_name = pool_name_map - .get(&pool_id) + let pool_full_name = pool_name_map + .get(pool_id) .ok_or_else(|| anyhow!("Pool ID not found"))?; - let (base_asset, quote_asset) = parse_pool_name(pool_name)?; + let (base_asset, quote_asset) = parse_pool_name(pool_full_name)?; let asset_info_map = get_asset_info_mapping(); let (base_coin_type, base_decimals) = asset_info_map @@ -431,8 +431,7 @@ async fn get_level2_ticks_from_mid( let base_coin_type = parse_type_input(base_coin_type)?; let quote_coin_type = parse_type_input(quote_coin_type)?; - // Add the Move call to the PTB - let pkg_id = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; // Mainnet package + let pkg_id = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; let package = ObjectID::from_hex_literal(pkg_id).map_err(|e| anyhow!(e))?; let module = "pool".to_string(); let function = "get_level2_ticks_from_mid".to_string(); @@ -443,22 +442,20 @@ async fn get_level2_ticks_from_mid( function, type_arguments: vec![base_coin_type, quote_coin_type], arguments: vec![ - Argument::Input(0), // pool.address - Argument::Input(1), // tickFromMid - Argument::Input(2), // SUI_CLOCK_OBJECT_ID + Argument::Input(0), + Argument::Input(1), + Argument::Input(2), ], }))); let builder = ptb.finish(); let tx = TransactionKind::ProgrammableTransaction(builder); - // use the read_api() to get the dev_inspect_transaction_block function. - // this does not require you to input any gas coins. + let result = sui_client .read_api() .dev_inspect_transaction_block(SuiAddress::default(), tx, None, None, None) .await?; - // parse the results. let binding = result.results.unwrap(); let bid_prices = &binding.get(0).unwrap().return_values.get(0).unwrap().0; @@ -473,14 +470,12 @@ async fn get_level2_ticks_from_mid( let mut result = HashMap::new(); - // Insert timestamp let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_millis() as i64; result.insert("timestamp".to_string(), Value::from(timestamp.to_string())); - // Generate bids let bids: Vec = bid_parsed_prices .into_iter() .zip(bid_parsed_quantities.into_iter()) @@ -495,7 +490,6 @@ async fn get_level2_ticks_from_mid( .collect(); result.insert("bids".to_string(), Value::Array(bids)); - // Generate asks let asks: Vec = ask_parsed_prices .into_iter() .zip(ask_parsed_quantities.into_iter()) From 2886422e811769079a61f919e4e02b779c6af407 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 17:24:51 -0500 Subject: [PATCH 14/34] move constant --- crates/sui-deepbook-indexer/src/server.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 3d5d20504bad5..3e2c75fc9f9bd 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -45,6 +45,8 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; pub const GET_ALL_HISTORICAL_VOLUME_PATH: &str = "/get_all_historical_volume"; pub const LEVEL2_PATH: &str = "/get_level2_ticks_from_mid/:pool_id"; +pub const DEEPBOOK_PACKAGE_ID: &str = + "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; pub fn run_server(socket_address: SocketAddr, state: PgDeepbookPersistent) -> JoinHandle<()> { tokio::spawn(async move { @@ -431,8 +433,7 @@ async fn get_level2_ticks_from_mid( let base_coin_type = parse_type_input(base_coin_type)?; let quote_coin_type = parse_type_input(quote_coin_type)?; - let pkg_id = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; - let package = ObjectID::from_hex_literal(pkg_id).map_err(|e| anyhow!(e))?; + let package = ObjectID::from_hex_literal(DEEPBOOK_PACKAGE_ID).map_err(|e| anyhow!(e))?; let module = "pool".to_string(); let function = "get_level2_ticks_from_mid".to_string(); From 55fc4ecd857b6d7818d49546646856739c59d819 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 18:04:05 -0500 Subject: [PATCH 15/34] level2 function --- crates/sui-deepbook-indexer/src/server.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 3e2c75fc9f9bd..623295006446e 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -44,7 +44,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; pub const GET_ALL_HISTORICAL_VOLUME_PATH: &str = "/get_all_historical_volume"; -pub const LEVEL2_PATH: &str = "/get_level2_ticks_from_mid/:pool_id"; +pub const LEVEL2_PATH: &str = "/orderbook/:pool_name"; pub const DEEPBOOK_PACKAGE_ID: &str = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; @@ -362,7 +362,25 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( async fn get_level2_ticks_from_mid( Path(pool_name): Path, + Query(params): Query>, ) -> Result>, DeepBookError> { + let depth = params + .get("depth") + .map(|v| v.parse::()) + .transpose() + .map_err(|_| anyhow!("Depth must be a non-negative integer"))?; + + if let Some(depth) = depth { + if depth <= 0 { + return Err(anyhow!("Depth must be a positive number").into()); + } + } + + let ticks_from_mid = match depth { + Some(depth) => (depth / 2) as u64, + None => 10u64, + }; + let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); @@ -392,7 +410,6 @@ async fn get_level2_ticks_from_mid( let pool_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(pool_object_ref)); ptb.input(pool_input)?; - let ticks_from_mid = 10u64; let input_argument = CallArg::Pure(bcs::to_bytes(&ticks_from_mid).unwrap()); ptb.input(input_argument)?; @@ -480,6 +497,7 @@ async fn get_level2_ticks_from_mid( let bids: Vec = bid_parsed_prices .into_iter() .zip(bid_parsed_quantities.into_iter()) + .take(ticks_from_mid as usize) .map(|(price, quantity)| { let price_factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); @@ -494,6 +512,7 @@ async fn get_level2_ticks_from_mid( let asks: Vec = ask_parsed_prices .into_iter() .zip(ask_parsed_quantities.into_iter()) + .take(ticks_from_mid as usize) .map(|(price, quantity)| { let price_factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); From d004f12b813ae9d0b6cd44b1c0a9a7bc3dacf150 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Mon, 2 Dec 2024 18:06:04 -0500 Subject: [PATCH 16/34] default 100 --- crates/sui-deepbook-indexer/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 623295006446e..6aeb950d19378 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -378,7 +378,7 @@ async fn get_level2_ticks_from_mid( let ticks_from_mid = match depth { Some(depth) => (depth / 2) as u64, - None => 10u64, + None => 100u64, }; let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; From b6821093e3cbb6c1966bdfa967dc8c3e7a0998ce Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 12:57:08 -0500 Subject: [PATCH 17/34] prep level --- crates/sui-deepbook-indexer/src/server.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 6aeb950d19378..67bd6d7f06188 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -366,7 +366,7 @@ async fn get_level2_ticks_from_mid( ) -> Result>, DeepBookError> { let depth = params .get("depth") - .map(|v| v.parse::()) + .map(|v| v.parse::()) .transpose() .map_err(|_| anyhow!("Depth must be a non-negative integer"))?; @@ -381,6 +381,12 @@ async fn get_level2_ticks_from_mid( None => 100u64, }; + let level = params + .get("level") + .map(|v| v.parse::()) + .transpose() + .map_err(|_| anyhow!("Level must be between 1 and 3"))?; + let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); From 7ee13ca6fd7c8e2747e0c1641784b62213fa57e5 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 13:43:55 -0500 Subject: [PATCH 18/34] level2 endpoint complete --- crates/sui-deepbook-indexer/src/server.rs | 30 +++++++++++++++-------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 67bd6d7f06188..e74e5b1b0eb36 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -368,24 +368,34 @@ async fn get_level2_ticks_from_mid( .get("depth") .map(|v| v.parse::()) .transpose() - .map_err(|_| anyhow!("Depth must be a non-negative integer"))?; + .map_err(|_| anyhow!("Depth must be a non-negative integer"))? + .map(|depth| if depth == 0 { 200 } else { depth }); if let Some(depth) = depth { - if depth <= 0 { - return Err(anyhow!("Depth must be a positive number").into()); + if depth == 1 { + return Err(anyhow!("Depth cannot be 1. Use a value greater than 1 or 0 for the entire orderbook").into()); } - } - - let ticks_from_mid = match depth { - Some(depth) => (depth / 2) as u64, - None => 100u64, - }; +} let level = params .get("level") .map(|v| v.parse::()) .transpose() - .map_err(|_| anyhow!("Level must be between 1 and 3"))?; + .map_err(|_| anyhow!("Level must be an integer between 1 and 2"))?; + + if let Some(level) = level { + if !(1..=2).contains(&level) { + return Err(anyhow!("Level must be 1 or 2").into()); + } + } + + let ticks_from_mid = match (depth, level) { + (Some(_), Some(1)) => 1u64, // Depth + Level 1 → Best bid and ask + (Some(depth), Some(2)) | (Some(depth), None) => (depth / 2) as u64, // Depth + Level 2 → Use depth + (None, Some(1)) => 1u64, // Only Level 1 → Best bid and ask + (None, Some(2)) | (None, None) => 100u64, // Level 2 or default → 100 ticks + _ => 100u64, // Fallback to default + }; let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; let mut ptb = ProgrammableTransactionBuilder::new(); From 53b63fd52bb0193f2bb1529423d582add38fbfcb Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 13:44:35 -0500 Subject: [PATCH 19/34] cargo fmt --- crates/sui-deepbook-indexer/src/server.rs | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index e74e5b1b0eb36..ab9ae79720566 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -19,10 +19,10 @@ use diesel::BoolExpressionMethods; use diesel::QueryDsl; use diesel::{ExpressionMethods, SelectableHelper}; use diesel_async::RunQueryDsl; +use serde_json::Value; use std::time::{SystemTime, UNIX_EPOCH}; use std::{collections::HashMap, net::SocketAddr}; use tokio::{net::TcpListener, task::JoinHandle}; -use serde_json::Value; use std::str::FromStr; use sui_json_rpc_types::{SuiObjectData, SuiObjectDataOptions, SuiObjectResponse}; @@ -373,9 +373,12 @@ async fn get_level2_ticks_from_mid( if let Some(depth) = depth { if depth == 1 { - return Err(anyhow!("Depth cannot be 1. Use a value greater than 1 or 0 for the entire orderbook").into()); + return Err(anyhow!( + "Depth cannot be 1. Use a value greater than 1 or 0 for the entire orderbook" + ) + .into()); } -} + } let level = params .get("level") @@ -394,7 +397,7 @@ async fn get_level2_ticks_from_mid( (Some(depth), Some(2)) | (Some(depth), None) => (depth / 2) as u64, // Depth + Level 2 → Use depth (None, Some(1)) => 1u64, // Only Level 1 → Best bid and ask (None, Some(2)) | (None, None) => 100u64, // Level 2 or default → 100 ticks - _ => 100u64, // Fallback to default + _ => 100u64, // Fallback to default }; let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; @@ -413,10 +416,10 @@ async fn get_level2_ticks_from_mid( .read_api() .get_object_with_options(pool_address, SuiObjectDataOptions::full_content()) .await?; - let pool_data: &SuiObjectData = pool_object - .data - .as_ref() - .ok_or(anyhow!("Missing data in pool object response for '{}'", pool_name))?; + let pool_data: &SuiObjectData = pool_object.data.as_ref().ok_or(anyhow!( + "Missing data in pool object response for '{}'", + pool_name + ))?; let pool_object_ref: ObjectRef = ( pool_data.object_id.clone(), SequenceNumber::from(pool_data.version), @@ -475,11 +478,7 @@ async fn get_level2_ticks_from_mid( module, function, type_arguments: vec![base_coin_type, quote_coin_type], - arguments: vec![ - Argument::Input(0), - Argument::Input(1), - Argument::Input(2), - ], + arguments: vec![Argument::Input(0), Argument::Input(1), Argument::Input(2)], }))); let builder = ptb.finish(); @@ -515,7 +514,8 @@ async fn get_level2_ticks_from_mid( .zip(bid_parsed_quantities.into_iter()) .take(ticks_from_mid as usize) .map(|(price, quantity)| { - let price_factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + let price_factor = + 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); Value::Array(vec![ Value::from((price as f64 / price_factor as f64).to_string()), @@ -530,7 +530,8 @@ async fn get_level2_ticks_from_mid( .zip(ask_parsed_quantities.into_iter()) .take(ticks_from_mid as usize) .map(|(price, quantity)| { - let price_factor = 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); + let price_factor = + 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); Value::Array(vec![ Value::from((price as f64 / price_factor as f64).to_string()), From b8fa5c95dcba18a55f2b2788dca47e687e559b91 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 13:48:01 -0500 Subject: [PATCH 20/34] pool constants --- crates/sui-deepbook-indexer/src/server.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index ab9ae79720566..f88272adc21fc 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -599,6 +599,14 @@ fn get_pool_name_mapping() -> HashMap { "0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", "TYPUS_SUI", ), + ( + "0x183df694ebc852a5f90a959f0f563b82ac9691e42357e9a9fe961d71a1b809c8", + "SUI_AUSD", + ), + ( + "0x5661fc7f88fbeb8cb881150a810758cf13700bb4e1f31274a244581b37c303c3", + "AUSD_USDC", + ), ] .iter() .map(|&(id, name)| (id.to_string(), name.to_string())) @@ -664,6 +672,13 @@ fn get_asset_info_mapping() -> HashMap { 9, ), ), + ( + "AUSD", + ( + "0x2053d08c1e2bd02791056171aab0fd12bd7cd7efad2ab8f6b9c8902f14df2ff2::ausd::AUSD", + 6, + ), + ), ] .iter() .map(|&(name, (type_str, decimals))| (name.to_string(), (type_str.to_string(), decimals))) From 2f48f7aabf7b3cefd72f3f67c8f84a1a08ae83d6 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 14:03:14 -0500 Subject: [PATCH 21/34] clippy errors --- crates/sui-deepbook-indexer/src/server.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index f88272adc21fc..1eea4154244ff 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -394,10 +394,10 @@ async fn get_level2_ticks_from_mid( let ticks_from_mid = match (depth, level) { (Some(_), Some(1)) => 1u64, // Depth + Level 1 → Best bid and ask - (Some(depth), Some(2)) | (Some(depth), None) => (depth / 2) as u64, // Depth + Level 2 → Use depth - (None, Some(1)) => 1u64, // Only Level 1 → Best bid and ask + (Some(depth), Some(2)) | (Some(depth), None) => depth / 2, // Depth + Level 2 → Use depth + (None, Some(1)) => 1u64, // Only Level 1 → Best bid and ask (None, Some(2)) | (None, None) => 100u64, // Level 2 or default → 100 ticks - _ => 100u64, // Fallback to default + _ => 100u64, // Fallback to default }; let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; @@ -420,11 +420,7 @@ async fn get_level2_ticks_from_mid( "Missing data in pool object response for '{}'", pool_name ))?; - let pool_object_ref: ObjectRef = ( - pool_data.object_id.clone(), - SequenceNumber::from(pool_data.version), - ObjectDigest::from(pool_data.digest.clone()), - ); + let pool_object_ref: ObjectRef = (pool_data.object_id, pool_data.version, pool_data.digest); let pool_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(pool_object_ref)); ptb.input(pool_input)?; @@ -444,11 +440,8 @@ async fn get_level2_ticks_from_mid( .as_ref() .ok_or(anyhow!("Missing data in clock object response"))?; - let sui_clock_object_ref: ObjectRef = ( - clock_data.object_id.clone(), - SequenceNumber::from(clock_data.version), - ObjectDigest::from(clock_data.digest.clone()), - ); + let sui_clock_object_ref: ObjectRef = + (clock_data.object_id, clock_data.version, clock_data.digest); let clock_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(sui_clock_object_ref)); ptb.input(clock_input)?; @@ -494,12 +487,12 @@ async fn get_level2_ticks_from_mid( let bid_prices = &binding.get(0).unwrap().return_values.get(0).unwrap().0; let bid_parsed_prices: Vec = bcs::from_bytes(&bid_prices).unwrap(); let bid_quantities = &binding.get(0).unwrap().return_values.get(1).unwrap().0; - let bid_parsed_quantities: Vec = bcs::from_bytes(&bid_quantities).unwrap(); + let bid_parsed_quantities: Vec = bcs::from_bytes(bid_quantities).unwrap(); let ask_prices = &binding.get(0).unwrap().return_values.get(2).unwrap().0; let ask_parsed_prices: Vec = bcs::from_bytes(&ask_prices).unwrap(); let ask_quantities = &binding.get(0).unwrap().return_values.get(3).unwrap().0; - let ask_parsed_quantities: Vec = bcs::from_bytes(&ask_quantities).unwrap(); + let ask_parsed_quantities: Vec = bcs::from_bytes(ask_quantities).unwrap(); let mut result = HashMap::new(); From 8e67b74c47da37b0b8bb365aefd71dbf30eb07e1 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 14:18:33 -0500 Subject: [PATCH 22/34] dereferencing error --- crates/sui-deepbook-indexer/src/server.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 1eea4154244ff..5ea100bf79251 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -28,8 +28,7 @@ use std::str::FromStr; use sui_json_rpc_types::{SuiObjectData, SuiObjectDataOptions, SuiObjectResponse}; use sui_sdk::SuiClientBuilder; use sui_types::{ - base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}, - digests::ObjectDigest, + base_types::{ObjectID, ObjectRef, SuiAddress}, programmable_transaction_builder::ProgrammableTransactionBuilder, transaction::{Argument, CallArg, Command, ObjectArg, ProgrammableMoveCall, TransactionKind}, type_input::TypeInput, @@ -485,12 +484,12 @@ async fn get_level2_ticks_from_mid( let binding = result.results.unwrap(); let bid_prices = &binding.get(0).unwrap().return_values.get(0).unwrap().0; - let bid_parsed_prices: Vec = bcs::from_bytes(&bid_prices).unwrap(); + let bid_parsed_prices: Vec = bcs::from_bytes(bid_prices).unwrap(); let bid_quantities = &binding.get(0).unwrap().return_values.get(1).unwrap().0; let bid_parsed_quantities: Vec = bcs::from_bytes(bid_quantities).unwrap(); let ask_prices = &binding.get(0).unwrap().return_values.get(2).unwrap().0; - let ask_parsed_prices: Vec = bcs::from_bytes(&ask_prices).unwrap(); + let ask_parsed_prices: Vec = bcs::from_bytes(ask_prices).unwrap(); let ask_quantities = &binding.get(0).unwrap().return_values.get(3).unwrap().0; let ask_parsed_quantities: Vec = bcs::from_bytes(ask_quantities).unwrap(); From 0d8d9d01f9b5381e052a67bb23e08775b94b616d Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 14:24:35 -0500 Subject: [PATCH 23/34] binding changes --- crates/sui-deepbook-indexer/src/server.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 5ea100bf79251..3cc22b9162d87 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -481,16 +481,15 @@ async fn get_level2_ticks_from_mid( .dev_inspect_transaction_block(SuiAddress::default(), tx, None, None, None) .await?; - let binding = result.results.unwrap(); - - let bid_prices = &binding.get(0).unwrap().return_values.get(0).unwrap().0; + let mut binding = result.results.unwrap(); + let bid_prices = &binding.first_mut().unwrap().return_values.get(0).unwrap().0; let bid_parsed_prices: Vec = bcs::from_bytes(bid_prices).unwrap(); - let bid_quantities = &binding.get(0).unwrap().return_values.get(1).unwrap().0; + let bid_quantities = &binding.first_mut().unwrap().return_values.get(1).unwrap().0; let bid_parsed_quantities: Vec = bcs::from_bytes(bid_quantities).unwrap(); - let ask_prices = &binding.get(0).unwrap().return_values.get(2).unwrap().0; + let ask_prices = &binding.first_mut().unwrap().return_values.get(2).unwrap().0; let ask_parsed_prices: Vec = bcs::from_bytes(ask_prices).unwrap(); - let ask_quantities = &binding.get(0).unwrap().return_values.get(3).unwrap().0; + let ask_quantities = &binding.first_mut().unwrap().return_values.get(3).unwrap().0; let ask_parsed_quantities: Vec = bcs::from_bytes(ask_quantities).unwrap(); let mut result = HashMap::new(); From fb1692ff0ea631055adc5ed5988cde5649c81d08 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Tue, 3 Dec 2024 14:41:54 -0500 Subject: [PATCH 24/34] first mut changes --- crates/sui-deepbook-indexer/src/server.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 3cc22b9162d87..e9dc4b9240395 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -482,7 +482,13 @@ async fn get_level2_ticks_from_mid( .await?; let mut binding = result.results.unwrap(); - let bid_prices = &binding.first_mut().unwrap().return_values.get(0).unwrap().0; + let bid_prices = &binding + .first_mut() + .unwrap() + .return_values + .first_mut() + .unwrap() + .0; let bid_parsed_prices: Vec = bcs::from_bytes(bid_prices).unwrap(); let bid_quantities = &binding.first_mut().unwrap().return_values.get(1).unwrap().0; let bid_parsed_quantities: Vec = bcs::from_bytes(bid_quantities).unwrap(); From 4b13680d7a40415e1c6e954ce0fded2b9f312715 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 09:58:54 -0500 Subject: [PATCH 25/34] use pool table --- crates/sui-deepbook-indexer/src/server.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index e9dc4b9240395..3d1b7a93dbd2f 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -166,30 +166,20 @@ async fn get_historical_volume( Ok(Json(normalize_pool_addresses(volume_by_pool))) } +/// Get all historical volume for all pools async fn get_all_historical_volume( Query(params): Query>, State(state): State, ) -> Result>, DeepBookError> { - // Step 1: Clone `state.pool` into a separate variable to ensure it lives long enough - let pool = state.pool.clone(); + let pools: Json> = get_pools(State(state.clone())).await?; - // Step 2: Use the cloned pool to get a connection - let connection = &mut pool.get().await?; - - // Step 3: Fetch all pool IDs - let pools: Vec = schema::pools::table - .select(Pools::as_select()) - .load(connection) - .await?; - - // Extract all pool IDs let pool_ids: String = pools + .0 .into_iter() .map(|pool| pool.pool_id) .collect::>() .join(","); - // Step 4: Call `get_historical_volume` with the pool IDs and query parameters get_historical_volume(Path(pool_ids), Query(params), State(state)).await } From 9cff5e9265ffc05c0727a18b482d9fc496cf749a Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 10:04:42 -0500 Subject: [PATCH 26/34] all historical volume endpoint --- crates/sui-deepbook-indexer/src/server.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 3d1b7a93dbd2f..2e9234cc92b1e 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -42,7 +42,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL: &str = pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; -pub const GET_ALL_HISTORICAL_VOLUME_PATH: &str = "/get_all_historical_volume"; +pub const ALL_HISTORICAL_VOLUME_PATH: &str = "/all_historical_volume"; pub const LEVEL2_PATH: &str = "/orderbook/:pool_name"; pub const DEEPBOOK_PACKAGE_ID: &str = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; @@ -60,8 +60,8 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { .route(GET_POOLS_PATH, get(get_pools)) .route(GET_HISTORICAL_VOLUME_PATH, get(get_historical_volume)) .route( - GET_ALL_HISTORICAL_VOLUME_PATH, - get(get_all_historical_volume), + ALL_HISTORICAL_VOLUME_PATH, + get(all_historical_volume), ) .route( GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL, @@ -167,7 +167,7 @@ async fn get_historical_volume( } /// Get all historical volume for all pools -async fn get_all_historical_volume( +async fn all_historical_volume( Query(params): Query>, State(state): State, ) -> Result>, DeepBookError> { From 7e66e73b435a6c4b058ed2e17b8cb0ec79838b66 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 10:09:37 -0500 Subject: [PATCH 27/34] historical volume --- crates/sui-deepbook-indexer/src/server.rs | 65 ++++++++++++----------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 2e9234cc92b1e..c49f87db73e5a 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -41,7 +41,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL: &str = "/get_historical_volume_by_balance_manager_id_with_interval/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; -pub const GET_HISTORICAL_VOLUME_PATH: &str = "/get_historical_volume/:pool_ids"; +pub const HISTORICAL_VOLUME_PATH: &str = "/historical_volume/:pool_ids"; pub const ALL_HISTORICAL_VOLUME_PATH: &str = "/all_historical_volume"; pub const LEVEL2_PATH: &str = "/orderbook/:pool_name"; pub const DEEPBOOK_PACKAGE_ID: &str = @@ -58,11 +58,8 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { Router::new() .route("/", get(health_check)) .route(GET_POOLS_PATH, get(get_pools)) - .route(GET_HISTORICAL_VOLUME_PATH, get(get_historical_volume)) - .route( - ALL_HISTORICAL_VOLUME_PATH, - get(all_historical_volume), - ) + .route(HISTORICAL_VOLUME_PATH, get(historical_volume)) + .route(ALL_HISTORICAL_VOLUME_PATH, get(all_historical_volume)) .route( GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL, get(get_historical_volume_by_balance_manager_id_with_interval), @@ -112,16 +109,32 @@ async fn get_pools( Ok(Json(results)) } -async fn get_historical_volume( - Path(pool_ids): Path, +async fn historical_volume( + Path(pool_names): Path, Query(params): Query>, State(state): State, ) -> Result>, DeepBookError> { - let connection = &mut state.pool.get().await?; + // Fetch all pools to map names to IDs + let pools: Json> = get_pools(State(state.clone())).await?; + let pool_name_to_id: HashMap = pools + .0 + .into_iter() + .map(|pool| (pool.pool_name, pool.pool_id)) + .collect(); - let pool_ids_list: Vec = pool_ids.split(',').map(|s| s.to_string()).collect(); + // Map provided pool names to pool IDs + let pool_ids_list: Vec = pool_names + .split(',') + .filter_map(|name| pool_name_to_id.get(name).cloned()) + .collect(); - // Get start_time and end_time from query parameters (in seconds) + if pool_ids_list.is_empty() { + return Err(DeepBookError::InternalError( + "No valid pool names provided".to_string(), + )); + } + + // Parse start_time and end_time from query parameters (in seconds) and convert to milliseconds let end_time = params .get("end_time") .and_then(|v| v.parse::().ok()) @@ -150,6 +163,8 @@ async fn get_historical_volume( sql::("quote_quantity") }; + // Query the database for the historical volume + let connection = &mut state.pool.get().await?; let results: Vec<(String, i64)> = schema::order_fills::table .select((schema::order_fills::pool_id, column_to_query)) .filter(schema::order_fills::pool_id.eq_any(pool_ids_list)) @@ -157,13 +172,19 @@ async fn get_historical_volume( .load(connection) .await?; - // Aggregate volume by pool + // Aggregate volume by pool ID and map back to pool names let mut volume_by_pool = HashMap::new(); for (pool_id, volume) in results { - *volume_by_pool.entry(pool_id).or_insert(0) += volume as u64; + if let Some(pool_name) = pool_name_to_id + .iter() + .find(|(_, id)| **id == pool_id) + .map(|(name, _)| name) + { + *volume_by_pool.entry(pool_name.clone()).or_insert(0) += volume as u64; + } } - Ok(Json(normalize_pool_addresses(volume_by_pool))) + Ok(Json(volume_by_pool)) } /// Get all historical volume for all pools @@ -531,22 +552,6 @@ async fn get_level2_ticks_from_mid( Ok(Json(result)) } -/// Helper function to normalize pool addresses -fn normalize_pool_addresses(raw_response: HashMap) -> HashMap { - let pool_map = get_pool_name_mapping(); - - raw_response - .into_iter() - .map(|(address, volume)| { - let pool_name = pool_map - .get(&address) - .unwrap_or(&"Unknown Pool".to_string()) - .to_string(); - (pool_name, volume) - }) - .collect() -} - /// This function can return what's in the pool table when stable fn get_pool_name_mapping() -> HashMap { [ From 34ce84850f795af6284c653fc9cc6afd60444602 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 10:10:09 -0500 Subject: [PATCH 28/34] cleanup --- crates/sui-deepbook-indexer/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index c49f87db73e5a..687f640450478 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -201,7 +201,7 @@ async fn all_historical_volume( .collect::>() .join(","); - get_historical_volume(Path(pool_ids), Query(params), State(state)).await + historical_volume(Path(pool_ids), Query(params), State(state)).await } async fn get_historical_volume_by_balance_manager_id( From 898dcb7d9e0332ab3c5b5fef23d2b4236942420d Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 10:45:01 -0500 Subject: [PATCH 29/34] endpoints updated --- crates/sui-deepbook-indexer/src/server.rs | 197 ++++------------------ 1 file changed, 30 insertions(+), 167 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 687f640450478..8ed48745bafc3 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -194,14 +194,14 @@ async fn all_historical_volume( ) -> Result>, DeepBookError> { let pools: Json> = get_pools(State(state.clone())).await?; - let pool_ids: String = pools + let pool_names: String = pools .0 .into_iter() - .map(|pool| pool.pool_id) + .map(|pool| pool.pool_name) .collect::>() .join(","); - historical_volume(Path(pool_ids), Query(params), State(state)).await + historical_volume(Path(pool_names), Query(params), State(state)).await } async fn get_historical_volume_by_balance_manager_id( @@ -369,10 +369,10 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( Ok(Json(metrics_by_interval)) } - async fn get_level2_ticks_from_mid( Path(pool_name): Path, Query(params): Query>, + State(state): State, ) -> Result>, DeepBookError> { let depth = params .get("depth") @@ -410,17 +410,28 @@ async fn get_level2_ticks_from_mid( _ => 100u64, // Fallback to default }; - let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; - let mut ptb = ProgrammableTransactionBuilder::new(); + // Fetch the pool data from the `pools` table + let connection = &mut state.pool.get().await?; + let pool_data = schema::pools::table + .filter(schema::pools::pool_name.eq(pool_name.clone())) + .select(( + schema::pools::pool_id, + schema::pools::base_asset_id, + schema::pools::base_asset_decimals, + schema::pools::quote_asset_id, + schema::pools::quote_asset_decimals, + )) + .first::<(String, String, i16, String, i16)>(connection) + .await?; - let pool_name_map = get_pool_name_mapping(); - let pool_id = pool_name_map - .iter() - .find(|(_, name)| name == &&pool_name) - .map(|(address, _)| address) - .ok_or_else(|| anyhow!("Pool name '{}' not found", pool_name))?; + let (pool_id, base_asset_id, base_decimals, quote_asset_id, quote_decimals) = pool_data; + let base_decimals = base_decimals as u8; + let quote_decimals = quote_decimals as u8; - let pool_address = ObjectID::from_hex_literal(pool_id)?; + let pool_address = ObjectID::from_hex_literal(&pool_id)?; + + let sui_client = SuiClientBuilder::default().build(SUI_MAINNET_URL).await?; + let mut ptb = ProgrammableTransactionBuilder::new(); let pool_object: SuiObjectResponse = sui_client .read_api() @@ -456,21 +467,8 @@ async fn get_level2_ticks_from_mid( let clock_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(sui_clock_object_ref)); ptb.input(clock_input)?; - let pool_full_name = pool_name_map - .get(pool_id) - .ok_or_else(|| anyhow!("Pool ID not found"))?; - let (base_asset, quote_asset) = parse_pool_name(pool_full_name)?; - - let asset_info_map = get_asset_info_mapping(); - let (base_coin_type, base_decimals) = asset_info_map - .get(&base_asset) - .ok_or_else(|| anyhow!("Base asset info not found"))?; - let (quote_coin_type, quote_decimals) = asset_info_map - .get("e_asset) - .ok_or_else(|| anyhow!("Quote asset info not found"))?; - - let base_coin_type = parse_type_input(base_coin_type)?; - let quote_coin_type = parse_type_input(quote_coin_type)?; + let base_coin_type = parse_type_input(&base_asset_id)?; + let quote_coin_type = parse_type_input("e_asset_id)?; let package = ObjectID::from_hex_literal(DEEPBOOK_PACKAGE_ID).map_err(|e| anyhow!(e))?; let module = "pool".to_string(); @@ -522,9 +520,8 @@ async fn get_level2_ticks_from_mid( .zip(bid_parsed_quantities.into_iter()) .take(ticks_from_mid as usize) .map(|(price, quantity)| { - let price_factor = - 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); - let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); + let price_factor = 10u64.pow((9 - base_decimals + quote_decimals).try_into().unwrap()); + let quantity_factor = 10u64.pow((base_decimals).try_into().unwrap()); Value::Array(vec![ Value::from((price as f64 / price_factor as f64).to_string()), Value::from((quantity as f64 / quantity_factor as f64).to_string()), @@ -538,9 +535,8 @@ async fn get_level2_ticks_from_mid( .zip(ask_parsed_quantities.into_iter()) .take(ticks_from_mid as usize) .map(|(price, quantity)| { - let price_factor = - 10u64.pow((9 - *base_decimals + *quote_decimals).try_into().unwrap()); - let quantity_factor = 10u64.pow((*base_decimals).try_into().unwrap()); + let price_factor = 10u64.pow((9 - base_decimals + quote_decimals).try_into().unwrap()); + let quantity_factor = 10u64.pow((base_decimals).try_into().unwrap()); Value::Array(vec![ Value::from((price as f64 / price_factor as f64).to_string()), Value::from((quantity as f64 / quantity_factor as f64).to_string()), @@ -552,139 +548,6 @@ async fn get_level2_ticks_from_mid( Ok(Json(result)) } -/// This function can return what's in the pool table when stable -fn get_pool_name_mapping() -> HashMap { - [ - ( - "0xb663828d6217467c8a1838a03793da896cbe745b150ebd57d82f814ca579fc22", - "DEEP_SUI", - ), - ( - "0xf948981b806057580f91622417534f491da5f61aeaf33d0ed8e69fd5691c95ce", - "DEEP_USDC", - ), - ( - "0xe05dafb5133bcffb8d59f4e12465dc0e9faeaa05e3e342a08fe135800e3e4407", - "SUI_USDC", - ), - ( - "0x1109352b9112717bd2a7c3eb9a416fff1ba6951760f5bdd5424cf5e4e5b3e65c", - "BWETH_USDC", - ), - ( - "0xa0b9ebefb38c963fd115f52d71fa64501b79d1adcb5270563f92ce0442376545", - "WUSDC_USDC", - ), - ( - "0x4e2ca3988246e1d50b9bf209abb9c1cbfec65bd95afdacc620a36c67bdb8452f", - "WUSDT_USDC", - ), - ( - "0x27c4fdb3b846aa3ae4a65ef5127a309aa3c1f466671471a806d8912a18b253e8", - "NS_SUI", - ), - ( - "0x0c0fdd4008740d81a8a7d4281322aee71a1b62c449eb5b142656753d89ebc060", - "NS_USDC", - ), - ( - "0xe8e56f377ab5a261449b92ac42c8ddaacd5671e9fec2179d7933dd1a91200eec", - "TYPUS_SUI", - ), - ( - "0x183df694ebc852a5f90a959f0f563b82ac9691e42357e9a9fe961d71a1b809c8", - "SUI_AUSD", - ), - ( - "0x5661fc7f88fbeb8cb881150a810758cf13700bb4e1f31274a244581b37c303c3", - "AUSD_USDC", - ), - ] - .iter() - .map(|&(id, name)| (id.to_string(), name.to_string())) - .collect() -} - -/// This function can return what's in the pool table when stable -fn get_asset_info_mapping() -> HashMap { - [ - ( - "SUI", - ( - "0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI", - 9, - ), - ), - ( - "DEEP", - ( - "0xdeeb7a4662eec9f2f3def03fb937a663dddaa2e215b8078a284d026b7946c270::deep::DEEP", - 6, - ), - ), - ( - "USDC", - ( - "0xdba34672e30cb065b1f93e3ab55318768fd6fef66c15942c9f7cb846e2f900e7::usdc::USDC", - 6, - ), - ), - ( - "BETH", - ( - "0xd0e89b2af5e4910726fbcd8b8dd37bb79b29e5f83f7491bca830e94f7f226d29::eth::ETH", - 8, - ), - ), - ( - "WUSDC", - ( - "0x5d4b302506645c37ff133b98c4b50a5ae14841659738d6d733d59d0d217a93bf::coin::COIN", - 6, - ), - ), - ( - "WUSDT", - ( - "0xc060006111016b8a020ad5b33834984a437aaa7d3c74c18e09a95d48aceab08c::coin::COIN", - 6, - ), - ), - ( - "NS", - ( - "0x5145494a5f5100e645e4b0aa950fa6b68f614e8c59e17bc5ded3495123a79178::ns::NS", - 6, - ), - ), - ( - "TYPUS", - ( - "0xf82dc05634970553615eef6112a1ac4fb7bf10272bf6cbe0f80ef44a6c489385::typus::TYPUS", - 9, - ), - ), - ( - "AUSD", - ( - "0x2053d08c1e2bd02791056171aab0fd12bd7cd7efad2ab8f6b9c8902f14df2ff2::ausd::AUSD", - 6, - ), - ), - ] - .iter() - .map(|&(name, (type_str, decimals))| (name.to_string(), (type_str.to_string(), decimals))) - .collect() -} - -fn parse_pool_name(pool_name: &str) -> Result<(String, String), anyhow::Error> { - let parts: Vec<&str> = pool_name.split('_').collect(); - if parts.len() != 2 { - return Err(anyhow::anyhow!("Invalid pool name format")); - } - Ok((parts[0].to_string(), parts[1].to_string())) -} - fn parse_type_input(type_str: &str) -> Result { let type_tag = TypeTag::from_str(type_str)?; Ok(TypeInput::from(type_tag)) From 47e282b9889d00b6b15650a3ad84a7522cc792a4 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 10:49:11 -0500 Subject: [PATCH 30/34] cleanup --- crates/sui-deepbook-indexer/src/server.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 8ed48745bafc3..bdea58a3f7475 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -68,7 +68,7 @@ pub(crate) fn make_router(state: PgDeepbookPersistent) -> Router { GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID, get(get_historical_volume_by_balance_manager_id), ) - .route(LEVEL2_PATH, get(get_level2_ticks_from_mid)) + .route(LEVEL2_PATH, get(orderbook)) .with_state(state) } @@ -369,7 +369,9 @@ async fn get_historical_volume_by_balance_manager_id_with_interval( Ok(Json(metrics_by_interval)) } -async fn get_level2_ticks_from_mid( + +/// Level2 data for all pools +async fn orderbook( Path(pool_name): Path, Query(params): Query>, State(state): State, From a31b617fdad91bbc3cff4619778946ca54a6db73 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 10:51:40 -0500 Subject: [PATCH 31/34] cleanup --- crates/sui-deepbook-indexer/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index bdea58a3f7475..347cc9519341c 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -41,7 +41,7 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID_WITH_INTERVAL: &str = "/get_historical_volume_by_balance_manager_id_with_interval/:pool_ids/:balance_manager_id"; pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = "/get_historical_volume_by_balance_manager_id/:pool_ids/:balance_manager_id"; -pub const HISTORICAL_VOLUME_PATH: &str = "/historical_volume/:pool_ids"; +pub const HISTORICAL_VOLUME_PATH: &str = "/historical_volume/:pool_names"; pub const ALL_HISTORICAL_VOLUME_PATH: &str = "/all_historical_volume"; pub const LEVEL2_PATH: &str = "/orderbook/:pool_name"; pub const DEEPBOOK_PACKAGE_ID: &str = From 71827b1106a1c43ae40de458cc714e70b6ba383d Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 11:06:49 -0500 Subject: [PATCH 32/34] remove anyhow errors --- crates/sui-deepbook-indexer/src/server.rs | 45 +++++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 347cc9519341c..256885ecc6520 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -7,7 +7,6 @@ use crate::{ schema::{self}, sui_deepbook_indexer::PgDeepbookPersistent, }; -use anyhow::anyhow; use axum::{ extract::{Path, Query, State}, http::StatusCode, @@ -380,15 +379,17 @@ async fn orderbook( .get("depth") .map(|v| v.parse::()) .transpose() - .map_err(|_| anyhow!("Depth must be a non-negative integer"))? + .map_err(|_| { + DeepBookError::InternalError("Depth must be a non-negative integer".to_string()) + })? .map(|depth| if depth == 0 { 200 } else { depth }); if let Some(depth) = depth { if depth == 1 { - return Err(anyhow!( + return Err(DeepBookError::InternalError( "Depth cannot be 1. Use a value greater than 1 or 0 for the entire orderbook" - ) - .into()); + .to_string(), + )); } } @@ -396,11 +397,15 @@ async fn orderbook( .get("level") .map(|v| v.parse::()) .transpose() - .map_err(|_| anyhow!("Level must be an integer between 1 and 2"))?; + .map_err(|_| { + DeepBookError::InternalError("Level must be an integer between 1 and 2".to_string()) + })?; if let Some(level) = level { if !(1..=2).contains(&level) { - return Err(anyhow!("Level must be 1 or 2").into()); + return Err(DeepBookError::InternalError( + "Level must be 1 or 2".to_string(), + )); } } @@ -439,10 +444,14 @@ async fn orderbook( .read_api() .get_object_with_options(pool_address, SuiObjectDataOptions::full_content()) .await?; - let pool_data: &SuiObjectData = pool_object.data.as_ref().ok_or(anyhow!( - "Missing data in pool object response for '{}'", - pool_name - ))?; + let pool_data: &SuiObjectData = + pool_object + .data + .as_ref() + .ok_or(DeepBookError::InternalError(format!( + "Missing data in pool object response for '{}'", + pool_name + )))?; let pool_object_ref: ObjectRef = (pool_data.object_id, pool_data.version, pool_data.digest); let pool_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(pool_object_ref)); @@ -458,10 +467,13 @@ async fn orderbook( .read_api() .get_object_with_options(sui_clock_object_id, SuiObjectDataOptions::full_content()) .await?; - let clock_data: &SuiObjectData = sui_clock_object - .data - .as_ref() - .ok_or(anyhow!("Missing data in clock object response"))?; + let clock_data: &SuiObjectData = + sui_clock_object + .data + .as_ref() + .ok_or(DeepBookError::InternalError( + "Missing data in clock object response".to_string(), + ))?; let sui_clock_object_ref: ObjectRef = (clock_data.object_id, clock_data.version, clock_data.digest); @@ -472,7 +484,8 @@ async fn orderbook( let base_coin_type = parse_type_input(&base_asset_id)?; let quote_coin_type = parse_type_input("e_asset_id)?; - let package = ObjectID::from_hex_literal(DEEPBOOK_PACKAGE_ID).map_err(|e| anyhow!(e))?; + let package = ObjectID::from_hex_literal(&DEEPBOOK_PACKAGE_ID) + .map_err(|e| DeepBookError::InternalError(format!("Invalid pool ID: {}", e)))?; let module = "pool".to_string(); let function = "get_level2_ticks_from_mid".to_string(); From 973ab9e7dbc376c2709c5bd6c7becf19fcf12fa5 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 11:26:29 -0500 Subject: [PATCH 33/34] clippy errors --- crates/sui-deepbook-indexer/src/server.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index 256885ecc6520..ad2cf4cc436f5 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -484,7 +484,7 @@ async fn orderbook( let base_coin_type = parse_type_input(&base_asset_id)?; let quote_coin_type = parse_type_input("e_asset_id)?; - let package = ObjectID::from_hex_literal(&DEEPBOOK_PACKAGE_ID) + let package = ObjectID::from_hex_literal(DEEPBOOK_PACKAGE_ID) .map_err(|e| DeepBookError::InternalError(format!("Invalid pool ID: {}", e)))?; let module = "pool".to_string(); let function = "get_level2_ticks_from_mid".to_string(); @@ -535,8 +535,8 @@ async fn orderbook( .zip(bid_parsed_quantities.into_iter()) .take(ticks_from_mid as usize) .map(|(price, quantity)| { - let price_factor = 10u64.pow((9 - base_decimals + quote_decimals).try_into().unwrap()); - let quantity_factor = 10u64.pow((base_decimals).try_into().unwrap()); + let price_factor = 10u64.pow((9 - base_decimals + quote_decimals).into()); + let quantity_factor = 10u64.pow((base_decimals).into()); Value::Array(vec![ Value::from((price as f64 / price_factor as f64).to_string()), Value::from((quantity as f64 / quantity_factor as f64).to_string()), @@ -550,8 +550,8 @@ async fn orderbook( .zip(ask_parsed_quantities.into_iter()) .take(ticks_from_mid as usize) .map(|(price, quantity)| { - let price_factor = 10u64.pow((9 - base_decimals + quote_decimals).try_into().unwrap()); - let quantity_factor = 10u64.pow((base_decimals).try_into().unwrap()); + let price_factor = 10u64.pow((9 - base_decimals + quote_decimals).into()); + let quantity_factor = 10u64.pow((base_decimals).into()); Value::Array(vec![ Value::from((price as f64 / price_factor as f64).to_string()), Value::from((quantity as f64 / quantity_factor as f64).to_string()), From bbeb61b07ff2c7891e156b88ba72ae7350050841 Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 4 Dec 2024 12:21:11 -0500 Subject: [PATCH 34/34] remove unwrap and constants --- crates/sui-deepbook-indexer/src/server.rs | 76 ++++++++++++++++++----- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index ad2cf4cc436f5..8e7bc27c58a49 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -43,6 +43,8 @@ pub const GET_HISTORICAL_VOLUME_BY_BALANCE_MANAGER_ID: &str = pub const HISTORICAL_VOLUME_PATH: &str = "/historical_volume/:pool_names"; pub const ALL_HISTORICAL_VOLUME_PATH: &str = "/all_historical_volume"; pub const LEVEL2_PATH: &str = "/orderbook/:pool_name"; +pub const LEVEL2_MODULE: &str = "pool"; +pub const LEVEL2_FUNCTION: &str = "get_level2_ticks_from_mid"; pub const DEEPBOOK_PACKAGE_ID: &str = "0x2c8d603bc51326b8c13cef9dd07031a408a48dddb541963357661df5d3204809"; @@ -457,7 +459,9 @@ async fn orderbook( let pool_input = CallArg::Object(ObjectArg::ImmOrOwnedObject(pool_object_ref)); ptb.input(pool_input)?; - let input_argument = CallArg::Pure(bcs::to_bytes(&ticks_from_mid).unwrap()); + let input_argument = CallArg::Pure(bcs::to_bytes(&ticks_from_mid).map_err(|_| { + DeepBookError::InternalError("Failed to serialize ticks_from_mid".to_string()) + })?); ptb.input(input_argument)?; let sui_clock_object_id = ObjectID::from_hex_literal( @@ -486,8 +490,8 @@ async fn orderbook( let package = ObjectID::from_hex_literal(DEEPBOOK_PACKAGE_ID) .map_err(|e| DeepBookError::InternalError(format!("Invalid pool ID: {}", e)))?; - let module = "pool".to_string(); - let function = "get_level2_ticks_from_mid".to_string(); + let module = LEVEL2_MODULE.to_string(); + let function = LEVEL2_FUNCTION.to_string(); ptb.command(Command::MoveCall(Box::new(ProgrammableMoveCall { package, @@ -505,28 +509,72 @@ async fn orderbook( .dev_inspect_transaction_block(SuiAddress::default(), tx, None, None, None) .await?; - let mut binding = result.results.unwrap(); + let mut binding = result.results.ok_or(DeepBookError::InternalError( + "No results from dev_inspect_transaction_block".to_string(), + ))?; let bid_prices = &binding .first_mut() - .unwrap() + .ok_or(DeepBookError::InternalError( + "No return values for bid prices".to_string(), + ))? .return_values .first_mut() - .unwrap() + .ok_or(DeepBookError::InternalError( + "No bid price data found".to_string(), + ))? .0; - let bid_parsed_prices: Vec = bcs::from_bytes(bid_prices).unwrap(); - let bid_quantities = &binding.first_mut().unwrap().return_values.get(1).unwrap().0; - let bid_parsed_quantities: Vec = bcs::from_bytes(bid_quantities).unwrap(); + let bid_parsed_prices: Vec = bcs::from_bytes(bid_prices).map_err(|_| { + DeepBookError::InternalError("Failed to deserialize bid prices".to_string()) + })?; + let bid_quantities = &binding + .first_mut() + .ok_or(DeepBookError::InternalError( + "No return values for bid quantities".to_string(), + ))? + .return_values + .get(1) + .ok_or(DeepBookError::InternalError( + "No bid quantity data found".to_string(), + ))? + .0; + let bid_parsed_quantities: Vec = bcs::from_bytes(bid_quantities).map_err(|_| { + DeepBookError::InternalError("Failed to deserialize bid quantities".to_string()) + })?; - let ask_prices = &binding.first_mut().unwrap().return_values.get(2).unwrap().0; - let ask_parsed_prices: Vec = bcs::from_bytes(ask_prices).unwrap(); - let ask_quantities = &binding.first_mut().unwrap().return_values.get(3).unwrap().0; - let ask_parsed_quantities: Vec = bcs::from_bytes(ask_quantities).unwrap(); + let ask_prices = &binding + .first_mut() + .ok_or(DeepBookError::InternalError( + "No return values for ask prices".to_string(), + ))? + .return_values + .get(2) + .ok_or(DeepBookError::InternalError( + "No ask price data found".to_string(), + ))? + .0; + let ask_parsed_prices: Vec = bcs::from_bytes(ask_prices).map_err(|_| { + DeepBookError::InternalError("Failed to deserialize ask prices".to_string()) + })?; + let ask_quantities = &binding + .first_mut() + .ok_or(DeepBookError::InternalError( + "No return values for ask quantities".to_string(), + ))? + .return_values + .get(3) + .ok_or(DeepBookError::InternalError( + "No ask quantity data found".to_string(), + ))? + .0; + let ask_parsed_quantities: Vec = bcs::from_bytes(ask_quantities).map_err(|_| { + DeepBookError::InternalError("Failed to deserialize ask quantities".to_string()) + })?; let mut result = HashMap::new(); let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) - .unwrap() + .map_err(|_| DeepBookError::InternalError("System time error".to_string()))? .as_millis() as i64; result.insert("timestamp".to_string(), Value::from(timestamp.to_string()));