Skip to content

Commit

Permalink
chor: refactor grpc command to use topograph to clean up the control …
Browse files Browse the repository at this point in the history
…flow. also pull in backfill to be used in ingest
  • Loading branch information
kespinola committed Jul 30, 2024
2 parents d80e7b8 + 138bf2e commit 2cc3411
Show file tree
Hide file tree
Showing 47 changed files with 1,630 additions and 1,104 deletions.
7 changes: 4 additions & 3 deletions Builder.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
FROM rust:1.75-bullseye AS builder
RUN apt-get update -y && \
apt-get install -y build-essential make git
apt-get install -y build-essential make git

RUN mkdir /rust
RUN mkdir /rust/bins
COPY Cargo.toml /rust
COPY core /rust/core
COPY backfill /rust/backfill
COPY das_api /rust/das_api
COPY digital_asset_types /rust/digital_asset_types
COPY integration_tests /rust/integration_tests
Expand All @@ -19,7 +20,7 @@ COPY blockbuster rust/blockbuster
WORKDIR /rust
RUN --mount=type=cache,target=/rust/target,id=das-rust \
cargo build --release --bins && cp `find /rust/target/release -maxdepth 1 -type f | sed 's/^\.\///' | grep -v "\." ` /rust/bins

FROM rust:1.75-slim-bullseye as final
COPY --from=builder /rust/bins /das/
CMD echo "Built the DAS API bins!"
127 changes: 123 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 8 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[workspace]
members = [
"backfill",
"blockbuster",
"core",
"das_api",
Expand Down Expand Up @@ -47,7 +48,9 @@ cargo-lock = "9.0.0"
chrono = "0.4.19"
clap = "4.2.2"
das-core = { path = "core" }
das-backfill = { path = "backfill" }
das_api = { path = "das_api" }
derive_more = { version = "0.99.17" }
digital_asset_types = { path = "digital_asset_types" }
enum-iterator = "1.2.0"
enum-iterator-derive = "1.1.0"
Expand Down Expand Up @@ -127,6 +130,9 @@ stretto = "0.8.4"
thiserror = "1.0.31"
tokio = "1.30.0"
tokio-stream = "0.1.14"
topograph = { version = "0.4.0", features = [
"tokio",
], git = "https://github.com/ray-kast/topograph", branch = "v0.4-wip" }
tower = "0.4.13"
tower-http = "0.3.5"
tracing = "0.1.35"
Expand All @@ -137,8 +143,8 @@ url = "2.3.1"
vergen = "8.2.1"
wasi = "0.7.0"
wasm-bindgen = "0.2.83"
yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin
yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin
yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin
yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin
yellowstone-grpc-tools = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28", default-features = false } # tag is geyser plugin

[workspace.lints.clippy]
Expand Down
1 change: 1 addition & 0 deletions Proxy.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ RUN cargo install wasm-pack
RUN mkdir /rust
COPY ./Cargo.toml /rust
COPY ./core /rust/core
COPY ./backfill /rust/backfill
COPY ./das_api /rust/das_api
COPY ./digital_asset_types /rust/digital_asset_types
COPY ./integration_tests /rust/integration_tests
Expand Down
40 changes: 40 additions & 0 deletions backfill/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
[package]
name = "das-backfill"
version = { workspace = true }
edition = { workspace = true }
repository = { workspace = true }
publish = { workspace = true }

[dependencies]
anyhow = { workspace = true }
blockbuster = { workspace = true }
bs58 = { workspace = true }
das-core = { workspace = true }
solana-client = { workspace = true }
borsh = { workspace = true }
digital_asset_types = { workspace = true, features = [
"json_types",
"sql_types",
] }
anchor-client = { workspace = true }
futures = { workspace = true }
clap = { workspace = true }
log = { workspace = true }
solana-program = { workspace = true }
program_transformers = { workspace = true }
heck = { workspace = true }
mpl-bubblegum = { workspace = true }
num-traits = { workspace = true }
sea-orm = { workspace = true }
serde_json = { workspace = true }
solana-sdk = { workspace = true }
solana-transaction-status = { workspace = true }
spl-account-compression = { workspace = true, features = ["no-entrypoint"] }
spl-token = { workspace = true, features = ["no-entrypoint"] }
sqlx = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["time"] }
tracing = { workspace = true }

[lints]
workspace = true
24 changes: 24 additions & 0 deletions backfill/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
## DAS Backfill

The DAS Backfill library facilitates the initial setup and data backfilling for DAS, focusing on the bubblegum program. This program's indexing heavily relies on transaction data. While the library supports parallel backfilling across different trees, it ensures that transactions within each tree are processed sequentially. This approach guarantees accurate representation of every modification in the merkle tree within DAS.

## Usage

```rust
use das_backfill::{
BubblegumBackfillArgs,
BubblegumBackfillContext,
start_bubblegum_backfill
};

#[tokio::main]
async fn main() -> anyhow::Result<()> {
let database_pool = sqlx::PgPool::connect("your_database_url").await?;
let solana_rpc = Rpc::new("your_solana_rpc_url");

let context = BubblegumBackfillContext::new(database_pool, solana_rpc);
let args = BubblegumBackfillArgs::parse(); // Parses args from CLI

start_bubblegum_backfill(context, args).await
}
```
19 changes: 19 additions & 0 deletions backfill/src/error.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#[derive(Debug, thiserror::Error)]
pub enum ErrorKind {
#[error("anchor")]
Anchor(#[from] anchor_client::anchor_lang::error::Error),
#[error("solana rpc")]
Rpc(#[from] solana_client::client_error::ClientError),
#[error("parse pubkey")]
ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError),
#[error("serialize tree response")]
SerializeTreeResponse,
#[error("sea orm")]
Database(#[from] sea_orm::DbErr),
#[error("try from pubkey")]
TryFromPubkey,
#[error("try from signature")]
TryFromSignature,
#[error("generic error: {0}")]
Generic(String),
}
Loading

0 comments on commit 2cc3411

Please sign in to comment.