Skip to content
This repository has been archived by the owner on Jun 6, 2024. It is now read-only.

Commit

Permalink
serde_json deserialize json plan
Browse files Browse the repository at this point in the history
  • Loading branch information
connortsui20 committed Feb 13, 2024
1 parent 37acf52 commit 5ddb185
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 2 deletions.
3 changes: 2 additions & 1 deletion eggstrain/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ anyhow = "1"
arrow = "50"
async-trait = "0.1"
datafusion = "35"
substrait = "0.24"
serde_json = "1"
substrait = { version = "0.24", features = ["serde"] }
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
rayon = "1"
13 changes: 13 additions & 0 deletions eggstrain/src/execution/substrait/deserialize.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
use std::fs;
use substrait::proto::Plan;

pub fn read_str(path: &str) -> String {
fs::read_to_string(path).unwrap_or_else(|_| panic!("Unable to read file {}", path))
}

pub fn get_json(path: &str) -> Plan {
let plan = serde_json::from_str::<Plan>(&read_str(path))
.unwrap_or_else(|_| panic!("Could not parse json {:?} into Plan", path));
println!("{}", serde_json::to_string_pretty(&plan).unwrap());
plan
}
2 changes: 1 addition & 1 deletion eggstrain/src/execution/substrait/example.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use substrait::proto::ReadRel;

/// We want the plan to look like this:
///
/// ```
/// ```text
/// |-+ Aggregate({sales = sum(quantity_price)}, group_by=(product_name, product_id))
/// |-+ InnerJoin(on=orders.product_id = products.product_id)
/// |- ReadTable(orders)
Expand Down
1 change: 1 addition & 0 deletions eggstrain/src/execution/substrait/mod.rs
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pub mod example;
pub mod deserialize;
4 changes: 4 additions & 0 deletions eggstrain/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ pub mod execution;
pub mod scheduler_client;
pub mod storage_client;

use execution::substrait::deserialize::get_json;


#[tokio::main]
async fn main() {
println!("Hello, world!");
get_json("substrait_plan_example.json");
}
184 changes: 184 additions & 0 deletions eggstrain/substrait_plan_example.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
{
"extensions": [
{
"extensionFunction": {
"functionAnchor": 1,
"name": "lte"
}
},
{
"extensionFunction": {
"functionAnchor": 2,
"name": "is_not_null"
}
},
{
"extensionFunction": {
"functionAnchor": 3,
"name": "and"
}
},
{
"extensionFunction": {
"functionAnchor": 4,
"name": "count"
}
}
],
"relations": [
{
"root": {
"input": {
"project": {
"input": {
"aggregate": {
"input": {
"read": {
"baseSchema": {
"names": [
"exercise",
"difficulty_level"
],
"struct": {
"types": [
{
"varchar": {
"length": 13,
"nullability": "NULLABILITY_NULLABLE"
}
},
{
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
}
],
"nullability": "NULLABILITY_REQUIRED"
}
},
"filter": {
"scalarFunction": {
"functionReference": 3,
"outputType": {
"bool": {
"nullability": "NULLABILITY_NULLABLE"
}
},
"arguments": [
{
"value": {
"scalarFunction": {
"functionReference": 1,
"outputType": {
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
"arguments": [
{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 1
}
},
"rootReference": {}
}
}
},
{
"value": {
"literal": {
"i32": 5
}
}
}
]
}
}
},
{
"value": {
"scalarFunction": {
"functionReference": 2,
"outputType": {
"i32": {
"nullability": "NULLABILITY_NULLABLE"
}
},
"arguments": [
{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 1
}
},
"rootReference": {}
}
}
}
]
}
}
}
]
}
},
"projection": {
"select": {
"structItems": [
{}
]
},
"maintainSingularStruct": true
},
"namedTable": {
"names": [
"crossfit"
]
}
}
},
"groupings": [
{}
],
"measures": [
{
"measure": {
"functionReference": 4,
"outputType": {
"i64": {
"nullability": "NULLABILITY_NULLABLE"
}
}
}
}
]
}
},
"expressions": [
{
"selection": {
"directReference": {
"structField": {}
},
"rootReference": {}
}
}
]
}
},
"names": [
"exercise"
]
}
}
],
"version": {
"minorNumber": 24,
"producer": "DuckDB"
}
}

0 comments on commit 5ddb185

Please sign in to comment.