Database Proxy (#2000)

* feat: Add proxy connection type

* feat: Add proxy database's proxy functions trait.

* fix: Remove some unused impl to fix the unit test

* test: Create the proxy by empty declaration.

* test: Try to genereate query and exec commands.

* perf: Add more query debug trait for debugging.

* chore: Add the example for wasi + proxy.

* chore: Try to read string from wasmtime vm.

* chore: Sucks, but how to do without tokio::spawn?

* chore: Complete the basic memory read logic.

* chore: Abandon the WASI demo, native demo first...

* refactor: Use single proxy connection generator
to avoid stack overflow

* refactor: Rename the inner structs' name

* fix: Fix CI clippy and unit test

* fix: Rename the example.

* chore: Try to embed surrealdb for proxy test.

* fix: Transfer the query result correctly.

* refactor: Rename the example.

* chore: Ready to add example for wasmtime proxy.

* feat: Try to compile sea-orm into wasm binary.
But it would failed on wasm32-wasi target because of the socket deps.
It can be compiled on wasm32-unknown-unknown target.

* fix: WASM targets can't use sqlx.

* fix: Try to fix CI by remove toml.

* fix: Try to fix CI by remove toml.

* fix: Move vm to the example's root dir.

* fix: Add a pre-build script.

* chore: Add README.

* fix: Try to fix CI.

* feat: Add proxy logic in wasm module.

* fix: Try to run the wasi module.
But WASI cannot support multi threads..
so the module was run failed.

* refactor: Bump wasmtime to 14.

* fix: Now we can use async traits on wasmtime.
The solution is add the current thread tag to tokio-wasi.

* build: Use build.rs instead of dynamic command.

* feat: Add the execute result's transfer logic.

* fix: Convert sqlx query result for sea-query.

* fix: Now we can transfer wasm's query to outside.

* refactor: Convert to ProxyRow first.
It's the solution to know the type information about the value.

* fix: Multiple time library reference.

* feat: Add a new proxy example which uses GlueSQL.

* test: Add the test cases for three new examples.
Just try to run once...

* ci: Add wasm component's compiler for unit test.

* ci: Add wasi target.

* ci: It may needs wasi target twice...

* feat: Add more keys for proxy execute result.
To transfer the fully information of the execute result.

* fix: Use custom id type instead of json value.

* fix: Wrong reference type.

* fix: Rewrite the transformer.

* perf: Add ToString trait for proxy exec result.

* revert: Again.
Refs: 9bac6e9

* revert: Back to the basic proxy exec result.
Refs: e0330dd

* refactor: Update GlueSQL and SurrealDB examples. (#1980)

* refactor: Bump gluesql to 0.15
Relate to gluesql/gluesql#1438

* Use SQLParser to parse and replace placeholders.

* Use SQLParser for surrealdb demo.

* Transform the query by SQLParser.

* Tweaks

* Remove wasmtime example. (#2001)

* ci: Add additional targets.

* Remove proxy wasmtime example.

* Format

---------

Co-authored-by: 伊欧 <[email protected]>
Co-authored-by: 伊欧 <[email protected]>

Cargo.toml

@@ -16,7 +16,7 @@ keywords = ["async", "orm", "mysql", "postgres", "sqlite"]
 rust-version = "1.65"
 
 [package.metadata.docs.rs]
-features = ["default", "sqlx-all", "mock", "runtime-async-std-native-tls", "postgres-array", "sea-orm-internal"]
+features = ["default", "sqlx-all", "mock", "proxy", "runtime-async-std-native-tls", "postgres-array", "sea-orm-internal"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -76,6 +76,7 @@ default = [
 ]
 macros = ["sea-orm-macros/derive"]
 mock = []
+proxy = ["serde_json"]
 with-json = ["serde_json", "sea-query/with-json", "chrono?/serde", "time?/serde", "uuid?/serde", "sea-query-binder?/with-json", "sqlx?/json"]
 with-chrono = ["chrono", "sea-query/with-chrono", "sea-query-binder?/with-chrono", "sqlx?/chrono"]
 with-rust_decimal = ["rust_decimal", "sea-query/with-rust_decimal", "sea-query-binder?/with-rust_decimal", "sqlx?/rust_decimal"]

examples/proxy_gluesql_example/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "sea-orm-proxy-gluesql-example"
+version = "0.1.0"
+authors = ["Langyo <[email protected]>"]
+edition = "2021"
+publish = false
+
+[workspace]
+
+[dependencies]
+async-std = { version = "1.12", features = ["attributes", "tokio1"] }
+serde_json = { version = "1" }
+serde = { version = "1" }
+futures = { version = "0.3" }
+async-stream = { version = "0.3" }
+futures-util = { version = "0.3" }
+
+sqlparser = "0.40"
+sea-orm = { path = "../../", features = [
+  "proxy",
+  "debug-print",
+] }
+gluesql = { version = "0.15", default-features = false, features = [
+  "memory-storage",
+] }
+
+[dev-dependencies]
+smol = { version = "1.2" }
+smol-potat = { version = "1.1" }

examples/proxy_gluesql_example/README.md

@@ -0,0 +1,7 @@
+# SeaORM Proxy Demo for GlueSQL
+
+Run this demo for [GlueSQL](https://gluesql.org/) with the following command:
+
+```bash
+cargo run
+```

examples/proxy_gluesql_example/src/entity/mod.rs

@@ -0,0 +1 @@
+pub mod post;

examples/proxy_gluesql_example/src/entity/post.rs

@@ -0,0 +1,17 @@
+use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Deserialize, Serialize)]
+#[sea_orm(table_name = "posts")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: i64,
+
+    pub title: String,
+    pub text: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

examples/proxy_gluesql_example/src/main.rs

@@ -0,0 +1,190 @@
+//! Proxy connection example.
+
+#![deny(missing_docs)]
+
+mod entity;
+
+use std::{
+    collections::BTreeMap,
+    sync::{Arc, Mutex},
+};
+
+use gluesql::{memory_storage::MemoryStorage, prelude::Glue};
+use sea_orm::{
+    ActiveValue::Set, Database, DbBackend, DbErr, EntityTrait, ProxyDatabaseTrait, ProxyExecResult,
+    ProxyRow, Statement,
+};
+
+use entity::post::{ActiveModel, Entity};
+
+struct ProxyDb {
+    mem: Mutex<Glue<MemoryStorage>>,
+}
+
+impl std::fmt::Debug for ProxyDb {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ProxyDb").finish()
+    }
+}
+
+impl ProxyDatabaseTrait for ProxyDb {
+    fn query(&self, statement: Statement) -> Result<Vec<ProxyRow>, DbErr> {
+        println!("SQL query: {:?}", statement);
+        let sql = statement.sql.clone();
+
+        let mut ret: Vec<ProxyRow> = vec![];
+        async_std::task::block_on(async {
+            for payload in self.mem.lock().unwrap().execute(sql).await.unwrap().iter() {
+                match payload {
+                    gluesql::prelude::Payload::Select { labels, rows } => {
+                        for row in rows.iter() {
+                            let mut map = BTreeMap::new();
+                            for (label, column) in labels.iter().zip(row.iter()) {
+                                map.insert(
+                                    label.to_owned(),
+                                    match column {
+                                        gluesql::prelude::Value::I64(val) => {
+                                            sea_orm::Value::BigInt(Some(*val))
+                                        }
+                                        gluesql::prelude::Value::Str(val) => {
+                                            sea_orm::Value::String(Some(Box::new(val.to_owned())))
+                                        }
+                                        _ => unreachable!("Unsupported value: {:?}", column),
+                                    },
+                                );
+                            }
+                            ret.push(map.into());
+                        }
+                    }
+                    _ => unreachable!("Unsupported payload: {:?}", payload),
+                }
+            }
+        });
+
+        Ok(ret)
+    }
+
+    fn execute(&self, statement: Statement) -> Result<ProxyExecResult, DbErr> {
+        let sql = if let Some(values) = statement.values {
+            // Replace all the '?' with the statement values
+            use sqlparser::ast::{Expr, Value};
+            use sqlparser::dialect::GenericDialect;
+            use sqlparser::parser::Parser;
+
+            let dialect = GenericDialect {};
+            let mut ast = Parser::parse_sql(&dialect, statement.sql.as_str()).unwrap();
+            match &mut ast[0] {
+                sqlparser::ast::Statement::Insert {
+                    columns, source, ..
+                } => {
+                    for item in columns.iter_mut() {
+                        item.quote_style = Some('"');
+                    }
+
+                    if let Some(obj) = source {
+                        match &mut *obj.body {
+                            sqlparser::ast::SetExpr::Values(obj) => {
+                                for (mut item, val) in obj.rows[0].iter_mut().zip(values.0.iter()) {
+                                    match &mut item {
+                                        Expr::Value(item) => {
+                                            *item = match val {
+                                                sea_orm::Value::String(val) => {
+                                                    Value::SingleQuotedString(match val {
+                                                        Some(val) => val.to_string(),
+                                                        None => "".to_string(),
+                                                    })
+                                                }
+                                                sea_orm::Value::BigInt(val) => Value::Number(
+                                                    val.unwrap_or(0).to_string(),
+                                                    false,
+                                                ),
+                                                _ => todo!(),
+                                            };
+                                        }
+                                        _ => todo!(),
+                                    }
+                                }
+                            }
+                            _ => todo!(),
+                        }
+                    }
+                }
+                _ => todo!(),
+            }
+
+            let statement = &ast[0];
+            statement.to_string()
+        } else {
+            statement.sql
+        };
+
+        println!("SQL execute: {}", sql);
+        async_std::task::block_on(async {
+            self.mem.lock().unwrap().execute(sql).await.unwrap();
+        });
+
+        Ok(ProxyExecResult {
+            last_insert_id: 1,
+            rows_affected: 1,
+        })
+    }
+}
+
+#[async_std::main]
+async fn main() {
+    let mem = MemoryStorage::default();
+    let mut glue = Glue::new(mem);
+
+    glue.execute(
+        r#"
+            CREATE TABLE IF NOT EXISTS posts (
+                id INTEGER PRIMARY KEY,
+                title TEXT NOT NULL,
+                text TEXT NOT NULL
+            )
+        "#,
+    )
+    .await
+    .unwrap();
+
+    let db = Database::connect_proxy(
+        DbBackend::Sqlite,
+        Arc::new(Mutex::new(Box::new(ProxyDb {
+            mem: Mutex::new(glue),
+        }))),
+    )
+    .await
+    .unwrap();
+
+    println!("Initialized");
+
+    let data = ActiveModel {
+        id: Set(11),
+        title: Set("Homo".to_owned()),
+        text: Set("いいよ、来いよ".to_owned()),
+    };
+    Entity::insert(data).exec(&db).await.unwrap();
+    let data = ActiveModel {
+        id: Set(45),
+        title: Set("Homo".to_owned()),
+        text: Set("そうだよ".to_owned()),
+    };
+    Entity::insert(data).exec(&db).await.unwrap();
+    let data = ActiveModel {
+        id: Set(14),
+        title: Set("Homo".to_owned()),
+        text: Set("悔い改めて".to_owned()),
+    };
+    Entity::insert(data).exec(&db).await.unwrap();
+
+    let list = Entity::find().all(&db).await.unwrap().to_vec();
+    println!("Result: {:?}", list);
+}
+
+#[cfg(test)]
+mod tests {
+    #[smol_potat::test]
+    async fn try_run() {
+        crate::main()
+    }
+}

examples/proxy_surrealdb_example/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "sea-orm-proxy-surrealdb-example"
+version = "0.1.0"
+authors = ["Langyo <[email protected]>"]
+edition = "2021"
+publish = false
+
+[workspace]
+
+[dependencies]
+async-std = { version = "1.12", features = ["attributes", "tokio1"] }
+serde_json = { version = "1" }
+serde = { version = "1" }
+futures = { version = "0.3" }
+async-stream = { version = "0.3" }
+futures-util = { version = "0.3" }
+
+sqlparser = "0.40"
+sea-orm = { path = "../../", features = [
+  "proxy",
+  "debug-print",
+] }
+surrealdb = { version = "1", features = ["kv-mem"] }
+
+[dev-dependencies]
+smol = { version = "1.2" }
+smol-potat = { version = "1.1" }

examples/proxy_surrealdb_example/README.md

@@ -0,0 +1,7 @@
+# SeaORM Proxy Demo for SurrealDB
+
+Run this demo for [SurrealDB](https://surrealdb.com/) with the following command:
+
+```bash
+cargo run
+```

examples/proxy_surrealdb_example/src/entity/mod.rs

@@ -0,0 +1 @@
+pub mod post;

examples/proxy_surrealdb_example/src/entity/post.rs

@@ -0,0 +1,17 @@
+use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Deserialize, Serialize)]
+#[sea_orm(table_name = "posts")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: String,
+
+    pub title: String,
+    pub text: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}