From 699711f6d018515ba9273cce41330cf18887fbc0 Mon Sep 17 00:00:00 2001 From: Chris Tsang Date: Tue, 9 Jan 2024 00:10:40 +0000 Subject: [PATCH 1/6] Rework SQLite type mapping --- src/sqlite/def/column.rs | 4 +- src/sqlite/def/table.rs | 5 +- src/sqlite/def/types.rs | 205 ++++++++-------------------------- tests/live/sqlite/src/main.rs | 92 +++++++++------ 4 files changed, 110 insertions(+), 196 deletions(-) diff --git a/src/sqlite/def/column.rs b/src/sqlite/def/column.rs index da6ae80c..00203c83 100644 --- a/src/sqlite/def/column.rs +++ b/src/sqlite/def/column.rs @@ -1,4 +1,4 @@ -use super::{DefaultType, Type}; +use super::{parse_type, DefaultType, Type}; use sea_query::{ foreign_key::ForeignKeyAction as SeaQueryForeignKeyAction, Alias, Index, IndexCreateStatement, }; @@ -28,7 +28,7 @@ impl ColumnInfo { Ok(ColumnInfo { cid: row.get(0), name: row.get(1), - r#type: Type::to_type(row.get(2))?, + r#type: parse_type(row.get(2))?, not_null: col_not_null != 0, default_value: if default_value == "NULL" { DefaultType::Null diff --git a/src/sqlite/def/table.rs b/src/sqlite/def/table.rs index 1837f7a9..9ebdeacb 100644 --- a/src/sqlite/def/table.rs +++ b/src/sqlite/def/table.rs @@ -224,7 +224,8 @@ impl TableDef { new_table.table(Alias::new(&self.name)); self.columns.iter().for_each(|column_info| { - let mut new_column = ColumnDef::new(Alias::new(&column_info.name)); + let mut new_column = + ColumnDef::new_with_type(Alias::new(&column_info.name), column_info.r#type.clone()); if column_info.not_null { new_column.not_null(); } @@ -235,8 +236,6 @@ impl TableDef { primary_keys.push(column_info.name.clone()); } - column_info.r#type.write_type(&mut new_column); - match &column_info.default_value { DefaultType::Integer(integer_value) => { new_column.default(Value::Int(Some(*integer_value))); diff --git a/src/sqlite/def/types.rs b/src/sqlite/def/types.rs index 67f2e7ef..ba3c90ab 100644 --- a/src/sqlite/def/types.rs +++ b/src/sqlite/def/types.rs @@ -1,166 +1,59 @@ -use sea_query::ColumnDef; +use sea_query::{BlobSize, ColumnType}; use std::num::ParseIntError; -/// A list of the offical SQLite types as outline at the official [SQLite Docs](https://www.sqlite.org/datatype3.html) -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum Type { - Int, - Integer, - TinyInt, - SmallInt, - MediumInt, - BigInt, - UnsignedBigInt, - Int2, - Int8, - Character { length: u8 }, - VarChar { length: u8 }, - VaryingCharacter { length: u8 }, - Nchar { length: u8 }, - NativeCharacter { length: u8 }, - NvarChar { length: u8 }, - Text, - Clob, - Blob, //No datatype specified - Real, - Double, - DoublePrecision, - Float, - Numeric, - Decimal { integral: u8, fractional: u8 }, - Boolean, - Date, - DateTime, - Timestamp, -} - -impl Type { - /// Maps a string type from an `SqliteRow` into a [Type] - pub fn to_type(data_type: &str) -> Result { - let data_type = data_type.to_uppercase(); - - let split_type: Vec<&str> = data_type.split('(').collect(); - let type_result = match split_type[0] { - "INT" => Type::Int, - "INTEGER" => Type::Integer, - "TINY INT" | "TINYINT" => Type::TinyInt, - "SMALL INT" | "SMALLINT" => Type::SmallInt, - "MEDIUM INT" | "MEDIUMINT" => Type::MediumInt, - "BIG INT" | "BIGINT" => Type::BigInt, - "UNSIGNED INT" | "UNSIGNEDBIGINT" => Type::UnsignedBigInt, - "INT2" => Type::Int2, - "INT8" => Type::Int8, - "TEXT" => Type::Text, - "CLOB" => Type::Clob, - "BLOB" => Type::Blob, - "REAL" => Type::Real, - "DOUBLE" => Type::Double, - "DOUBLE PRECISION" => Type::DoublePrecision, - "FLOAT" => Type::Float, - "NUMERIC" => Type::Numeric, - "DECIMAL" => { - let decimals = split_type[1].chars().collect::>(); - - let integral = decimals[0].to_string().parse::()?; - let fractional = decimals[2].to_string().parse::()?; +pub type Type = ColumnType; - Type::Decimal { - integral, - fractional, +pub fn parse_type(data_type: &str) -> Result { + let mut type_name = data_type; + let mut parts: Vec = Vec::new(); + if let Some((prefix, suffix)) = data_type.split_once('(') { + if let Some(suffix) = suffix.strip_suffix(')') { + type_name = prefix; + for part in suffix.split(",") { + if let Ok(part) = part.trim().parse() { + parts.push(part); + } else { + break; } } - "BOOLEAN" => Type::Boolean, - "DATE" => Type::Date, - "DATETIME" => Type::DateTime, - "TIMESTAMP" => Type::Timestamp, - _ => Type::variable_types(&split_type)?, - }; - - Ok(type_result) - } - - /// Write a [Type] to a [ColumnDef] - pub fn write_type(&self, column_def: &mut ColumnDef) { - match self { - Self::Int | Self::Integer | Self::MediumInt | Self::Int2 | Self::Int8 => { - column_def.integer(); - } - Self::TinyInt => { - column_def.tiny_integer(); - } - Self::SmallInt => { - column_def.small_integer(); - } - Self::BigInt | Self::UnsignedBigInt => { - column_def.big_integer(); - } - Self::Character { .. } - | Self::VarChar { .. } - | Self::VaryingCharacter { .. } - | Self::Nchar { .. } - | Self::NativeCharacter { .. } - | Self::NvarChar { .. } - | Self::Text - | Self::Clob => { - column_def.string(); - } - Self::Blob => { - column_def.binary(); - } - Self::Real | Self::Double | Self::DoublePrecision | Self::Float | Self::Numeric => { - column_def.double(); - } - Self::Decimal { - integral, - fractional, - } => { - column_def.decimal_len((*integral) as u32, (*fractional) as u32); - } - Self::Boolean => { - column_def.boolean(); - } - Self::Date => { - column_def.date(); - } - Self::DateTime => { - column_def.date_time(); - } - Self::Timestamp => { - column_def.timestamp(); - } } } - - #[allow(dead_code)] - fn concat_type(&self, type_name: &str, length: &u8) -> String { - let mut value = String::default(); - value.push_str(type_name); - value.push('('); - value.push_str(&length.to_string()); - value.push(')'); - - value - } - - fn variable_types(split_type: &[&str]) -> Result { - let length = if !split_type.len() == 1 { - let maybe_size = split_type[1].replace(')', ""); - maybe_size.parse::()? + Ok(match type_name.to_lowercase().as_str() { + "char" => ColumnType::Char(parts.into_iter().next()), + "varchar" => ColumnType::String(parts.into_iter().next()), + "text" => ColumnType::Text, + "tinyint" => ColumnType::TinyInteger, + "smallint" => ColumnType::SmallInteger, + "integer" => ColumnType::Integer, + "bigint" => ColumnType::BigInteger, + "float" => ColumnType::Float, + "double" => ColumnType::Double, + "decimal" => ColumnType::Decimal(if parts.len() == 2 { + Some((parts[0], parts[1])) } else { - 255_u8 - }; - - let type_result = match split_type[0] { - "VARCHAR" => Type::VarChar { length }, - "CHARACTER" => Type::Character { length }, - "VARYING CHARACTER" => Type::VaryingCharacter { length }, - "NCHAR" => Type::Nchar { length }, - "NATIVE CHARACTER" => Type::NativeCharacter { length }, - "NVARCHAR" => Type::NvarChar { length }, - _ => Type::Blob, - }; - Ok(type_result) - } + None + }), + "datetime_text" => ColumnType::DateTime, + "timestamp_text" => ColumnType::Timestamp, + "timestamp_with_timezone_text" => ColumnType::TimestampWithTimeZone, + "time_text" => ColumnType::Time, + "date_text" => ColumnType::Date, + "tinyblob" => ColumnType::Binary(BlobSize::Tiny), + "mediumblob" => ColumnType::Binary(BlobSize::Medium), + "longblob" => ColumnType::Binary(BlobSize::Long), + "blob" => ColumnType::Binary(BlobSize::Blob(parts.into_iter().next())), + "varbinary_blob" if parts.len() == 1 => ColumnType::VarBinary(parts[0]), + "boolean" => ColumnType::Boolean, + "money" => ColumnType::Money(if parts.len() == 2 { + Some((parts[0], parts[1])) + } else { + None + }), + "json_text" => ColumnType::Json, + "jsonb_text" => ColumnType::JsonBinary, + "uuid_text" => ColumnType::Uuid, + _ => ColumnType::custom(data_type), + }) } /// The default types for an SQLite `dflt_value` @@ -170,6 +63,6 @@ pub enum DefaultType { Float(f32), String(String), Null, - Unspecified, //FIXME For other types + Unspecified, CurrentTimestamp, } diff --git a/tests/live/sqlite/src/main.rs b/tests/live/sqlite/src/main.rs index 1743b731..d85d59a7 100644 --- a/tests/live/sqlite/src/main.rs +++ b/tests/live/sqlite/src/main.rs @@ -4,8 +4,8 @@ use sqlx::SqlitePool; use std::collections::HashMap; use sea_schema::sea_query::{ - Alias, ColumnDef, Expr, ForeignKey, ForeignKeyAction, ForeignKeyCreateStatement, Index, Query, - SqliteQueryBuilder, Table, TableCreateStatement, TableRef, + Alias, BlobSize, ColumnDef, Expr, ForeignKey, ForeignKeyAction, ForeignKeyCreateStatement, + Index, Query, SqliteQueryBuilder, Table, TableCreateStatement, TableRef, }; use sea_schema::sqlite::{ def::TableDef, @@ -15,10 +15,7 @@ use sea_schema::sqlite::{ #[cfg_attr(test, async_std::test)] #[cfg_attr(not(test), async_std::main)] async fn main() -> DiscoveryResult<()> { - // env_logger::builder() - // .filter_level(log::LevelFilter::Debug) - // .is_test(true) - // .init(); + env_logger::init(); test_001().await?; test_002().await?; @@ -52,7 +49,7 @@ async fn test_001() -> DiscoveryResult<()> { .table(Alias::new("Programming_Langs")) .col( ColumnDef::new(Alias::new("Name")) - .custom(Alias::new("INTEGER")) + .integer() .not_null() .auto_increment() .primary_key(), @@ -65,7 +62,7 @@ async fn test_001() -> DiscoveryResult<()> { ) .col( ColumnDef::new(Alias::new("SemVer")) - .custom(Alias::new("VARCHAR(255)")) + .string_len(255) .not_null(), ) .col( @@ -126,17 +123,13 @@ async fn test_001() -> DiscoveryResult<()> { // Tests foreign key discovery let table_create_suppliers = Table::create() .table(Alias::new("suppliers")) - .col(ColumnDef::new(Alias::new("supplier_id")).custom(Alias::new("INTEGER"))) + .col(ColumnDef::new(Alias::new("supplier_id")).integer()) .col( ColumnDef::new(Alias::new("supplier_name")) - .custom(Alias::new("TEXT")) - .not_null(), - ) - .col( - ColumnDef::new(Alias::new("group_id")) - .custom(Alias::new("INTEGER")) + .text() .not_null(), ) + .col(ColumnDef::new(Alias::new("group_id")).integer().not_null()) .primary_key(Index::create().col(Alias::new("supplier_id"))) .foreign_key( ForeignKeyCreateStatement::new() @@ -150,12 +143,8 @@ async fn test_001() -> DiscoveryResult<()> { let table_create_supplier_groups = Table::create() .table(Alias::new("supplier_groups")) - .col(ColumnDef::new(Alias::new("group_id")).custom(Alias::new("INTEGER"))) - .col( - ColumnDef::new(Alias::new("group_name")) - .custom(Alias::new("TEXT")) - .not_null(), - ) + .col(ColumnDef::new(Alias::new("group_id")).integer()) + .col(ColumnDef::new(Alias::new("group_name")).text().not_null()) .primary_key(Index::create().col(Alias::new("group_id"))) .to_owned(); @@ -215,13 +204,6 @@ async fn test_001() -> DiscoveryResult<()> { let schema = SchemaDiscovery::new(sqlite_pool.clone()).discover().await?; - let convert_column_types = |str: String| { - str.replace("INTEGER", "integer") - .replace("INT8", "integer") - .replace("TEXT", "text") - .replace("VARCHAR(255)", "text") - .replace("DATETIME", "text") - }; let expected_sql = [ create_table.to_string(SqliteQueryBuilder), create_table_inventors.to_string(SqliteQueryBuilder), @@ -229,12 +211,11 @@ async fn test_001() -> DiscoveryResult<()> { table_create_suppliers.to_string(SqliteQueryBuilder), ] .into_iter() - .map(convert_column_types) .collect::>(); assert_eq!(schema.tables.len(), expected_sql.len()); for (i, table) in schema.tables.into_iter().enumerate() { - let sql = convert_column_types(table.write().to_string(SqliteQueryBuilder)); + let sql = table.write().to_string(SqliteQueryBuilder); if sql == expected_sql[i] { println!("[OK] {sql}"); } @@ -273,6 +254,7 @@ async fn test_002() -> DiscoveryResult<()> { create_lineitem_table(), create_parent_table(), create_child_table(), + create_strange_table(), ]; for tbl_create_stmt in tbl_create_stmts.iter() { @@ -301,12 +283,12 @@ async fn test_002() -> DiscoveryResult<()> { [ r#"CREATE TABLE "order" ("#, r#""id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,"#, - r#""total" real,"#, + r#""total" decimal,"#, r#""bakery_id" integer NOT NULL,"#, r#""customer_id" integer NOT NULL,"#, - r#""placed_at" text NOT NULL DEFAULT CURRENT_TIMESTAMP,"#, - r#""updated" text NOT NULL DEFAULT '2023-06-07 16:24:00',"#, - r#""net_weight" real NOT NULL DEFAULT 10.05,"#, + r#""placed_at" datetime_text NOT NULL DEFAULT CURRENT_TIMESTAMP,"#, + r#""updated" datetime_text NOT NULL DEFAULT '2023-06-07 16:24:00',"#, + r#""net_weight" double NOT NULL DEFAULT 10.05,"#, r#""priority" integer NOT NULL DEFAULT 5,"#, r#"FOREIGN KEY ("customer_id") REFERENCES "customer" ("id") ON DELETE CASCADE ON UPDATE CASCADE,"#, r#"FOREIGN KEY ("bakery_id") REFERENCES "bakery" ("id") ON DELETE CASCADE ON UPDATE CASCADE"#, @@ -316,7 +298,7 @@ async fn test_002() -> DiscoveryResult<()> { [ r#"CREATE TABLE "lineitem" ("#, r#""id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,"#, - r#""price" real,"#, + r#""price" decimal,"#, r#""quantity" integer,"#, r#""order_id" integer NOT NULL,"#, r#""cake_id" integer NOT NULL,"#, @@ -566,3 +548,43 @@ fn create_child_table() -> TableCreateStatement { ) .to_owned() } + +fn create_strange_table() -> TableCreateStatement { + Table::create() + .table(Alias::new("strange")) + .col( + ColumnDef::new(Alias::new("id")) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Alias::new("int1")).integer()) + .col(ColumnDef::new(Alias::new("int2")).tiny_integer()) + .col(ColumnDef::new(Alias::new("int3")).small_integer()) + .col(ColumnDef::new(Alias::new("int4")).big_integer()) + .col(ColumnDef::new(Alias::new("string1")).string()) + .col(ColumnDef::new(Alias::new("string2")).string_len(24)) + .col(ColumnDef::new(Alias::new("char1")).char()) + .col(ColumnDef::new(Alias::new("char2")).char_len(24)) + .col(ColumnDef::new(Alias::new("text_col")).text()) + .col(ColumnDef::new(Alias::new("json_col")).json()) + .col(ColumnDef::new(Alias::new("uuid_col")).uuid()) + .col(ColumnDef::new(Alias::new("decimal1")).decimal()) + .col(ColumnDef::new(Alias::new("decimal2")).decimal_len(12, 4)) + .col(ColumnDef::new(Alias::new("money1")).money()) + .col(ColumnDef::new(Alias::new("money2")).money_len(12, 4)) + .col(ColumnDef::new(Alias::new("float_col")).float()) + .col(ColumnDef::new(Alias::new("double_col")).double()) + .col(ColumnDef::new(Alias::new("date_col")).date()) + .col(ColumnDef::new(Alias::new("time_col")).time()) + .col(ColumnDef::new(Alias::new("datetime_col")).date_time()) + .col(ColumnDef::new(Alias::new("boolean_col")).boolean()) + .col(ColumnDef::new(Alias::new("binary1")).binary()) + .col(ColumnDef::new(Alias::new("binary2")).binary_len(1024)) + .col(ColumnDef::new(Alias::new("binary3")).var_binary(1024)) + .col(ColumnDef::new(Alias::new("binary4")).blob(BlobSize::Tiny)) + .col(ColumnDef::new(Alias::new("binary5")).blob(BlobSize::Medium)) + .col(ColumnDef::new(Alias::new("binary6")).blob(BlobSize::Long)) + .to_owned() +} From 13dacb70d1594d441aec3143bca6c872308b85f2 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Jan 2024 16:41:28 +0800 Subject: [PATCH 2/6] Update SeaQuery dep --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6bec7da4..318cd78b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,8 +36,8 @@ path = "src/lib.rs" [dependencies] futures = { version = "0.3", default-features = false, optional = true, features = ["alloc"] } sea-schema-derive = { version = "0.2.0", path = "sea-schema-derive", default-features = false } -sea-query = { version = "0.30.0", default-features = false, features = ["derive"] } -sea-query-binder = { version = "0.5.0", default-features = false, optional = true } +sea-query = { version = "0.30.0", git = "https://github.com/seaql/sea-query", branch = "sqlite-types", default-features = false, features = ["derive"] } +sea-query-binder = { version = "0.5.0", git = "https://github.com/seaql/sea-query", branch = "sqlite-types", default-features = false, optional = true } serde = { version = "1", default-features = false, optional = true, features = ["derive"] } sqlx = { version = "0.7", default-features = false, optional = true } log = { version = "0.4", default-features = false, optional = true } From 02dcde1f4cdb61be0762dfb1ee73db95c4c949be Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Tue, 23 Jan 2024 18:23:03 +0800 Subject: [PATCH 3/6] sqlite: decimal_text --- src/sqlite/def/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite/def/types.rs b/src/sqlite/def/types.rs index ba3c90ab..d85bf684 100644 --- a/src/sqlite/def/types.rs +++ b/src/sqlite/def/types.rs @@ -28,7 +28,7 @@ pub fn parse_type(data_type: &str) -> Result { "bigint" => ColumnType::BigInteger, "float" => ColumnType::Float, "double" => ColumnType::Double, - "decimal" => ColumnType::Decimal(if parts.len() == 2 { + "decimal_text" => ColumnType::Decimal(if parts.len() == 2 { Some((parts[0], parts[1])) } else { None From 73a9231b9707d55321afcee4308492ee7f522813 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Wed, 24 Jan 2024 11:27:14 +0800 Subject: [PATCH 4/6] clippy --- src/sqlite/def/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite/def/types.rs b/src/sqlite/def/types.rs index d85bf684..f7313d59 100644 --- a/src/sqlite/def/types.rs +++ b/src/sqlite/def/types.rs @@ -9,7 +9,7 @@ pub fn parse_type(data_type: &str) -> Result { if let Some((prefix, suffix)) = data_type.split_once('(') { if let Some(suffix) = suffix.strip_suffix(')') { type_name = prefix; - for part in suffix.split(",") { + for part in suffix.split(',') { if let Ok(part) = part.trim().parse() { parts.push(part); } else { From c9f574785c79f83f2bf1c53e91637aa5548c4580 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Wed, 24 Jan 2024 11:30:12 +0800 Subject: [PATCH 5/6] fix test cases --- tests/live/sqlite/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/live/sqlite/src/main.rs b/tests/live/sqlite/src/main.rs index d85d59a7..a38b7c1d 100644 --- a/tests/live/sqlite/src/main.rs +++ b/tests/live/sqlite/src/main.rs @@ -283,7 +283,7 @@ async fn test_002() -> DiscoveryResult<()> { [ r#"CREATE TABLE "order" ("#, r#""id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,"#, - r#""total" decimal,"#, + r#""total" decimal_text,"#, r#""bakery_id" integer NOT NULL,"#, r#""customer_id" integer NOT NULL,"#, r#""placed_at" datetime_text NOT NULL DEFAULT CURRENT_TIMESTAMP,"#, @@ -298,7 +298,7 @@ async fn test_002() -> DiscoveryResult<()> { [ r#"CREATE TABLE "lineitem" ("#, r#""id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,"#, - r#""price" decimal,"#, + r#""price" decimal_text,"#, r#""quantity" integer,"#, r#""order_id" integer NOT NULL,"#, r#""cake_id" integer NOT NULL,"#, From c69430c2c0403f5b6878adc25ca52f0b4ff9ca23 Mon Sep 17 00:00:00 2001 From: Billy Chan Date: Wed, 24 Jan 2024 17:12:51 +0800 Subject: [PATCH 6/6] sqlite: decimal -> real --- src/sqlite/def/types.rs | 2 +- tests/live/sqlite/src/main.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sqlite/def/types.rs b/src/sqlite/def/types.rs index f7313d59..7aa6576f 100644 --- a/src/sqlite/def/types.rs +++ b/src/sqlite/def/types.rs @@ -28,7 +28,7 @@ pub fn parse_type(data_type: &str) -> Result { "bigint" => ColumnType::BigInteger, "float" => ColumnType::Float, "double" => ColumnType::Double, - "decimal_text" => ColumnType::Decimal(if parts.len() == 2 { + "real" => ColumnType::Decimal(if parts.len() == 2 { Some((parts[0], parts[1])) } else { None diff --git a/tests/live/sqlite/src/main.rs b/tests/live/sqlite/src/main.rs index a38b7c1d..e75dacad 100644 --- a/tests/live/sqlite/src/main.rs +++ b/tests/live/sqlite/src/main.rs @@ -283,7 +283,7 @@ async fn test_002() -> DiscoveryResult<()> { [ r#"CREATE TABLE "order" ("#, r#""id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,"#, - r#""total" decimal_text,"#, + r#""total" real,"#, r#""bakery_id" integer NOT NULL,"#, r#""customer_id" integer NOT NULL,"#, r#""placed_at" datetime_text NOT NULL DEFAULT CURRENT_TIMESTAMP,"#, @@ -298,7 +298,7 @@ async fn test_002() -> DiscoveryResult<()> { [ r#"CREATE TABLE "lineitem" ("#, r#""id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,"#, - r#""price" decimal_text,"#, + r#""price" real,"#, r#""quantity" integer,"#, r#""order_id" integer NOT NULL,"#, r#""cake_id" integer NOT NULL,"#,