From 7572da13ef88b854c74b6e21fe71bbff09bbd55f Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 24 Sep 2024 14:51:22 +0200 Subject: [PATCH 01/24] feat(sqlite): add support for serde_json::Value using the Json and Jsonbtypes --- diesel/src/pg/types/mod.rs | 86 ----------------- diesel/src/sql_types/mod.rs | 99 +++++++++++++++++++ diesel/src/sqlite/types/json.rs | 164 ++++++++++++++++++++++++++++++++ diesel/src/sqlite/types/mod.rs | 1 + 4 files changed, 264 insertions(+), 86 deletions(-) create mode 100644 diesel/src/sqlite/types/json.rs diff --git a/diesel/src/pg/types/mod.rs b/diesel/src/pg/types/mod.rs index 14b8ed9b0fc1..513627018c38 100644 --- a/diesel/src/pg/types/mod.rs +++ b/diesel/src/pg/types/mod.rs @@ -310,92 +310,6 @@ pub mod sql_types { #[doc(hidden)] pub type Bpchar = crate::sql_types::VarChar; - /// The [`jsonb`] SQL type. This type can only be used with `feature = - /// "serde_json"` - /// - /// `jsonb` offers [several advantages][adv] over regular JSON: - /// - /// > There are two JSON data types: `json` and `jsonb`. They accept almost - /// > identical sets of values as input. The major practical difference - /// > is one of efficiency. The `json` data type stores an exact copy of - /// > the input text, which processing functions must reparse on each - /// > execution; while `jsonb` data is stored in a decomposed binary format - /// > that makes it slightly slower to input due to added conversion - /// > overhead, but significantly faster to process, since no reparsing - /// > is needed. `jsonb` also supports indexing, which can be a significant - /// > advantage. - /// > - /// > ...In general, most applications should prefer to store JSON data as - /// > `jsonb`, unless there are quite specialized needs, such as legacy - /// > assumptions about ordering of object keys. - /// - /// [adv]: https://www.postgresql.org/docs/current/static/datatype-json.html - /// - /// ### [`ToSql`] impls - /// - /// - [`serde_json::Value`] - /// - /// ### [`FromSql`] impls - /// - /// - [`serde_json::Value`] - /// - /// [`ToSql`]: crate::serialize::ToSql - /// [`FromSql`]: crate::deserialize::FromSql - /// [`jsonb`]: https://www.postgresql.org/docs/current/datatype-json.html - #[cfg_attr( - feature = "serde_json", - doc = "[`serde_json::Value`]: serde_json::value::Value" - )] - #[cfg_attr( - not(feature = "serde_json"), - doc = "[`serde_json::Value`]: https://docs.rs/serde_json/1.0.64/serde_json/value/enum.Value.html" - )] - /// - /// # Examples - /// - /// ```rust - /// # #![allow(dead_code)] - /// # include!("../../doctest_setup.rs"); - /// # - /// table! { - /// contacts { - /// id -> Integer, - /// name -> VarChar, - /// address -> Jsonb, - /// } - /// } - /// - /// # #[cfg(feature = "serde_json")] - /// # fn main() -> Result<(), Box> { - /// # use diesel::insert_into; - /// # use self::contacts::dsl::*; - /// # let connection = &mut connection_no_data(); - /// # diesel::sql_query("CREATE TABLE contacts ( - /// # id SERIAL PRIMARY KEY, - /// # name VARCHAR NOT NULL, - /// # address JSONB NOT NULL - /// # )").execute(connection)?; - /// let santas_address: serde_json::Value = serde_json::from_str(r#"{ - /// "street": "Article Circle Expressway 1", - /// "city": "North Pole", - /// "postcode": "99705", - /// "state": "Alaska" - /// }"#)?; - /// let inserted_address = insert_into(contacts) - /// .values((name.eq("Claus"), address.eq(&santas_address))) - /// .returning(address) - /// .get_result::(connection)?; - /// assert_eq!(santas_address, inserted_address); - /// # Ok(()) - /// # } - /// # #[cfg(not(feature = "serde_json"))] - /// # fn main() {} - /// ``` - #[cfg(feature = "postgres_backend")] - #[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] - #[diesel(postgres_type(oid = 3802, array_oid = 3807))] - pub struct Jsonb; - /// The PostgreSQL [Money](https://www.postgresql.org/docs/current/static/datatype-money.html) type. /// /// ### [`ToSql`] impls diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index 7448791b77b1..fb1717b318c4 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -395,11 +395,110 @@ pub struct Timestamp; /// [`ToSql`]: /serialize/trait.ToSql.html /// [`FromSql`]: /deserialize/trait.FromSql.html /// [`serde_json::Value`]: /../serde_json/value/enum.Value.html +#[cfg(all( + any( + feature = "postgres_backend", + feature = "sqlite", + feature = "mysql_backend" + ), + feature = "serde_json" +))] #[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] #[diesel(postgres_type(oid = 114, array_oid = 199))] #[diesel(mysql_type(name = "String"))] +#[diesel(sqlite_type(name = "Text"))] pub struct Json; +/// The [`jsonb`] SQL type. This type can only be used with `feature = +/// "serde_json"` +/// +/// `jsonb` offers [several advantages][adv] over regular JSON: +/// +/// > There are two JSON data types: `json` and `jsonb`. They accept almost +/// > identical sets of values as input. The major practical difference +/// > is one of efficiency. The `json` data type stores an exact copy of +/// > the input text, which processing functions must reparse on each +/// > execution; while `jsonb` data is stored in a decomposed binary format +/// > that makes it slightly slower to input due to added conversion +/// > overhead, but significantly faster to process, since no reparsing +/// > is needed. `jsonb` also supports indexing, which can be a significant +/// > advantage. +/// > +/// > ...In general, most applications should prefer to store JSON data as +/// > `jsonb`, unless there are quite specialized needs, such as legacy +/// > assumptions about ordering of object keys. +/// +/// [adv]: https://www.postgresql.org/docs/current/static/datatype-json.html +/// +/// ### [`ToSql`] impls +/// +/// - [`serde_json::Value`] +/// +/// ### [`FromSql`] impls +/// +/// - [`serde_json::Value`] +/// +/// [`ToSql`]: crate::serialize::ToSql +/// [`FromSql`]: crate::deserialize::FromSql +/// [`jsonb`]: https://www.postgresql.org/docs/current/datatype-json.html +#[cfg_attr( + feature = "serde_json", + doc = "[`serde_json::Value`]: serde_json::value::Value" +)] +#[cfg_attr( + not(feature = "serde_json"), + doc = "[`serde_json::Value`]: https://docs.rs/serde_json/1.0.64/serde_json/value/enum.Value.html" +)] +/// +/// # Examples +/// +/// ```rust +/// # #![allow(dead_code)] +/// # include!("../../doctest_setup.rs"); +/// # +/// table! { +/// contacts { +/// id -> Integer, +/// name -> VarChar, +/// address -> Jsonb, +/// } +/// } +/// +/// # #[cfg(feature = "serde_json")] +/// # fn main() -> Result<(), Box> { +/// # use diesel::insert_into; +/// # use self::contacts::dsl::*; +/// # let connection = &mut connection_no_data(); +/// # diesel::sql_query("CREATE TABLE contacts ( +/// # id SERIAL PRIMARY KEY, +/// # name VARCHAR NOT NULL, +/// # address JSONB NOT NULL +/// # )").execute(connection)?; +/// let santas_address: serde_json::Value = serde_json::from_str(r#"{ +/// "street": "Article Circle Expressway 1", +/// "city": "North Pole", +/// "postcode": "99705", +/// "state": "Alaska" +/// }"#)?; +/// let inserted_address = insert_into(contacts) +/// .values((name.eq("Claus"), address.eq(&santas_address))) +/// .returning(address) +/// .get_result::(connection)?; +/// assert_eq!(santas_address, inserted_address); +/// # Ok(()) +/// # } +/// # #[cfg(not(feature = "serde_json"))] +/// # fn main() {} +/// ``` +#[cfg(all( + any(feature = "postgres_backend", feature = "sqlite"), + feature = "serde_json" +))] +#[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] +#[diesel(postgres_type(oid = 3802, array_oid = 3807))] +#[diesel(sqlite_type(name = "Binary"))] +pub struct Jsonb; + /// The nullable SQL type. /// /// This wraps another SQL type to indicate that it can be null. diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs new file mode 100644 index 000000000000..630eadc5d277 --- /dev/null +++ b/diesel/src/sqlite/types/json.rs @@ -0,0 +1,164 @@ +extern crate serde_json; + +use crate::deserialize::{self, FromSql}; +use crate::serialize::{self, IsNull, Output, ToSql}; +use crate::sql_types; +use crate::sqlite::{Sqlite, SqliteValue}; + +#[cfg(all(feature = "sqlite", feature = "serde_json"))] +impl FromSql for serde_json::Value { + fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { + serde_json::from_str(value.read_text()).map_err(|_| "Invalid Json".into()) + } +} + +#[cfg(all(feature = "sqlite", feature = "serde_json"))] +impl ToSql for serde_json::Value { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { + out.set_value(serde_json::to_string(self)?); + Ok(IsNull::No) + } +} + +#[cfg(all(feature = "sqlite", feature = "serde_json"))] +impl FromSql for serde_json::Value { + fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { + let bytes = value.read_blob(); + if bytes[0] != 1 { + return Err("Unsupported JSONB encoding version".into()); + } + serde_json::from_slice(&bytes[1..]).map_err(|_| "Invalid Json".into()) + } +} + +#[cfg(all(feature = "sqlite", feature = "serde_json"))] +impl ToSql for serde_json::Value { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { + out.set_value(serde_json::to_string(self)?.into_bytes()); + Ok(IsNull::No) + } +} + +#[cfg(test)] +mod tests { + use crate::deserialize::FromSql; + use crate::dsl::sql; + use crate::prelude::*; + use crate::select; + use crate::serialize::{Output, ToSql}; + use crate::sql_types; + use crate::sql_types::{Json, Text}; + use crate::sqlite::connection::SqliteBindCollector; + use crate::sqlite::Sqlite; + use crate::sqlite::SqliteBindValue; + use crate::sqlite::SqliteValue; + use crate::test_helpers::connection; + use serde_json::json; + + // #[test] + // fn json_to_sql() { + // let buffer = SqliteBindValue::from(0i32); + // let mut out = Output::<'_, 'static, Sqlite>::test(buffer); + // let test_json = serde_json::Value::Bool(true); + // ToSql::::to_sql(&test_json, &mut out).unwrap(); + // assert_eq!(buffer.inner, SqliteBindValue::from(1i32).inner); + // } + + // #[test] + // fn json_to_sql() { + // crate::table! { + // #[allow(unused_parens)] + // test_insert_json_into_table_as_text(id) { + // id -> Integer, + // json -> Text, + // } + // } + // let conn = &mut connection(); + // crate::sql_query( + // "CREATE TABLE test_insert_json_into_table_as_text(id INTEGER PRIMARY KEY, json TEXT);", + // ) + // .execute(conn) + // .unwrap(); + + // let value = json!(true); + + // crate::insert_into(test_insert_json_into_table_as_text::table) + // .values(( + // test_insert_json_into_table_as_text::id.eq(1), + // test_insert_json_into_table_as_text::json.eq(value), + // )) + // .execute(conn) + // .unwrap(); + // } + + #[test] + fn some_json_from_sql() { + let input_json = b"true"; + let output_json: serde_json::Value = + FromSql::::from_sql(SqliteValue::for_test(input_json)) + .unwrap(); + assert_eq!(output_json, serde_json::Value::Bool(true)); + } + + // #[test] + // fn bad_json_from_sql() { + // let uuid: Result = + // FromSql::::from_sql(SqliteValue::for_test(b"boom")); + // assert_eq!(uuid.unwrap_err().to_string(), "Invalid Json"); + // } + + // #[test] + // fn no_json_from_sql() { + // let uuid: Result = + // FromSql::::from_nullable_sql(None); + // assert_eq!( + // uuid.unwrap_err().to_string(), + // "Unexpected null for non-null column" + // ); + // } + + // #[test] + // fn jsonb_to_sql() { + // let mut buffer = Vec::new(); + // let mut bytes = Output::test(ByteWrapper(&mut buffer)); + // let test_json = serde_json::Value::Bool(true); + // ToSql::::to_sql(&test_json, &mut bytes).unwrap(); + // assert_eq!(buffer, b"\x01true"); + // } + + // #[test] + // fn some_jsonb_from_sql() { + // let input_json = b"\x01true"; + // let output_json: serde_json::Value = + // FromSql::::from_sql(SqliteValue::for_test(input_json)) + // .unwrap(); + // assert_eq!(output_json, serde_json::Value::Bool(true)); + // } + + // #[test] + // fn bad_jsonb_from_sql() { + // let uuid: Result = + // FromSql::::from_sql(SqliteValue::for_test(b"\x01boom")); + // assert_eq!(uuid.unwrap_err().to_string(), "Invalid Json"); + // } + + // #[test] + // fn bad_jsonb_version_from_sql() { + // let uuid: Result = + // FromSql::::from_sql(SqliteValue::for_test(b"\x02true")); + // assert_eq!( + // uuid.unwrap_err().to_string(), + // "Unsupported JSONB encoding version" + // ); + // } + + // #[test] + // fn no_jsonb_from_sql() { + // let uuid: Result = + // FromSql::::from_nullable_sql(None); + // assert_eq!( + // uuid.unwrap_err().to_string(), + // "Unexpected null for non-null column" + // ); + // } +} diff --git a/diesel/src/sqlite/types/mod.rs b/diesel/src/sqlite/types/mod.rs index 88ef5f8cde26..22c3f66f4297 100644 --- a/diesel/src/sqlite/types/mod.rs +++ b/diesel/src/sqlite/types/mod.rs @@ -1,4 +1,5 @@ mod date_and_time; +mod json; mod numeric; use super::connection::SqliteValue; From eea59fdfd659fb38775809dbe66b5afcfee8de7d Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 24 Sep 2024 22:04:09 +0200 Subject: [PATCH 02/24] first implementation of a SQLite JSONB format reader --- diesel/src/sqlite/types/json.rs | 218 ++++++++++++++++++++++++++++---- diesel/src/sqlite/types/mod.rs | 1 + 2 files changed, 196 insertions(+), 23 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 630eadc5d277..79cda384a70a 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -5,14 +5,12 @@ use crate::serialize::{self, IsNull, Output, ToSql}; use crate::sql_types; use crate::sqlite::{Sqlite, SqliteValue}; -#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl FromSql for serde_json::Value { fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { serde_json::from_str(value.read_text()).map_err(|_| "Invalid Json".into()) } } -#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl ToSql for serde_json::Value { fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { out.set_value(serde_json::to_string(self)?); @@ -20,18 +18,23 @@ impl ToSql for serde_json::Value { } } -#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl FromSql for serde_json::Value { fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { let bytes = value.read_blob(); - if bytes[0] != 1 { - return Err("Unsupported JSONB encoding version".into()); + + // Ensure we have at least one byte for the version check + if bytes.is_empty() { + return Err("Empty blob cannot be decoded as JSONB".into()); } - serde_json::from_slice(&bytes[1..]).map_err(|_| "Invalid Json".into()) + + // Parse the first byte to determine the header size and the type + let (element_type, payload_size, remaining_bytes) = read_jsonb_header(bytes)?; + + // Parse the payload based on the element type + read_jsonb_element(element_type, payload_size, remaining_bytes) } } -#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl ToSql for serde_json::Value { fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { out.set_value(serde_json::to_string(self)?.into_bytes()); @@ -39,6 +42,173 @@ impl ToSql for serde_json::Value { } } +// Parse the header, including both the element type and the payload size +fn read_jsonb_header(bytes: &[u8]) -> deserialize::Result<(u8, usize, &[u8])> { + let first_byte = bytes[0]; + + // The upper 4 bits of the first byte determine the header size or the payload size directly + let header_size_encoding = (first_byte & 0xf0) >> 4; + let element_type = first_byte & 0x0f; + + let (payload_size, remaining_bytes) = match header_size_encoding { + 0x00..=0x0b => { + // If upper bits are between 0 and 11, payload size is stored in those bits directly + let payload_size = header_size_encoding as usize; + (payload_size, &bytes[1..]) + } + 0x0c => { + // Upper bits are 12, so payload size is in the next byte (2-byte header) + if bytes.len() < 2 { + return Err("Invalid JSONB: insufficient bytes for payload size".into()); + } + let payload_size = bytes[1] as usize; + (payload_size, &bytes[2..]) + } + 0x0d => { + // Upper bits are 13, so payload size is in the next 2 bytes (3-byte header) + if bytes.len() < 3 { + return Err("Invalid JSONB: insufficient bytes for payload size".into()); + } + let payload_size = u16::from_be_bytes([bytes[1], bytes[2]]) as usize; + (payload_size, &bytes[3..]) + } + 0x0e => { + // Upper bits are 14, so payload size is in the next 4 bytes (5-byte header) + if bytes.len() < 5 { + return Err("Invalid JSONB: insufficient bytes for payload size".into()); + } + let payload_size = + u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize; + (payload_size, &bytes[5..]) + } + 0x0f => { + // Upper bits are 15, so payload size is in the next 8 bytes (9-byte header) + if bytes.len() < 9 { + return Err("Invalid JSONB: insufficient bytes for payload size".into()); + } + let payload_size = u64::from_be_bytes([ + bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], + ]) as usize; + (payload_size, &bytes[9..]) + } + _ => return Err("Invalid header encoding".into()), + }; + + Ok((element_type, payload_size, remaining_bytes)) +} + +// Parse the actual element based on its type and payload size +fn read_jsonb_element( + element_type: u8, + payload_size: usize, + bytes: &[u8], +) -> deserialize::Result { + match element_type { + 0x00 => Ok(serde_json::Value::Null), // NULL + 0x01 => Ok(serde_json::Value::Bool(true)), // TRUE + 0x02 => Ok(serde_json::Value::Bool(false)), // FALSE + 0x03 => read_jsonb_integer(bytes), // INT + 0x04 => read_jsonb_float(bytes), // FLOAT + 0x05 => read_jsonb_text(payload_size, bytes), // TEXT + 0x06 => read_jsonb_array(bytes), // ARRAY + 0x07 => read_jsonb_object(payload_size, bytes), // OBJECT + _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), + } +} + +// Parse a JSONB integer +fn read_jsonb_integer(bytes: &[u8]) -> deserialize::Result { + let int_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB integer")?; + let int_value = int_str + .parse::() + .map_err(|_| "Failed to parse JSONB integer")?; + Ok(serde_json::Value::Number(serde_json::Number::from( + int_value, + ))) +} + +// Parse a JSONB float +fn read_jsonb_float(bytes: &[u8]) -> deserialize::Result { + let float_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB float")?; + let float_value = float_str + .parse::() + .map_err(|_| "Failed to parse JSONB float")?; + Ok(serde_json::Value::Number( + serde_json::Number::from_f64(float_value).unwrap(), + )) +} + +// Parse a JSONB text (string) +fn read_jsonb_text(payload_size: usize, bytes: &[u8]) -> deserialize::Result { + if bytes.len() < payload_size { + return Err("Invalid JSONB text: insufficient bytes".into()); + } + let text = + std::str::from_utf8(&bytes[..payload_size]).map_err(|_| "Invalid UTF-8 in JSONB text")?; + Ok(serde_json::Value::String(text.to_string())) +} + +// Parse a JSONB array (recursive parsing) +fn read_jsonb_array(bytes: &[u8]) -> deserialize::Result { + let mut elements = Vec::new(); + let mut remaining_bytes = bytes; + + while !remaining_bytes.is_empty() { + let (element_type, element_size, rest) = read_jsonb_header(remaining_bytes)?; + let element = read_jsonb_element(element_type, element_size, rest)?; + elements.push(element); + remaining_bytes = &remaining_bytes[element_size + 1..]; // Adjust for header + } + + Ok(serde_json::Value::Array(elements)) +} + +// Parse a JSONB object (recursive parsing) +fn read_jsonb_object(payload_size: usize, bytes: &[u8]) -> deserialize::Result { + let mut object = serde_json::Map::new(); + let mut remaining_bytes = bytes; + let mut total_read = 0; + + // Loop through the object key-value pairs + while total_read < payload_size { + // Read the key header + let (key_type, key_size, rest) = read_jsonb_header(remaining_bytes)?; + + // Ensure the key is a valid string type (TEXT, TEXTJ, TEXT5, or TEXTRAW) + match key_type { + 0x05 | 0x06 | 0x07 | 0x08 => { + // Valid string types: TEXT, TEXTJ, TEXT5, TEXTRAW + let key = read_jsonb_text(key_size, rest)? + .as_str() + .ok_or("Invalid object key in JSONB")? + .to_string(); + + // Move the remaining bytes pointer past the key + remaining_bytes = &rest[key_size..]; + + // Read the value header + let (value_type, value_size, rest_after_value) = + read_jsonb_header(remaining_bytes)?; + + // Parse the value based on its type + let value = read_jsonb_element(value_type, value_size, rest_after_value)?; + + // Insert the key-value pair into the object map + object.insert(key, value); + + // Move the remaining bytes pointer past the value + remaining_bytes = &remaining_bytes[value_size + 1..]; + total_read += key_size + value_size + 2; // Adjust total read for key and value size and headers + } + _ => { + return Err(format!("Invalid JSONB object key type: {}", key_type).into()); + } + } + } + + Ok(serde_json::Value::Object(object)) +} + #[cfg(test)] mod tests { use crate::deserialize::FromSql; @@ -47,6 +217,7 @@ mod tests { use crate::select; use crate::serialize::{Output, ToSql}; use crate::sql_types; + use crate::sql_types::Jsonb; use crate::sql_types::{Json, Text}; use crate::sqlite::connection::SqliteBindCollector; use crate::sqlite::Sqlite; @@ -55,14 +226,15 @@ mod tests { use crate::test_helpers::connection; use serde_json::json; - // #[test] - // fn json_to_sql() { - // let buffer = SqliteBindValue::from(0i32); - // let mut out = Output::<'_, 'static, Sqlite>::test(buffer); - // let test_json = serde_json::Value::Bool(true); - // ToSql::::to_sql(&test_json, &mut out).unwrap(); - // assert_eq!(buffer.inner, SqliteBindValue::from(1i32).inner); - // } + #[test] + fn json_to_sql() { + let conn = &mut connection(); + let value = json!(true); + let res = diesel::select(value.into_sql::().eq(sql("true"))) + .get_result::(conn) + .unwrap(); + assert!(res); + } // #[test] // fn json_to_sql() { @@ -91,14 +263,14 @@ mod tests { // .unwrap(); // } - #[test] - fn some_json_from_sql() { - let input_json = b"true"; - let output_json: serde_json::Value = - FromSql::::from_sql(SqliteValue::for_test(input_json)) - .unwrap(); - assert_eq!(output_json, serde_json::Value::Bool(true)); - } + // #[test] + // fn some_json_from_sql() { + // let input_json = b"true"; + // let output_json: serde_json::Value = + // FromSql::::from_sql(SqliteValue::for_test(input_json)) + // .unwrap(); + // assert_eq!(output_json, serde_json::Value::Bool(true)); + // } // #[test] // fn bad_json_from_sql() { diff --git a/diesel/src/sqlite/types/mod.rs b/diesel/src/sqlite/types/mod.rs index 22c3f66f4297..8f3235c96d0f 100644 --- a/diesel/src/sqlite/types/mod.rs +++ b/diesel/src/sqlite/types/mod.rs @@ -1,4 +1,5 @@ mod date_and_time; +#[cfg(all(feature = "sqlite", feature = "serde_json"))] mod json; mod numeric; From c3a6d7062750e17ef63e89143c563b72497c10f6 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 24 Sep 2024 22:32:56 +0200 Subject: [PATCH 03/24] first implementation of a SQLite JSONB format writer --- diesel/src/sqlite/types/json.rs | 365 +++++++++++++++++++------------- 1 file changed, 215 insertions(+), 150 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 79cda384a70a..d80b53b0ee01 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -5,6 +5,20 @@ use crate::serialize::{self, IsNull, Output, ToSql}; use crate::sql_types; use crate::sqlite::{Sqlite, SqliteValue}; +const JSONB_NULL: u8 = 0x00; +const JSONB_TRUE: u8 = 0x01; +const JSONB_FALSE: u8 = 0x02; +const JSONB_INT: u8 = 0x03; +const JSONB_INT5: u8 = 0x04; +const JSONB_FLOAT: u8 = 0x05; +const JSONB_FLOAT5: u8 = 0x06; +const JSONB_TEXT: u8 = 0x07; +const JSONB_TEXTJ: u8 = 0x08; +const JSONB_TEXT5: u8 = 0x09; +const JSONB_TEXTRAW: u8 = 0x0A; +const JSONB_ARRAY: u8 = 0x0B; +const JSONB_OBJECT: u8 = 0x0C; + impl FromSql for serde_json::Value { fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { serde_json::from_str(value.read_text()).map_err(|_| "Invalid Json".into()) @@ -22,102 +36,81 @@ impl FromSql for serde_json::Value { fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { let bytes = value.read_blob(); - // Ensure we have at least one byte for the version check if bytes.is_empty() { return Err("Empty blob cannot be decoded as JSONB".into()); } - // Parse the first byte to determine the header size and the type - let (element_type, payload_size, remaining_bytes) = read_jsonb_header(bytes)?; - - // Parse the payload based on the element type - read_jsonb_element(element_type, payload_size, remaining_bytes) + // Read the JSONB value from the byte stream + read_jsonb_value(&bytes) } } impl ToSql for serde_json::Value { fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { - out.set_value(serde_json::to_string(self)?.into_bytes()); + // Create a buffer to hold the binary JSONB encoding + let mut buffer = Vec::new(); + + // Write the JSON value into the buffer in JSONB format + write_jsonb_value(self, &mut buffer)?; + + // Set the serialized binary data to the output + out.set_value(buffer); + Ok(IsNull::No) } } -// Parse the header, including both the element type and the payload size -fn read_jsonb_header(bytes: &[u8]) -> deserialize::Result<(u8, usize, &[u8])> { - let first_byte = bytes[0]; - - // The upper 4 bits of the first byte determine the header size or the payload size directly - let header_size_encoding = (first_byte & 0xf0) >> 4; - let element_type = first_byte & 0x0f; +// Helper function to read a JSONB value from the byte stream +fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result { + if bytes.is_empty() { + return Err("Empty JSONB data".into()); + } - let (payload_size, remaining_bytes) = match header_size_encoding { - 0x00..=0x0b => { - // If upper bits are between 0 and 11, payload size is stored in those bits directly - let payload_size = header_size_encoding as usize; - (payload_size, &bytes[1..]) - } - 0x0c => { - // Upper bits are 12, so payload size is in the next byte (2-byte header) - if bytes.len() < 2 { - return Err("Invalid JSONB: insufficient bytes for payload size".into()); - } - let payload_size = bytes[1] as usize; - (payload_size, &bytes[2..]) - } - 0x0d => { - // Upper bits are 13, so payload size is in the next 2 bytes (3-byte header) - if bytes.len() < 3 { - return Err("Invalid JSONB: insufficient bytes for payload size".into()); - } - let payload_size = u16::from_be_bytes([bytes[1], bytes[2]]) as usize; - (payload_size, &bytes[3..]) - } - 0x0e => { - // Upper bits are 14, so payload size is in the next 4 bytes (5-byte header) - if bytes.len() < 5 { - return Err("Invalid JSONB: insufficient bytes for payload size".into()); - } - let payload_size = - u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize; - (payload_size, &bytes[5..]) - } - 0x0f => { - // Upper bits are 15, so payload size is in the next 8 bytes (9-byte header) - if bytes.len() < 9 { - return Err("Invalid JSONB: insufficient bytes for payload size".into()); - } - let payload_size = u64::from_be_bytes([ + // The first byte contains both the element type and potentially the payload size + let first_byte = bytes[0]; + let element_type = first_byte & 0x0F; + let payload_size_hint = (first_byte & 0xF0) >> 4; + + // Determine payload size and handle accordingly + let (payload_size, payload_start) = match payload_size_hint { + 0x00..=0x0B => (payload_size_hint as usize, 1), // Payload size is encoded in the upper four bits directly + 0x0C => (bytes[1] as usize, 2), // 1 additional byte for payload size + 0x0D => (u16::from_be_bytes([bytes[1], bytes[2]]) as usize, 3), // 2 additional bytes for payload size + 0x0E => ( + u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize, + 5, + ), // 4 additional bytes + 0x0F => ( + u64::from_be_bytes([ bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], - ]) as usize; - (payload_size, &bytes[9..]) - } - _ => return Err("Invalid header encoding".into()), + ]) as usize, + 9, + ), // 8 additional bytes for payload size (unlikely in practice) + _ => return Err("Invalid payload size hint".into()), }; - Ok((element_type, payload_size, remaining_bytes)) -} + let remaining_bytes = &bytes[payload_start..]; -// Parse the actual element based on its type and payload size -fn read_jsonb_element( - element_type: u8, - payload_size: usize, - bytes: &[u8], -) -> deserialize::Result { match element_type { - 0x00 => Ok(serde_json::Value::Null), // NULL - 0x01 => Ok(serde_json::Value::Bool(true)), // TRUE - 0x02 => Ok(serde_json::Value::Bool(false)), // FALSE - 0x03 => read_jsonb_integer(bytes), // INT - 0x04 => read_jsonb_float(bytes), // FLOAT - 0x05 => read_jsonb_text(payload_size, bytes), // TEXT - 0x06 => read_jsonb_array(bytes), // ARRAY - 0x07 => read_jsonb_object(payload_size, bytes), // OBJECT + JSONB_NULL => Ok(serde_json::Value::Null), // Null has no payload + JSONB_TRUE => Ok(serde_json::Value::Bool(true)), // True has no payload + JSONB_FALSE => Ok(serde_json::Value::Bool(false)), // False has no payload + JSONB_INT => read_jsonb_int(remaining_bytes, payload_size), + JSONB_INT5 => Err("INT5 is not supported in this implementation".into()), // INT5 not supported + JSONB_FLOAT => read_jsonb_float(remaining_bytes, payload_size), + JSONB_FLOAT5 => Err("FLOAT5 is not supported in this implementation".into()), // FLOAT5 not supported + JSONB_TEXT => read_jsonb_text(remaining_bytes, payload_size), + JSONB_TEXTJ => read_jsonb_text(remaining_bytes, payload_size), // Handle TEXTJ similarly to TEXT for now + JSONB_TEXT5 => Err("TEXT5 is not supported in this implementation".into()), // TEXT5 not supported + JSONB_TEXTRAW => read_jsonb_text(remaining_bytes, payload_size), // Handle TEXTRAW similarly to TEXT for now + JSONB_ARRAY => read_jsonb_array(remaining_bytes, payload_size), + JSONB_OBJECT => read_jsonb_object(remaining_bytes, payload_size), _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), } } -// Parse a JSONB integer -fn read_jsonb_integer(bytes: &[u8]) -> deserialize::Result { +// Read a JSON integer in canonical format (INT) +fn read_jsonb_int(bytes: &[u8], _payload_size: usize) -> deserialize::Result { let int_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB integer")?; let int_value = int_str .parse::() @@ -127,8 +120,8 @@ fn read_jsonb_integer(bytes: &[u8]) -> deserialize::Result { ))) } -// Parse a JSONB float -fn read_jsonb_float(bytes: &[u8]) -> deserialize::Result { +// Read a JSON float in canonical format (FLOAT) +fn read_jsonb_float(bytes: &[u8], _payload_size: usize) -> deserialize::Result { let float_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB float")?; let float_value = float_str .parse::() @@ -138,77 +131,176 @@ fn read_jsonb_float(bytes: &[u8]) -> deserialize::Result { )) } -// Parse a JSONB text (string) -fn read_jsonb_text(payload_size: usize, bytes: &[u8]) -> deserialize::Result { - if bytes.len() < payload_size { - return Err("Invalid JSONB text: insufficient bytes".into()); - } - let text = - std::str::from_utf8(&bytes[..payload_size]).map_err(|_| "Invalid UTF-8 in JSONB text")?; +// Read a JSON string +fn read_jsonb_text(bytes: &[u8], payload_size: usize) -> deserialize::Result { + let text_bytes = &bytes[..payload_size]; + let text = std::str::from_utf8(text_bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; Ok(serde_json::Value::String(text.to_string())) } -// Parse a JSONB array (recursive parsing) -fn read_jsonb_array(bytes: &[u8]) -> deserialize::Result { +// Read a JSON array +fn read_jsonb_array(bytes: &[u8], payload_size: usize) -> deserialize::Result { let mut elements = Vec::new(); let mut remaining_bytes = bytes; + let mut total_read = 0; - while !remaining_bytes.is_empty() { - let (element_type, element_size, rest) = read_jsonb_header(remaining_bytes)?; - let element = read_jsonb_element(element_type, element_size, rest)?; + // Loop through the array elements and parse each one + while total_read < payload_size { + let element = read_jsonb_value(remaining_bytes)?; + let element_size = remaining_bytes.len() - bytes.len(); elements.push(element); - remaining_bytes = &remaining_bytes[element_size + 1..]; // Adjust for header + remaining_bytes = &remaining_bytes[element_size..]; + total_read += element_size; } Ok(serde_json::Value::Array(elements)) } -// Parse a JSONB object (recursive parsing) -fn read_jsonb_object(payload_size: usize, bytes: &[u8]) -> deserialize::Result { +// Read a JSON object +fn read_jsonb_object(bytes: &[u8], payload_size: usize) -> deserialize::Result { let mut object = serde_json::Map::new(); let mut remaining_bytes = bytes; let mut total_read = 0; - // Loop through the object key-value pairs + // Loop through the object key-value pairs and parse each one while total_read < payload_size { - // Read the key header - let (key_type, key_size, rest) = read_jsonb_header(remaining_bytes)?; - - // Ensure the key is a valid string type (TEXT, TEXTJ, TEXT5, or TEXTRAW) - match key_type { - 0x05 | 0x06 | 0x07 | 0x08 => { - // Valid string types: TEXT, TEXTJ, TEXT5, TEXTRAW - let key = read_jsonb_text(key_size, rest)? - .as_str() - .ok_or("Invalid object key in JSONB")? - .to_string(); - - // Move the remaining bytes pointer past the key - remaining_bytes = &rest[key_size..]; - - // Read the value header - let (value_type, value_size, rest_after_value) = - read_jsonb_header(remaining_bytes)?; - - // Parse the value based on its type - let value = read_jsonb_element(value_type, value_size, rest_after_value)?; - - // Insert the key-value pair into the object map - object.insert(key, value); - - // Move the remaining bytes pointer past the value - remaining_bytes = &remaining_bytes[value_size + 1..]; - total_read += key_size + value_size + 2; // Adjust total read for key and value size and headers - } - _ => { - return Err(format!("Invalid JSONB object key type: {}", key_type).into()); - } + let key_type = remaining_bytes[0] & 0x0F; + + // Ensure the key is a valid string type (TEXT, TEXTJ, TEXT5, TEXTRAW) + if key_type != JSONB_TEXT + && key_type != JSONB_TEXTJ + && key_type != JSONB_TEXT5 + && key_type != JSONB_TEXTRAW + { + return Err(format!("Invalid JSONB object key type: {}", key_type).into()); } + + // Read the key + let key = read_jsonb_text(&remaining_bytes[1..], payload_size)? + .as_str() + .ok_or("Invalid object key in JSONB")? + .to_string(); + let key_size = remaining_bytes.len() - bytes.len(); + remaining_bytes = &remaining_bytes[key_size + 1..]; + total_read += key_size + 1; + + // Read the value + let value = read_jsonb_value(remaining_bytes)?; + let value_size = remaining_bytes.len() - bytes.len(); + object.insert(key, value); + remaining_bytes = &remaining_bytes[value_size..]; + total_read += value_size; } Ok(serde_json::Value::Object(object)) } +// Helper function to write a JSON value into a JSONB binary format +fn write_jsonb_value(value: &serde_json::Value, buffer: &mut Vec) -> serialize::Result { + match value { + serde_json::Value::Null => write_jsonb_null(buffer), + serde_json::Value::Bool(b) => write_jsonb_bool(*b, buffer), + serde_json::Value::Number(n) => write_jsonb_number(n, buffer), + serde_json::Value::String(s) => write_jsonb_string(s, buffer), + serde_json::Value::Array(arr) => write_jsonb_array(arr, buffer), + serde_json::Value::Object(obj) => write_jsonb_object(obj, buffer), + } +} + +// Write a JSON null +fn write_jsonb_null(buffer: &mut Vec) -> serialize::Result { + // Use the constant for null + buffer.push(JSONB_NULL); + Ok(IsNull::No) +} + +// Write a JSON boolean +fn write_jsonb_bool(b: bool, buffer: &mut Vec) -> serialize::Result { + // Use the constants for true and false + let byte = if b { JSONB_TRUE } else { JSONB_FALSE }; + buffer.push(byte); + Ok(IsNull::No) +} + +// Write a JSON number (integers and floats) +fn write_jsonb_number(n: &serde_json::Number, buffer: &mut Vec) -> serialize::Result { + if let Some(i) = n.as_i64() { + // Write an integer (INT type) + write_jsonb_int(i, buffer) + } else if let Some(f) = n.as_f64() { + // Write a float (FLOAT type) + write_jsonb_float(f, buffer) + } else { + Err("Invalid number type".into()) + } +} + +// Write an integer in JSONB format +fn write_jsonb_int(i: i64, buffer: &mut Vec) -> serialize::Result { + // Use the constant for INT + buffer.push(JSONB_INT); + + // Write the ASCII text representation of the integer as the payload + buffer.extend_from_slice(i.to_string().as_bytes()); + + Ok(IsNull::No) +} + +// Write a floating-point number in JSONB format +fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { + // Use the constant for FLOAT + buffer.push(JSONB_FLOAT); + + // Write the ASCII text representation of the float as the payload + buffer.extend_from_slice(f.to_string().as_bytes()); + + Ok(IsNull::No) +} + +// Write a JSON string +fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { + // Use the constant for TEXT + buffer.push(JSONB_TEXT); + + // Write the UTF-8 text of the string as the payload (no delimiters) + buffer.extend_from_slice(s.as_bytes()); + + Ok(IsNull::No) +} + +// Write a JSON array +fn write_jsonb_array(arr: &[serde_json::Value], buffer: &mut Vec) -> serialize::Result { + // Use the constant for ARRAY + buffer.push(JSONB_ARRAY); + + // Recursively write each element of the array + for element in arr { + write_jsonb_value(element, buffer)?; + } + + Ok(IsNull::No) +} + +// Write a JSON object +fn write_jsonb_object( + obj: &serde_json::Map, + buffer: &mut Vec, +) -> serialize::Result { + // Use the constant for OBJECT + buffer.push(JSONB_OBJECT); + + // Recursively write each key-value pair of the object + for (key, value) in obj { + // Write the key (which must be a string) + write_jsonb_string(key, buffer)?; + + // Write the value + write_jsonb_value(value, buffer)?; + } + + Ok(IsNull::No) +} + #[cfg(test)] mod tests { use crate::deserialize::FromSql; @@ -236,33 +328,6 @@ mod tests { assert!(res); } - // #[test] - // fn json_to_sql() { - // crate::table! { - // #[allow(unused_parens)] - // test_insert_json_into_table_as_text(id) { - // id -> Integer, - // json -> Text, - // } - // } - // let conn = &mut connection(); - // crate::sql_query( - // "CREATE TABLE test_insert_json_into_table_as_text(id INTEGER PRIMARY KEY, json TEXT);", - // ) - // .execute(conn) - // .unwrap(); - - // let value = json!(true); - - // crate::insert_into(test_insert_json_into_table_as_text::table) - // .values(( - // test_insert_json_into_table_as_text::id.eq(1), - // test_insert_json_into_table_as_text::json.eq(value), - // )) - // .execute(conn) - // .unwrap(); - // } - // #[test] // fn some_json_from_sql() { // let input_json = b"true"; From d917a8996cc2ac9ad667afed048f14885f09391c Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 24 Sep 2024 23:45:04 +0200 Subject: [PATCH 04/24] add unit tests for the SQLite JSONB read/write functions --- diesel/src/sqlite/types/json.rs | 404 +++++++++++++++++++++++--------- 1 file changed, 298 insertions(+), 106 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index d80b53b0ee01..1157de836115 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -41,7 +41,7 @@ impl FromSql for serde_json::Value { } // Read the JSONB value from the byte stream - read_jsonb_value(&bytes) + read_jsonb_value(&bytes).map(|(v, _)| v) } } @@ -61,7 +61,7 @@ impl ToSql for serde_json::Value { } // Helper function to read a JSONB value from the byte stream -fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result { +fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result<(serde_json::Value, usize)> { if bytes.is_empty() { return Err("Empty JSONB data".into()); } @@ -69,52 +69,90 @@ fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result { // The first byte contains both the element type and potentially the payload size let first_byte = bytes[0]; let element_type = first_byte & 0x0F; - let payload_size_hint = (first_byte & 0xF0) >> 4; - - // Determine payload size and handle accordingly - let (payload_size, payload_start) = match payload_size_hint { - 0x00..=0x0B => (payload_size_hint as usize, 1), // Payload size is encoded in the upper four bits directly - 0x0C => (bytes[1] as usize, 2), // 1 additional byte for payload size - 0x0D => (u16::from_be_bytes([bytes[1], bytes[2]]) as usize, 3), // 2 additional bytes for payload size - 0x0E => ( - u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize, - 5, - ), // 4 additional bytes - 0x0F => ( - u64::from_be_bytes([ - bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], - ]) as usize, - 9, - ), // 8 additional bytes for payload size (unlikely in practice) + let size_hint = (first_byte & 0xF0) >> 4; + + let (payload_size, header_size) = match size_hint { + 0x00..=0x0B => (size_hint as usize, 1), // Payload size is directly in the upper nibble + 0x0C => { + if bytes.len() < 2 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + (bytes[1] as usize, 2) // 1 additional byte for payload size + } + 0x0D => { + if bytes.len() < 3 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + (u16::from_be_bytes([bytes[1], bytes[2]]) as usize, 3) // 2 additional bytes + } + 0x0E => { + if bytes.len() < 5 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + ( + u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize, + 5, + ) // 4 additional bytes + } + 0x0F => { + if bytes.len() < 9 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + ( + u64::from_be_bytes([ + bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], + ]) as usize, + 9, + ) // 8 additional bytes + } _ => return Err("Invalid payload size hint".into()), }; - let remaining_bytes = &bytes[payload_start..]; - - match element_type { - JSONB_NULL => Ok(serde_json::Value::Null), // Null has no payload - JSONB_TRUE => Ok(serde_json::Value::Bool(true)), // True has no payload - JSONB_FALSE => Ok(serde_json::Value::Bool(false)), // False has no payload - JSONB_INT => read_jsonb_int(remaining_bytes, payload_size), - JSONB_INT5 => Err("INT5 is not supported in this implementation".into()), // INT5 not supported - JSONB_FLOAT => read_jsonb_float(remaining_bytes, payload_size), - JSONB_FLOAT5 => Err("FLOAT5 is not supported in this implementation".into()), // FLOAT5 not supported - JSONB_TEXT => read_jsonb_text(remaining_bytes, payload_size), - JSONB_TEXTJ => read_jsonb_text(remaining_bytes, payload_size), // Handle TEXTJ similarly to TEXT for now - JSONB_TEXT5 => Err("TEXT5 is not supported in this implementation".into()), // TEXT5 not supported - JSONB_TEXTRAW => read_jsonb_text(remaining_bytes, payload_size), // Handle TEXTRAW similarly to TEXT for now - JSONB_ARRAY => read_jsonb_array(remaining_bytes, payload_size), - JSONB_OBJECT => read_jsonb_object(remaining_bytes, payload_size), - _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), + let total_size = header_size + payload_size; + if bytes.len() < total_size { + return Err("Invalid JSONB data: insufficient bytes for value".into()); } + + let payload_bytes = &bytes[header_size..total_size]; + + let value = match element_type { + JSONB_NULL => Ok(serde_json::Value::Null), + JSONB_TRUE => Ok(serde_json::Value::Bool(true)), + JSONB_FALSE => Ok(serde_json::Value::Bool(false)), + JSONB_INT => read_jsonb_int(payload_bytes, payload_size), + JSONB_INT5 => Err("INT5 is not supported in this implementation".into()), + JSONB_FLOAT => read_jsonb_float(payload_bytes, payload_size), + JSONB_FLOAT5 => Err("FLOAT5 is not supported in this implementation".into()), + JSONB_TEXT | JSONB_TEXTJ | JSONB_TEXTRAW => read_jsonb_text(payload_bytes, payload_size), + JSONB_TEXT5 => Err("TEXT5 is not supported in this implementation".into()), + JSONB_ARRAY => read_jsonb_array(payload_bytes, payload_size), + JSONB_OBJECT => read_jsonb_object(payload_bytes, payload_size), + _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), + }?; + + Ok((value, total_size)) } // Read a JSON integer in canonical format (INT) -fn read_jsonb_int(bytes: &[u8], _payload_size: usize) -> deserialize::Result { - let int_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB integer")?; +fn read_jsonb_int(bytes: &[u8], payload_size: usize) -> deserialize::Result { + // Ensure the bytes are at least as large as the payload size + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } + + // Read only the number of bytes specified by the payload size + let int_str = std::str::from_utf8(&bytes[..payload_size]) + .map_err(|_| "Invalid ASCII in JSONB integer")?; + // Parse the integer string into an i64 let int_value = int_str .parse::() .map_err(|_| "Failed to parse JSONB integer")?; + Ok(serde_json::Value::Number(serde_json::Number::from( int_value, ))) @@ -141,60 +179,53 @@ fn read_jsonb_text(bytes: &[u8], payload_size: usize) -> deserialize::Result deserialize::Result { let mut elements = Vec::new(); - let mut remaining_bytes = bytes; let mut total_read = 0; - // Loop through the array elements and parse each one while total_read < payload_size { - let element = read_jsonb_value(remaining_bytes)?; - let element_size = remaining_bytes.len() - bytes.len(); + let (element, consumed) = read_jsonb_value(&bytes[total_read..payload_size])?; + elements.push(element); - remaining_bytes = &remaining_bytes[element_size..]; - total_read += element_size; + total_read += consumed; + } + + if total_read != payload_size { + return Err("Array payload size mismatch".into()); } Ok(serde_json::Value::Array(elements)) } -// Read a JSON object fn read_jsonb_object(bytes: &[u8], payload_size: usize) -> deserialize::Result { let mut object = serde_json::Map::new(); - let mut remaining_bytes = bytes; let mut total_read = 0; - // Loop through the object key-value pairs and parse each one while total_read < payload_size { - let key_type = remaining_bytes[0] & 0x0F; - - // Ensure the key is a valid string type (TEXT, TEXTJ, TEXT5, TEXTRAW) - if key_type != JSONB_TEXT - && key_type != JSONB_TEXTJ - && key_type != JSONB_TEXT5 - && key_type != JSONB_TEXTRAW - { - return Err(format!("Invalid JSONB object key type: {}", key_type).into()); - } - // Read the key - let key = read_jsonb_text(&remaining_bytes[1..], payload_size)? + let (key_value, key_consumed) = read_jsonb_value(&bytes[total_read..payload_size])?; + let key_str = key_value .as_str() - .ok_or("Invalid object key in JSONB")? + .ok_or("Object key is not a string")? .to_string(); - let key_size = remaining_bytes.len() - bytes.len(); - remaining_bytes = &remaining_bytes[key_size + 1..]; - total_read += key_size + 1; + total_read += key_consumed; // Read the value - let value = read_jsonb_value(remaining_bytes)?; - let value_size = remaining_bytes.len() - bytes.len(); - object.insert(key, value); - remaining_bytes = &remaining_bytes[value_size..]; - total_read += value_size; + let (value, value_consumed) = read_jsonb_value(&bytes[total_read..payload_size])?; + total_read += value_consumed; + + object.insert(key_str, value); + } + + if total_read != payload_size { + return Err("Object payload size mismatch".into()); } Ok(serde_json::Value::Object(object)) } +fn write_jsonb_header(buffer: &mut Vec, element_type: u8, payload_size: u8) { + buffer.push((payload_size << 4) | element_type); +} + // Helper function to write a JSON value into a JSONB binary format fn write_jsonb_value(value: &serde_json::Value, buffer: &mut Vec) -> serialize::Result { match value { @@ -209,8 +240,7 @@ fn write_jsonb_value(value: &serde_json::Value, buffer: &mut Vec) -> seriali // Write a JSON null fn write_jsonb_null(buffer: &mut Vec) -> serialize::Result { - // Use the constant for null - buffer.push(JSONB_NULL); + write_jsonb_header(buffer, JSONB_NULL, 0x0); Ok(IsNull::No) } @@ -237,30 +267,31 @@ fn write_jsonb_number(n: &serde_json::Number, buffer: &mut Vec) -> serialize // Write an integer in JSONB format fn write_jsonb_int(i: i64, buffer: &mut Vec) -> serialize::Result { - // Use the constant for INT - buffer.push(JSONB_INT); + let int_str = i.to_string(); + + write_jsonb_header(buffer, JSONB_INT, int_str.len() as u8); // Write the ASCII text representation of the integer as the payload - buffer.extend_from_slice(i.to_string().as_bytes()); + buffer.extend_from_slice(int_str.as_bytes()); Ok(IsNull::No) } // Write a floating-point number in JSONB format fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { - // Use the constant for FLOAT - buffer.push(JSONB_FLOAT); + let float_str = f.to_string(); + + write_jsonb_header(buffer, JSONB_FLOAT, float_str.len() as u8); // Write the ASCII text representation of the float as the payload - buffer.extend_from_slice(f.to_string().as_bytes()); + buffer.extend_from_slice(float_str.as_bytes()); Ok(IsNull::No) } // Write a JSON string fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { - // Use the constant for TEXT - buffer.push(JSONB_TEXT); + write_jsonb_header(buffer, JSONB_TEXT, s.len() as u8); // Write the UTF-8 text of the string as the payload (no delimiters) buffer.extend_from_slice(s.as_bytes()); @@ -270,14 +301,17 @@ fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { // Write a JSON array fn write_jsonb_array(arr: &[serde_json::Value], buffer: &mut Vec) -> serialize::Result { - // Use the constant for ARRAY - buffer.push(JSONB_ARRAY); + let mut tmp_buffer = Vec::new(); // Recursively write each element of the array for element in arr { - write_jsonb_value(element, buffer)?; + write_jsonb_value(element, &mut tmp_buffer)?; } + write_jsonb_header(buffer, JSONB_ARRAY, tmp_buffer.len() as u8); + + buffer.extend_from_slice(&tmp_buffer); + Ok(IsNull::No) } @@ -286,48 +320,206 @@ fn write_jsonb_object( obj: &serde_json::Map, buffer: &mut Vec, ) -> serialize::Result { - // Use the constant for OBJECT - buffer.push(JSONB_OBJECT); + let mut tmp_buffer = Vec::new(); // Recursively write each key-value pair of the object for (key, value) in obj { // Write the key (which must be a string) - write_jsonb_string(key, buffer)?; + write_jsonb_string(key, &mut tmp_buffer)?; // Write the value - write_jsonb_value(value, buffer)?; + write_jsonb_value(value, &mut tmp_buffer)?; } + write_jsonb_header(buffer, JSONB_OBJECT, tmp_buffer.len() as u8); + + buffer.extend_from_slice(&tmp_buffer); + Ok(IsNull::No) } #[cfg(test)] mod tests { - use crate::deserialize::FromSql; - use crate::dsl::sql; - use crate::prelude::*; - use crate::select; - use crate::serialize::{Output, ToSql}; - use crate::sql_types; - use crate::sql_types::Jsonb; - use crate::sql_types::{Json, Text}; - use crate::sqlite::connection::SqliteBindCollector; - use crate::sqlite::Sqlite; - use crate::sqlite::SqliteBindValue; - use crate::sqlite::SqliteValue; - use crate::test_helpers::connection; - use serde_json::json; + use super::*; + use serde_json::{json, Value}; + + // Helper function to create a basic JSONB header byte + fn create_header(element_type: u8, payload_size: u8) -> u8 { + (payload_size << 4) | element_type + } #[test] - fn json_to_sql() { - let conn = &mut connection(); - let value = json!(true); - let res = diesel::select(value.into_sql::().eq(sql("true"))) - .get_result::(conn) - .unwrap(); - assert!(res); + fn test_read_jsonb_null() { + let data = vec![JSONB_NULL]; + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, Value::Null); } + #[test] + fn test_read_jsonb_true() { + let data = vec![JSONB_TRUE]; + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, Value::Bool(true)); + } + + #[test] + fn test_read_jsonb_false() { + let data = vec![JSONB_FALSE]; + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, Value::Bool(false)); + } + + #[test] + fn test_read_jsonb_int() { + let data = vec![create_header(JSONB_INT, 0x01), b'1']; // JSONB_INT with payload "1" + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, json!(1)); + } + + #[test] + fn test_read_jsonb_float() { + let data = vec![create_header(JSONB_FLOAT, 0x03), b'1', b'.', b'5']; // JSONB_FLOAT with payload "1.5" + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, json!(1.5)); + } + + #[test] + fn test_read_jsonb_text() { + let data = vec![create_header(JSONB_TEXT, 0x03), b'f', b'o', b'o']; // JSONB_TEXT with payload "foo" + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, json!("foo")); + } + + #[test] + fn test_read_jsonb_array() { + // JSONB_ARRAY with two elements: 1 and true + let data = vec![ + create_header(JSONB_ARRAY, 0x03), // Array header + create_header(JSONB_INT, 0x01), + b'1', // Element 1: integer "1" + create_header(JSONB_TRUE, 0x00), // Element 2: true + ]; + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, json!([1, true])); + } + + #[test] + fn test_read_jsonb_object() { + // JSONB_OBJECT with one key-value pair: "key": 42 + let data = vec![ + create_header(JSONB_OBJECT, 0x07), // Object header + create_header(JSONB_TEXT, 0x03), + b'k', + b'e', + b'y', // Key: "key" + create_header(JSONB_INT, 0x02), + b'4', + b'2', // Value: 42 + ]; + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!(result, json!({"key": 42})); + } + + #[test] + fn test_write_jsonb_null() { + let value = serde_json::Value::Null; + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!(buffer, vec![JSONB_NULL]); + } + + #[test] + fn test_write_jsonb_true() { + let value = serde_json::Value::Bool(true); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!(buffer, vec![JSONB_TRUE]); + } + + #[test] + fn test_write_jsonb_false() { + let value = serde_json::Value::Bool(false); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!(buffer, vec![JSONB_FALSE]); + } + + #[test] + fn test_write_jsonb_int() { + let value = serde_json::Value::Number(serde_json::Number::from(1)); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!(buffer, vec![create_header(JSONB_INT, 0x01), b'1']); + } + + #[test] + fn test_write_jsonb_float() { + let value = serde_json::Value::Number(serde_json::Number::from_f64(1.5).unwrap()); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!( + buffer, + vec![create_header(JSONB_FLOAT, 0x03), b'1', b'.', b'5'] + ); + } + + #[test] + fn test_write_jsonb_text() { + let value = serde_json::Value::String("foo".to_string()); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!( + buffer, + vec![create_header(JSONB_TEXT, 0x03), b'f', b'o', b'o'] + ); + } + + #[test] + fn test_write_jsonb_array() { + let value = json!([1, true]); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!( + buffer, + vec![ + create_header(JSONB_ARRAY, 0x03), // Array header + create_header(JSONB_INT, 0x01), // Integer header + b'1', // Integer payload "1" + create_header(JSONB_TRUE, 0x00), // Boolean header for "true" + ] + ); + } + + #[test] + fn test_write_jsonb_object() { + let value = json!({"key": 42}); + let mut buffer = Vec::new(); + write_jsonb_value(&value, &mut buffer).unwrap(); + assert_eq!( + buffer, + vec![ + create_header(JSONB_OBJECT, 0x07), // Object header + create_header(JSONB_TEXT, 0x03), // Key header: "key" + b'k', + b'e', + b'y', // Key: "key" + create_header(JSONB_INT, 0x02), // Value header: integer "42" + b'4', + b'2', // Value: 42 + ] + ); + } + + // #[test] + // fn json_to_sql() { + // let conn = &mut connection(); + // let value = json!(true); + // let res = diesel::select(value.into_sql::().eq(sql("true"))) + // .get_result::(conn) + // .unwrap(); + // assert!(res); + // } + // #[test] // fn some_json_from_sql() { // let input_json = b"true"; From cd7018d6d395739559c1c3dde156bffa1356c0ff Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Wed, 25 Sep 2024 00:16:09 +0200 Subject: [PATCH 05/24] fix clippy warning --- diesel/src/sqlite/types/json.rs | 53 ++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 1157de836115..af6be397f685 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -41,7 +41,7 @@ impl FromSql for serde_json::Value { } // Read the JSONB value from the byte stream - read_jsonb_value(&bytes).map(|(v, _)| v) + read_jsonb_value(bytes).map(|(v, _)| v) } } @@ -99,9 +99,10 @@ fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result<(serde_json::Value, usi return Err("Invalid JSONB data: insufficient bytes for payload size".into()); } ( - u64::from_be_bytes([ + usize::try_from(u64::from_be_bytes([ bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], - ]) as usize, + ])) + .map_err(Box::new)?, 9, ) // 8 additional bytes } @@ -165,7 +166,7 @@ fn read_jsonb_float(bytes: &[u8], _payload_size: usize) -> deserialize::Result() .map_err(|_| "Failed to parse JSONB float")?; Ok(serde_json::Value::Number( - serde_json::Number::from_f64(float_value).unwrap(), + serde_json::Number::from_f64(float_value).ok_or("Invalid float value")?, )) } @@ -222,8 +223,16 @@ fn read_jsonb_object(bytes: &[u8], payload_size: usize) -> deserialize::Result, element_type: u8, payload_size: u8) { +fn write_jsonb_header( + buffer: &mut Vec, + element_type: u8, + payload_size: usize, +) -> serialize::Result { + let payload_size = u8::try_from(payload_size).map_err(Box::new)?; + buffer.push((payload_size << 4) | element_type); + + Ok(IsNull::No) } // Helper function to write a JSON value into a JSONB binary format @@ -240,7 +249,7 @@ fn write_jsonb_value(value: &serde_json::Value, buffer: &mut Vec) -> seriali // Write a JSON null fn write_jsonb_null(buffer: &mut Vec) -> serialize::Result { - write_jsonb_header(buffer, JSONB_NULL, 0x0); + write_jsonb_header(buffer, JSONB_NULL, 0x0)?; Ok(IsNull::No) } @@ -269,7 +278,7 @@ fn write_jsonb_number(n: &serde_json::Number, buffer: &mut Vec) -> serialize fn write_jsonb_int(i: i64, buffer: &mut Vec) -> serialize::Result { let int_str = i.to_string(); - write_jsonb_header(buffer, JSONB_INT, int_str.len() as u8); + write_jsonb_header(buffer, JSONB_INT, int_str.len())?; // Write the ASCII text representation of the integer as the payload buffer.extend_from_slice(int_str.as_bytes()); @@ -281,7 +290,7 @@ fn write_jsonb_int(i: i64, buffer: &mut Vec) -> serialize::Result { fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { let float_str = f.to_string(); - write_jsonb_header(buffer, JSONB_FLOAT, float_str.len() as u8); + write_jsonb_header(buffer, JSONB_FLOAT, float_str.len())?; // Write the ASCII text representation of the float as the payload buffer.extend_from_slice(float_str.as_bytes()); @@ -291,7 +300,7 @@ fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { // Write a JSON string fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { - write_jsonb_header(buffer, JSONB_TEXT, s.len() as u8); + write_jsonb_header(buffer, JSONB_TEXT, s.len())?; // Write the UTF-8 text of the string as the payload (no delimiters) buffer.extend_from_slice(s.as_bytes()); @@ -308,7 +317,7 @@ fn write_jsonb_array(arr: &[serde_json::Value], buffer: &mut Vec) -> seriali write_jsonb_value(element, &mut tmp_buffer)?; } - write_jsonb_header(buffer, JSONB_ARRAY, tmp_buffer.len() as u8); + write_jsonb_header(buffer, JSONB_ARRAY, tmp_buffer.len())?; buffer.extend_from_slice(&tmp_buffer); @@ -331,7 +340,7 @@ fn write_jsonb_object( write_jsonb_value(value, &mut tmp_buffer)?; } - write_jsonb_header(buffer, JSONB_OBJECT, tmp_buffer.len() as u8); + write_jsonb_header(buffer, JSONB_OBJECT, tmp_buffer.len())?; buffer.extend_from_slice(&tmp_buffer); @@ -341,7 +350,11 @@ fn write_jsonb_object( #[cfg(test)] mod tests { use super::*; + use crate::query_dsl::RunQueryDsl; + use crate::test_helpers::connection; + use crate::{dsl::sql, IntoSql}; use serde_json::{json, Value}; + use sql_types::Jsonb; // Helper function to create a basic JSONB header byte fn create_header(element_type: u8, payload_size: u8) -> u8 { @@ -510,15 +523,15 @@ mod tests { ); } - // #[test] - // fn json_to_sql() { - // let conn = &mut connection(); - // let value = json!(true); - // let res = diesel::select(value.into_sql::().eq(sql("true"))) - // .get_result::(conn) - // .unwrap(); - // assert!(res); - // } + #[test] + fn json_to_sql() { + let conn = &mut connection(); + let value = json!(true); + let res = diesel::select(value.into_sql::().eq(&sql("true"))) + .get_result::(conn) + .unwrap(); + assert!(res); + } // #[test] // fn some_json_from_sql() { From a557574b012c5749589735ed7f7ab70089510d8b Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Wed, 25 Sep 2024 08:40:58 +0200 Subject: [PATCH 06/24] update the Jsonb rustdoc --- diesel/src/sql_types/mod.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index fb1717b318c4..85c1ef01a211 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -412,7 +412,24 @@ pub struct Json; /// The [`jsonb`] SQL type. This type can only be used with `feature = /// "serde_json"` /// -/// `jsonb` offers [several advantages][adv] over regular JSON: +/// In SQLite, `jsonb` brings mainly [performance improvements][sqlite-adv] over +/// regular JSON: +/// +/// > The advantage of JSONB in SQLite is that it is smaller and faster than +/// > text JSON - potentially several times faster. There is space in the +/// > on-disk JSONB format to add enhancements and future versions of SQLite +/// > might include options to provide O(1) lookup of elements in JSONB, but no +/// > such capability is currently available. +/// +///
+/// In SQLite, JSONB is intended for internal use by SQLite only. Thus, future +/// SQLite updates might break our JSONB implementation. And one might have to +/// wait and then upgrade diesel for those changes to be accounted +/// for. If you do not want this, prefer the regular +/// Json type. +///
+/// +/// In PostgreSQL, `jsonb` offers [several advantages][pg-adv] over regular JSON: /// /// > There are two JSON data types: `json` and `jsonb`. They accept almost /// > identical sets of values as input. The major practical difference @@ -428,7 +445,8 @@ pub struct Json; /// > `jsonb`, unless there are quite specialized needs, such as legacy /// > assumptions about ordering of object keys. /// -/// [adv]: https://www.postgresql.org/docs/current/static/datatype-json.html +/// [pg-adv]: https://www.postgresql.org/docs/current/static/datatype-json.html +/// [sqlite-adv]: https://sqlite.org/draft/jsonb.html /// /// ### [`ToSql`] impls /// From c7b121e46aa52c3a0184d7a7b8813834bdbc1035 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Wed, 25 Sep 2024 16:44:10 +0200 Subject: [PATCH 07/24] working json_to_sql and jsonb_to_sql tests for sqlite --- diesel/src/sqlite/types/json.rs | 22 +++++++++++++++------- diesel/src/type_impls/json.rs | 4 ++-- diesel/src/type_impls/mod.rs | 6 +++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index af6be397f685..f3f4613cb709 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -256,8 +256,7 @@ fn write_jsonb_null(buffer: &mut Vec) -> serialize::Result { // Write a JSON boolean fn write_jsonb_bool(b: bool, buffer: &mut Vec) -> serialize::Result { // Use the constants for true and false - let byte = if b { JSONB_TRUE } else { JSONB_FALSE }; - buffer.push(byte); + write_jsonb_header(buffer, if b { JSONB_TRUE } else { JSONB_FALSE }, 0x0)?; Ok(IsNull::No) } @@ -352,9 +351,10 @@ mod tests { use super::*; use crate::query_dsl::RunQueryDsl; use crate::test_helpers::connection; + use crate::ExpressionMethods; use crate::{dsl::sql, IntoSql}; use serde_json::{json, Value}; - use sql_types::Jsonb; + use sql_types::{Json, Jsonb}; // Helper function to create a basic JSONB header byte fn create_header(element_type: u8, payload_size: u8) -> u8 { @@ -527,10 +527,18 @@ mod tests { fn json_to_sql() { let conn = &mut connection(); let value = json!(true); - let res = diesel::select(value.into_sql::().eq(&sql("true"))) - .get_result::(conn) - .unwrap(); - assert!(res); + let res = diesel::select(value.into_sql::().eq(&sql("json('true')"))) + .get_result::(conn); + assert!(res.unwrap()); + } + + #[test] + fn jsonb_to_sql() { + let conn = &mut connection(); + let value = json!(true); + let res = diesel::select(value.into_sql::().eq(&sql("jsonb('true')"))) + .get_result::(conn); + assert!(res.unwrap()); } // #[test] diff --git a/diesel/src/type_impls/json.rs b/diesel/src/type_impls/json.rs index 15b22a46112e..9d19c464fa90 100644 --- a/diesel/src/type_impls/json.rs +++ b/diesel/src/type_impls/json.rs @@ -3,11 +3,11 @@ use crate::deserialize::FromSqlRow; use crate::expression::AsExpression; use crate::sql_types::Json; -#[cfg(feature = "postgres_backend")] +#[cfg(any(feature = "postgres_backend", feature = "sqlite"))] use crate::sql_types::Jsonb; #[derive(AsExpression, FromSqlRow)] #[diesel(foreign_derive)] #[diesel(sql_type = Json)] -#[cfg_attr(feature = "postgres_backend", diesel(sql_type = Jsonb))] +#[cfg_attr(any(feature = "postgres_backend", feature = "sqlite"), diesel(sql_type = Jsonb))] struct SerdeJsonValueProxy(serde_json::Value); diff --git a/diesel/src/type_impls/mod.rs b/diesel/src/type_impls/mod.rs index ff167bd536a8..629fc02d06d7 100644 --- a/diesel/src/type_impls/mod.rs +++ b/diesel/src/type_impls/mod.rs @@ -2,7 +2,11 @@ mod date_and_time; mod decimal; #[cfg(all( feature = "serde_json", - any(feature = "postgres_backend", feature = "mysql_backend") + any( + feature = "postgres_backend", + feature = "mysql_backend", + feature = "sqlite" + ) ))] mod json; mod option; From 1e1c31b22d74d842d14f2deff01875d84f424ccf Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Wed, 25 Sep 2024 18:33:10 +0200 Subject: [PATCH 08/24] add more json/jsonb tests for sqlite --- diesel/src/sqlite/types/json.rs | 109 ++++++++++---------------------- 1 file changed, 35 insertions(+), 74 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index f3f4613cb709..726d1c93fe47 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -526,8 +526,7 @@ mod tests { #[test] fn json_to_sql() { let conn = &mut connection(); - let value = json!(true); - let res = diesel::select(value.into_sql::().eq(&sql("json('true')"))) + let res = diesel::select(json!(true).into_sql::().eq(&sql("json('true')"))) .get_result::(conn); assert!(res.unwrap()); } @@ -535,80 +534,42 @@ mod tests { #[test] fn jsonb_to_sql() { let conn = &mut connection(); - let value = json!(true); - let res = diesel::select(value.into_sql::().eq(&sql("jsonb('true')"))) + let res = diesel::select(json!(true).into_sql::().eq(&sql("jsonb('true')"))) .get_result::(conn); assert!(res.unwrap()); } - // #[test] - // fn some_json_from_sql() { - // let input_json = b"true"; - // let output_json: serde_json::Value = - // FromSql::::from_sql(SqliteValue::for_test(input_json)) - // .unwrap(); - // assert_eq!(output_json, serde_json::Value::Bool(true)); - // } - - // #[test] - // fn bad_json_from_sql() { - // let uuid: Result = - // FromSql::::from_sql(SqliteValue::for_test(b"boom")); - // assert_eq!(uuid.unwrap_err().to_string(), "Invalid Json"); - // } - - // #[test] - // fn no_json_from_sql() { - // let uuid: Result = - // FromSql::::from_nullable_sql(None); - // assert_eq!( - // uuid.unwrap_err().to_string(), - // "Unexpected null for non-null column" - // ); - // } - - // #[test] - // fn jsonb_to_sql() { - // let mut buffer = Vec::new(); - // let mut bytes = Output::test(ByteWrapper(&mut buffer)); - // let test_json = serde_json::Value::Bool(true); - // ToSql::::to_sql(&test_json, &mut bytes).unwrap(); - // assert_eq!(buffer, b"\x01true"); - // } - - // #[test] - // fn some_jsonb_from_sql() { - // let input_json = b"\x01true"; - // let output_json: serde_json::Value = - // FromSql::::from_sql(SqliteValue::for_test(input_json)) - // .unwrap(); - // assert_eq!(output_json, serde_json::Value::Bool(true)); - // } - - // #[test] - // fn bad_jsonb_from_sql() { - // let uuid: Result = - // FromSql::::from_sql(SqliteValue::for_test(b"\x01boom")); - // assert_eq!(uuid.unwrap_err().to_string(), "Invalid Json"); - // } - - // #[test] - // fn bad_jsonb_version_from_sql() { - // let uuid: Result = - // FromSql::::from_sql(SqliteValue::for_test(b"\x02true")); - // assert_eq!( - // uuid.unwrap_err().to_string(), - // "Unsupported JSONB encoding version" - // ); - // } - - // #[test] - // fn no_jsonb_from_sql() { - // let uuid: Result = - // FromSql::::from_nullable_sql(None); - // assert_eq!( - // uuid.unwrap_err().to_string(), - // "Unexpected null for non-null column" - // ); - // } + #[test] + fn bad_json_from_sql() { + let conn = &mut connection(); + let res = diesel::select(json!(true).into_sql::().eq(&sql("json('boom')"))) + .get_result::(conn); + assert_eq!(res.unwrap_err().to_string(), "malformed JSON"); + } + + #[test] + fn bad_jsonb_from_sql() { + let conn = &mut connection(); + let res = diesel::select(json!(true).into_sql::().eq(&sql("jsonb('boom')"))) + .get_result::(conn); + assert_eq!(res.unwrap_err().to_string(), "malformed JSON"); + } + + #[test] + fn no_json_from_sql() { + let uuid: Result = FromSql::::from_nullable_sql(None); + assert_eq!( + uuid.unwrap_err().to_string(), + "Unexpected null for non-null column" + ); + } + + #[test] + fn no_jsonb_from_sql() { + let uuid: Result = FromSql::::from_nullable_sql(None); + assert_eq!( + uuid.unwrap_err().to_string(), + "Unexpected null for non-null column" + ); + } } From 0f0d9a60704e972da876908cc858e3619b159315 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Wed, 25 Sep 2024 18:42:02 +0200 Subject: [PATCH 09/24] fix broken ref to doctest_setup.rs --- diesel/src/sql_types/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index 85c1ef01a211..e041f99db188 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -472,7 +472,7 @@ pub struct Json; /// /// ```rust /// # #![allow(dead_code)] -/// # include!("../../doctest_setup.rs"); +/// # include!("../doctest_setup.rs"); /// # /// table! { /// contacts { From 209c862cd7d7c4c958c33f25d203f2061b3f34d9 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Thu, 26 Sep 2024 15:48:10 +0200 Subject: [PATCH 10/24] add Jsonb to SQLite tests --- diesel/src/sqlite/types/json.rs | 362 +++++++++++++++++++++++++------- 1 file changed, 287 insertions(+), 75 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 726d1c93fe47..b70366ee416c 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -41,7 +41,9 @@ impl FromSql for serde_json::Value { } // Read the JSONB value from the byte stream - read_jsonb_value(bytes).map(|(v, _)| v) + let (jsonb, size) = read_jsonb_value(bytes)?; + + Ok(jsonb) } } @@ -53,6 +55,9 @@ impl ToSql for serde_json::Value { // Write the JSON value into the buffer in JSONB format write_jsonb_value(self, &mut buffer)?; + let (test, size): (serde_json::Value, _) = read_jsonb_value(&buffer).unwrap(); + let test = serde_json::to_string_pretty(&test).unwrap(); + // Set the serialized binary data to the output out.set_value(buffer); @@ -111,7 +116,13 @@ fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result<(serde_json::Value, usi let total_size = header_size + payload_size; if bytes.len() < total_size { - return Err("Invalid JSONB data: insufficient bytes for value".into()); + return Err(format!( + "Invalid JSONB data: insufficient bytes for value of type {}, expected {} bytes, got {}", + element_type, + total_size, + bytes.len() + ) + .into()); } let payload_bytes = &bytes[header_size..total_size]; @@ -125,7 +136,7 @@ fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result<(serde_json::Value, usi JSONB_FLOAT => read_jsonb_float(payload_bytes, payload_size), JSONB_FLOAT5 => Err("FLOAT5 is not supported in this implementation".into()), JSONB_TEXT | JSONB_TEXTJ | JSONB_TEXTRAW => read_jsonb_text(payload_bytes, payload_size), - JSONB_TEXT5 => Err("TEXT5 is not supported in this implementation".into()), + JSONB_TEXT5 => read_jsonb_text(payload_bytes, payload_size), JSONB_ARRAY => read_jsonb_array(payload_bytes, payload_size), JSONB_OBJECT => read_jsonb_object(payload_bytes, payload_size), _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), @@ -201,21 +212,21 @@ fn read_jsonb_object(bytes: &[u8], payload_size: usize) -> deserialize::Result deserialize::Result Result, String> { + // Check if payload size exceeds the maximum allowed size + if payload_size > 2_147_483_647 { + return Err("Payload size exceeds the maximum allowed size of 2GB".into()); + } + + let mut header = Vec::new(); + + if payload_size <= 0x0B { + // Small payloads (size fits in 4 bits) + header.push((payload_size as u8) << 4 | element_type); + } else if payload_size <= 0xFF { + // Medium payloads, 1 additional byte for size + header.push(0x0C << 4 | element_type); + header.push(payload_size as u8); + } else if payload_size <= 0xFFFF { + // Larger payloads, 2 additional bytes for size + header.push(0x0D << 4 | element_type); + header.extend_from_slice(&(payload_size as u16).to_be_bytes()); + } else { + // Very large payloads, 4 additional bytes for size (up to 2 GiB) + header.push(0x0E << 4 | element_type); + header.extend_from_slice(&(payload_size as u32).to_be_bytes()); + } + + Ok(header) +} + fn write_jsonb_header( buffer: &mut Vec, element_type: u8, payload_size: usize, ) -> serialize::Result { - let payload_size = u8::try_from(payload_size).map_err(Box::new)?; - - buffer.push((payload_size << 4) | element_type); - + // Create the header and append it to the buffer + let header = create_jsonb_header(element_type, payload_size)?; + buffer.extend(header); Ok(IsNull::No) } @@ -356,9 +395,13 @@ mod tests { use serde_json::{json, Value}; use sql_types::{Json, Jsonb}; - // Helper function to create a basic JSONB header byte - fn create_header(element_type: u8, payload_size: u8) -> u8 { - (payload_size << 4) | element_type + #[test] + fn json_to_sql() { + let conn = &mut connection(); + let res = diesel::select(json!(true).into_sql::().eq(&sql("json('true')"))) + .get_result::(conn) + .unwrap(); + assert!(res); } #[test] @@ -384,21 +427,33 @@ mod tests { #[test] fn test_read_jsonb_int() { - let data = vec![create_header(JSONB_INT, 0x01), b'1']; // JSONB_INT with payload "1" + // JSONB_INT with payload "1" + let mut data = Vec::new(); + data.extend(create_jsonb_header(JSONB_INT, 0x01).unwrap()); + data.push(b'1'); // Add the payload (integer "1") + let result = read_jsonb_value(&data).unwrap().0; assert_eq!(result, json!(1)); } #[test] fn test_read_jsonb_float() { - let data = vec![create_header(JSONB_FLOAT, 0x03), b'1', b'.', b'5']; // JSONB_FLOAT with payload "1.5" + // JSONB_FLOAT with payload "1.5" + let mut data = Vec::new(); + data.extend(create_jsonb_header(JSONB_FLOAT, 0x03).unwrap()); + data.extend_from_slice(b"1.5"); // Add the payload (float "1.5") + let result = read_jsonb_value(&data).unwrap().0; assert_eq!(result, json!(1.5)); } #[test] fn test_read_jsonb_text() { - let data = vec![create_header(JSONB_TEXT, 0x03), b'f', b'o', b'o']; // JSONB_TEXT with payload "foo" + // JSONB_TEXT with payload "foo" + let mut data = Vec::new(); + data.extend(create_jsonb_header(JSONB_TEXT, 0x03).unwrap()); + data.extend_from_slice(b"foo"); // Add the payload (text "foo") + let result = read_jsonb_value(&data).unwrap().0; assert_eq!(result, json!("foo")); } @@ -406,12 +461,16 @@ mod tests { #[test] fn test_read_jsonb_array() { // JSONB_ARRAY with two elements: 1 and true - let data = vec![ - create_header(JSONB_ARRAY, 0x03), // Array header - create_header(JSONB_INT, 0x01), - b'1', // Element 1: integer "1" - create_header(JSONB_TRUE, 0x00), // Element 2: true - ]; + let mut data = Vec::new(); + data.extend(create_jsonb_header(JSONB_ARRAY, 0x03).unwrap()); // Array header + + // Element 1: integer "1" + data.extend(create_jsonb_header(JSONB_INT, 0x01).unwrap()); + data.push(b'1'); + + // Element 2: true + data.extend(create_jsonb_header(JSONB_TRUE, 0x00).unwrap()); + let result = read_jsonb_value(&data).unwrap().0; assert_eq!(result, json!([1, true])); } @@ -419,20 +478,55 @@ mod tests { #[test] fn test_read_jsonb_object() { // JSONB_OBJECT with one key-value pair: "key": 42 - let data = vec![ - create_header(JSONB_OBJECT, 0x07), // Object header - create_header(JSONB_TEXT, 0x03), - b'k', - b'e', - b'y', // Key: "key" - create_header(JSONB_INT, 0x02), - b'4', - b'2', // Value: 42 - ]; + let mut data = Vec::new(); + data.extend(create_jsonb_header(JSONB_OBJECT, 0x07).unwrap()); // Object header + + // Key: "key" + data.extend(create_jsonb_header(JSONB_TEXT, 0x03).unwrap()); + data.extend_from_slice(b"key"); // Add the key payload + + // Value: 42 (integer) + data.extend(create_jsonb_header(JSONB_INT, 0x02).unwrap()); + data.extend_from_slice(b"42"); // Add the integer payload + let result = read_jsonb_value(&data).unwrap().0; assert_eq!(result, json!({"key": 42})); } + #[test] + fn test_read_jsonb_nested_object() { + let mut data = Vec::new(); + + data.extend(create_jsonb_header(JSONB_OBJECT, 42).unwrap()); + + data.extend(create_jsonb_header(JSONB_TEXT, 9).unwrap()); + data.extend_from_slice(b"outer_key"); + + data.extend(create_jsonb_header(JSONB_OBJECT, 13).unwrap()); + + data.extend(create_jsonb_header(JSONB_TEXT, 9).unwrap()); + data.extend_from_slice(b"inner_key"); + + data.extend(create_jsonb_header(JSONB_INT, 2).unwrap()); + data.extend_from_slice(b"42"); + + data.extend(create_jsonb_header(JSONB_TEXT, 14).unwrap()); + data.extend_from_slice(b"additional_key"); + + data.extend(create_jsonb_header(JSONB_TRUE, 0).unwrap()); + + let result = read_jsonb_value(&data).unwrap().0; + assert_eq!( + result, + json!({ + "outer_key": { + "inner_key": 42 + }, + "additional_key": true + }) + ); + } + #[test] fn test_write_jsonb_null() { let value = serde_json::Value::Null; @@ -462,7 +556,12 @@ mod tests { let value = serde_json::Value::Number(serde_json::Number::from(1)); let mut buffer = Vec::new(); write_jsonb_value(&value, &mut buffer).unwrap(); - assert_eq!(buffer, vec![create_header(JSONB_INT, 0x01), b'1']); + + let mut expected_buffer = Vec::new(); + expected_buffer.extend(create_jsonb_header(JSONB_INT, 0x01).unwrap()); + expected_buffer.push(b'1'); // Payload: integer "1" + + assert_eq!(buffer, expected_buffer); } #[test] @@ -470,10 +569,12 @@ mod tests { let value = serde_json::Value::Number(serde_json::Number::from_f64(1.5).unwrap()); let mut buffer = Vec::new(); write_jsonb_value(&value, &mut buffer).unwrap(); - assert_eq!( - buffer, - vec![create_header(JSONB_FLOAT, 0x03), b'1', b'.', b'5'] - ); + + let mut expected_buffer = Vec::new(); + expected_buffer.extend(create_jsonb_header(JSONB_FLOAT, 0x03).unwrap()); + expected_buffer.extend_from_slice(b"1.5"); // Payload: float "1.5" + + assert_eq!(buffer, expected_buffer); } #[test] @@ -481,10 +582,12 @@ mod tests { let value = serde_json::Value::String("foo".to_string()); let mut buffer = Vec::new(); write_jsonb_value(&value, &mut buffer).unwrap(); - assert_eq!( - buffer, - vec![create_header(JSONB_TEXT, 0x03), b'f', b'o', b'o'] - ); + + let mut expected_buffer = Vec::new(); + expected_buffer.extend(create_jsonb_header(JSONB_TEXT, 0x03).unwrap()); + expected_buffer.extend_from_slice(b"foo"); // Payload: text "foo" + + assert_eq!(buffer, expected_buffer); } #[test] @@ -492,15 +595,14 @@ mod tests { let value = json!([1, true]); let mut buffer = Vec::new(); write_jsonb_value(&value, &mut buffer).unwrap(); - assert_eq!( - buffer, - vec![ - create_header(JSONB_ARRAY, 0x03), // Array header - create_header(JSONB_INT, 0x01), // Integer header - b'1', // Integer payload "1" - create_header(JSONB_TRUE, 0x00), // Boolean header for "true" - ] - ); + + let mut expected_buffer = Vec::new(); + expected_buffer.extend(create_jsonb_header(JSONB_ARRAY, 0x03).unwrap()); // Array header + expected_buffer.extend(create_jsonb_header(JSONB_INT, 0x01).unwrap()); // Integer header + expected_buffer.push(b'1'); // Integer payload "1" + expected_buffer.extend(create_jsonb_header(JSONB_TRUE, 0x00).unwrap()); // Boolean header for "true" + + assert_eq!(buffer, expected_buffer); } #[test] @@ -508,35 +610,145 @@ mod tests { let value = json!({"key": 42}); let mut buffer = Vec::new(); write_jsonb_value(&value, &mut buffer).unwrap(); - assert_eq!( - buffer, - vec![ - create_header(JSONB_OBJECT, 0x07), // Object header - create_header(JSONB_TEXT, 0x03), // Key header: "key" - b'k', - b'e', - b'y', // Key: "key" - create_header(JSONB_INT, 0x02), // Value header: integer "42" - b'4', - b'2', // Value: 42 - ] - ); + + let mut expected = Vec::new(); + expected.extend(create_jsonb_header(JSONB_OBJECT, 7).unwrap()); + expected.extend(create_jsonb_header(JSONB_TEXT, 3).unwrap()); + expected.extend_from_slice(b"key"); + expected.extend(create_jsonb_header(JSONB_INT, 2).unwrap()); + expected.extend_from_slice(b"42"); + + assert_eq!(buffer, expected,); } #[test] - fn json_to_sql() { + fn jsonb_to_sql_bool() { let conn = &mut connection(); - let res = diesel::select(json!(true).into_sql::().eq(&sql("json('true')"))) - .get_result::(conn); - assert!(res.unwrap()); + let res = diesel::select(json!(true).into_sql::().eq(&sql("jsonb('true')"))) + .get_result::(conn) + .unwrap(); + assert!(res); } #[test] - fn jsonb_to_sql() { + fn jsonb_to_sql_null() { let conn = &mut connection(); - let res = diesel::select(json!(true).into_sql::().eq(&sql("jsonb('true')"))) - .get_result::(conn); - assert!(res.unwrap()); + let res = diesel::select(json!(null).into_sql::().eq(&sql("jsonb('null')"))) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_integer() { + let conn = &mut connection(); + let res = diesel::select(json!(42).into_sql::().eq(&sql("jsonb('42')"))) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_float() { + let conn = &mut connection(); + let res = diesel::select(json!(3.14).into_sql::().eq(&sql("jsonb('3.14')"))) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_string() { + let conn = &mut connection(); + let res = diesel::select( + json!("hello") + .into_sql::() + .eq(&sql("jsonb('\"hello\"')")), + ) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_array() { + let conn = &mut connection(); + let res = diesel::select( + json!([1, true, "foo"]) + .into_sql::() + .eq(&sql("jsonb('[1, true, \"foo\"]')")), + ) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_object() { + let conn = &mut connection(); + let res = diesel::select( + json!({"key": "value"}) + .into_sql::() + .eq(&sql("jsonb('{\"key\": \"value\"}')")), + ) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_object_in_object() { + let conn = &mut connection(); + let json_value = json!({ + "outer_key": { + "additional_key": true, + "inner_key": { + "nested_key": 42 + }, + } + }); + let res = diesel::select(json_value.into_sql::().eq(&sql( + r#"jsonb('{"outer_key": {"additional_key": true, "inner_key": {"nested_key": 42}}}')"#, + ))) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_array_in_object() { + let conn = &mut connection(); + let json_value = json!({ + "is_valid": false, + "key": [1, 2, 3], + }); + let res = diesel::select( + json_value + .into_sql::() + .eq(&sql(r#"jsonb('{"is_valid": false, "key": [1, 2, 3]}')"#)), + ) + .get_result::(conn) + .unwrap(); + assert!(res); + } + + #[test] + fn jsonb_to_sql_object_in_array() { + let conn = &mut connection(); + let json_value = json!([ + { + "nested_key": "nested_value" + }, + { + "int_value": 99 + } + ]); + let res = diesel::select(json_value.into_sql::().eq(&sql( + r#"jsonb('[{"nested_key": "nested_value"}, {"int_value": 99}]')"#, + ))) + .get_result::(conn) + .unwrap(); + assert!(res); } #[test] From fd7f56588e0494c6389bf24b1e1c01a671ecfeea Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Thu, 26 Sep 2024 17:14:04 +0200 Subject: [PATCH 11/24] add support for SQLite JSONB TEXTJ fields --- diesel/src/sqlite/types/json.rs | 92 ++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index b70366ee416c..1360a1d7a8d9 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -132,11 +132,13 @@ fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result<(serde_json::Value, usi JSONB_TRUE => Ok(serde_json::Value::Bool(true)), JSONB_FALSE => Ok(serde_json::Value::Bool(false)), JSONB_INT => read_jsonb_int(payload_bytes, payload_size), - JSONB_INT5 => Err("INT5 is not supported in this implementation".into()), + JSONB_INT5 => Err("INT5 is not supported".into()), JSONB_FLOAT => read_jsonb_float(payload_bytes, payload_size), - JSONB_FLOAT5 => Err("FLOAT5 is not supported in this implementation".into()), - JSONB_TEXT | JSONB_TEXTJ | JSONB_TEXTRAW => read_jsonb_text(payload_bytes, payload_size), - JSONB_TEXT5 => read_jsonb_text(payload_bytes, payload_size), + JSONB_FLOAT5 => Err("FLOAT5 is not supported".into()), + JSONB_TEXT => read_jsonb_text(payload_bytes, payload_size), + JSONB_TEXTJ => read_jsonb_textj(payload_bytes, payload_size), + JSONB_TEXTRAW => Err("TEXTRAW is not supported".into()), + JSONB_TEXT5 => Err("TEXT5 is not supported".into()), JSONB_ARRAY => read_jsonb_array(payload_bytes, payload_size), JSONB_OBJECT => read_jsonb_object(payload_bytes, payload_size), _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), @@ -188,6 +190,17 @@ fn read_jsonb_text(bytes: &[u8], payload_size: usize) -> deserialize::Result deserialize::Result { + let text_bytes = &bytes[..payload_size]; + let text = std::str::from_utf8(text_bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; + + // Unescape JSON escape sequences (e.g., "\n", "\u0020") + let unescaped_text = serde_json::from_str(&format!("\"{}\"", text)) + .map_err(|_| "Failed to parse JSON-escaped text in TEXTJ")?; + + Ok(unescaped_text) +} + // Read a JSON array fn read_jsonb_array(bytes: &[u8], payload_size: usize) -> deserialize::Result { let mut elements = Vec::new(); @@ -336,12 +349,31 @@ fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { Ok(IsNull::No) } -// Write a JSON string fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { - write_jsonb_header(buffer, JSONB_TEXT, s.len())?; + if s.chars().any(|c| c.is_control()) { + // If the string contains control characters, treat it as TEXTJ (escaped JSON) + write_jsonb_textj(s, buffer) + } else { + write_jsonb_header(buffer, JSONB_TEXT, s.len())?; + // Write the UTF-8 text of the string as the payload (no delimiters) + buffer.extend_from_slice(s.as_bytes()); + Ok(IsNull::No) + } +} - // Write the UTF-8 text of the string as the payload (no delimiters) - buffer.extend_from_slice(s.as_bytes()); +fn write_jsonb_textj(s: &str, buffer: &mut Vec) -> serialize::Result { + // Escaping the string for JSON (e.g., \n, \uXXXX) + let escaped_string = + serde_json::to_string(s).map_err(|_| "Failed to serialize string for TEXTJ")?; + + // Remove the surrounding quotes from serde_json::to_string result + let escaped_string = &escaped_string[1..escaped_string.len() - 1]; + + // Write the header (JSONB_TEXTJ) and the length of the escaped string + write_jsonb_header(buffer, JSONB_TEXTJ, escaped_string.len())?; + + // Write the escaped string as the payload + buffer.extend_from_slice(escaped_string.as_bytes()); Ok(IsNull::No) } @@ -519,10 +551,10 @@ mod tests { assert_eq!( result, json!({ + "additional_key": true, "outer_key": { "inner_key": 42 }, - "additional_key": true }) ); } @@ -579,13 +611,26 @@ mod tests { #[test] fn test_write_jsonb_text() { - let value = serde_json::Value::String("foo".to_string()); let mut buffer = Vec::new(); - write_jsonb_value(&value, &mut buffer).unwrap(); + let input_string = "hello"; + write_jsonb_string(input_string, &mut buffer).unwrap(); let mut expected_buffer = Vec::new(); - expected_buffer.extend(create_jsonb_header(JSONB_TEXT, 0x03).unwrap()); - expected_buffer.extend_from_slice(b"foo"); // Payload: text "foo" + expected_buffer.extend(create_jsonb_header(JSONB_TEXT, 0x05).unwrap()); + expected_buffer.extend_from_slice(b"hello"); + + assert_eq!(buffer, expected_buffer); + } + + #[test] + fn test_write_jsonb_textj() { + let mut buffer = Vec::new(); + let input_string = "hello\nworld"; // Contains a newline, requires escaping + write_jsonb_string(input_string, &mut buffer).unwrap(); + + let mut expected_buffer = Vec::new(); + expected_buffer.extend(create_jsonb_header(JSONB_TEXTJ, 12).unwrap()); + expected_buffer.extend_from_slice(b"hello\\nworld"); assert_eq!(buffer, expected_buffer); } @@ -658,8 +703,10 @@ mod tests { } #[test] - fn jsonb_to_sql_string() { + fn jsonb_to_sql_text() { let conn = &mut connection(); + + // Test for TEXT (simple string) let res = diesel::select( json!("hello") .into_sql::() @@ -667,6 +714,23 @@ mod tests { ) .get_result::(conn) .unwrap(); + + assert!(res); + } + + #[test] + fn jsonb_to_sql_textj() { + let conn = &mut connection(); + + // Test for TEXTJ (JSON-escaped string, e.g., containing \n or \uXXXX) + let res = diesel::select( + json!("hello\nworld") + .into_sql::() + .eq(&sql("jsonb('\"hello\\nworld\"')")), // The string is JSON-escaped + ) + .get_result::(conn) + .unwrap(); + assert!(res); } From 1b95507978282e1edad551b4ae388e35ea890098 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Thu, 26 Sep 2024 17:34:22 +0200 Subject: [PATCH 12/24] fix clippy errors --- diesel/src/sqlite/types/json.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 1360a1d7a8d9..8028637185b5 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -41,7 +41,7 @@ impl FromSql for serde_json::Value { } // Read the JSONB value from the byte stream - let (jsonb, size) = read_jsonb_value(bytes)?; + let (jsonb, _size) = read_jsonb_value(bytes)?; Ok(jsonb) } @@ -55,9 +55,6 @@ impl ToSql for serde_json::Value { // Write the JSON value into the buffer in JSONB format write_jsonb_value(self, &mut buffer)?; - let (test, size): (serde_json::Value, _) = read_jsonb_value(&buffer).unwrap(); - let test = serde_json::to_string_pretty(&test).unwrap(); - // Set the serialized binary data to the output out.set_value(buffer); @@ -258,19 +255,19 @@ fn create_jsonb_header(element_type: u8, payload_size: usize) -> Result, if payload_size <= 0x0B { // Small payloads (size fits in 4 bits) - header.push((payload_size as u8) << 4 | element_type); + header.push((u8::try_from(payload_size).unwrap()) << 4 | element_type); } else if payload_size <= 0xFF { // Medium payloads, 1 additional byte for size header.push(0x0C << 4 | element_type); - header.push(payload_size as u8); + header.push(u8::try_from(payload_size).unwrap()); } else if payload_size <= 0xFFFF { // Larger payloads, 2 additional bytes for size header.push(0x0D << 4 | element_type); - header.extend_from_slice(&(payload_size as u16).to_be_bytes()); + header.extend_from_slice(&(u16::try_from(payload_size).unwrap()).to_be_bytes()); } else { // Very large payloads, 4 additional bytes for size (up to 2 GiB) header.push(0x0E << 4 | element_type); - header.extend_from_slice(&(payload_size as u32).to_be_bytes()); + header.extend_from_slice(&(u32::try_from(payload_size).unwrap()).to_be_bytes()); } Ok(header) @@ -696,7 +693,7 @@ mod tests { #[test] fn jsonb_to_sql_float() { let conn = &mut connection(); - let res = diesel::select(json!(3.14).into_sql::().eq(&sql("jsonb('3.14')"))) + let res = diesel::select(json!(42.23).into_sql::().eq(&sql("jsonb('42.23')"))) .get_result::(conn) .unwrap(); assert!(res); From 3a1a612c27ebe5aec9c8c72db2440d5258d6fdef Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 12:14:47 +0200 Subject: [PATCH 13/24] add a separate SQLite + Jsonb example --- diesel/src/sql_types/mod.rs | 67 +++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index e041f99db188..cea13fb586f2 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -395,13 +395,10 @@ pub struct Timestamp; /// [`ToSql`]: /serialize/trait.ToSql.html /// [`FromSql`]: /deserialize/trait.FromSql.html /// [`serde_json::Value`]: /../serde_json/value/enum.Value.html -#[cfg(all( - any( - feature = "postgres_backend", - feature = "sqlite", - feature = "mysql_backend" - ), - feature = "serde_json" +#[cfg(any( + feature = "postgres_backend", + feature = "mysql_backend", + all(feature = "sqlite", feature = "serde_json") ))] #[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] #[diesel(postgres_type(oid = 114, array_oid = 199))] @@ -468,7 +465,9 @@ pub struct Json; doc = "[`serde_json::Value`]: https://docs.rs/serde_json/1.0.64/serde_json/value/enum.Value.html" )] /// -/// # Examples +/// ## Examples +/// +/// ### PostgreSQL /// /// ```rust /// # #![allow(dead_code)] @@ -492,6 +491,52 @@ pub struct Json; /// # name VARCHAR NOT NULL, /// # address JSONB NOT NULL /// # )").execute(connection)?; +/// +/// let santas_address: serde_json::Value = serde_json::from_str(r#"{ +/// "street": "Article Circle Expressway 1", +/// "city": "North Pole", +/// "postcode": "99705", +/// "state": "Alaska" +/// }"#)?; +/// let inserted_address = insert_into(contacts) +/// .values((name.eq("Claus"), address.eq(&santas_address))) +/// .returning(address) +/// .get_result::(connection)?; +/// assert_eq!(santas_address, inserted_address); +/// # Ok(()) +/// # } +/// # #[cfg(not(feature = "serde_json"))] +/// # fn main() {} +/// ``` +/// +/// ### SQLite +/// +/// ```rust +/// # #![allow(dead_code)] +/// # include!("../doctest_setup.rs"); +/// # +/// # #[cfg(all(feature = "sqlite", feature = "serde_json"))] +/// table! { +/// contacts { +/// id -> Integer, +/// name -> Text, +/// address -> Jsonb, +/// } +/// } +/// +/// # #[cfg(all(feature = "sqlite", feature = "serde_json"))] +/// # fn main() -> Result<(), Box> { +/// # use diesel::insert_into; +/// # use diesel::query_dsl::RunQueryDsl; +/// # use diesel::test_helpers::connection; +/// # use diesel::ExpressionMethods; +/// # use diesel::{dsl::sql, IntoSql}; +/// # let connection = &mut connection(); +/// # diesel::sql_query("CREATE TABLE contacts ( +/// # id INT PRIMARY KEY, +/// # name TEXT NOT NULL, +/// # address BLOB NOT NULL +/// # )").execute(connection)?; /// let santas_address: serde_json::Value = serde_json::from_str(r#"{ /// "street": "Article Circle Expressway 1", /// "city": "North Pole", @@ -508,9 +553,9 @@ pub struct Json; /// # #[cfg(not(feature = "serde_json"))] /// # fn main() {} /// ``` -#[cfg(all( - any(feature = "postgres_backend", feature = "sqlite"), - feature = "serde_json" +#[cfg(any( + feature = "postgres_backend", + all(feature = "sqlite", feature = "serde_json") ))] #[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] #[diesel(postgres_type(oid = 3802, array_oid = 3807))] From 187ea471a2d939a473239e4c0f56d304672560d6 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 12:54:47 +0200 Subject: [PATCH 14/24] fix clippy errors --- diesel/src/sqlite/types/json.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 8028637185b5..b7385019b5ac 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -255,19 +255,23 @@ fn create_jsonb_header(element_type: u8, payload_size: usize) -> Result, if payload_size <= 0x0B { // Small payloads (size fits in 4 bits) - header.push((u8::try_from(payload_size).unwrap()) << 4 | element_type); + header.push((u8::try_from(payload_size).map_err(|e| e.to_string())?) << 4 | element_type); } else if payload_size <= 0xFF { // Medium payloads, 1 additional byte for size header.push(0x0C << 4 | element_type); - header.push(u8::try_from(payload_size).unwrap()); + header.push(u8::try_from(payload_size).map_err(|e| e.to_string())?); } else if payload_size <= 0xFFFF { // Larger payloads, 2 additional bytes for size header.push(0x0D << 4 | element_type); - header.extend_from_slice(&(u16::try_from(payload_size).unwrap()).to_be_bytes()); + header.extend_from_slice( + &(u16::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), + ); } else { // Very large payloads, 4 additional bytes for size (up to 2 GiB) header.push(0x0E << 4 | element_type); - header.extend_from_slice(&(u32::try_from(payload_size).unwrap()).to_be_bytes()); + header.extend_from_slice( + &(u32::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), + ); } Ok(header) From 268a3df167d8917c641990e3ac364448de4d3a26 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 13:17:15 +0200 Subject: [PATCH 15/24] fix Jsonb SQLite example --- diesel/src/sql_types/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index cea13fb586f2..14ca0afe8dbe 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -530,7 +530,7 @@ pub struct Json; /// # use diesel::query_dsl::RunQueryDsl; /// # use diesel::test_helpers::connection; /// # use diesel::ExpressionMethods; -/// # use diesel::{dsl::sql, IntoSql}; +/// # use diesel::dsl::*; /// # let connection = &mut connection(); /// # diesel::sql_query("CREATE TABLE contacts ( /// # id INT PRIMARY KEY, From def9cee04f0ddacd07e73c3d1100fa1e7f05dc9d Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 23:19:06 +0200 Subject: [PATCH 16/24] factorize the Jsonb SQLite/Pg example --- diesel/src/sql_types/mod.rs | 61 ++++++++++++------------------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index 14ca0afe8dbe..d760507f0e7e 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -467,12 +467,11 @@ pub struct Json; /// /// ## Examples /// -/// ### PostgreSQL -/// /// ```rust /// # #![allow(dead_code)] /// # include!("../doctest_setup.rs"); /// # +/// # #[cfg(all(feature = "postgres_backend", feature = "serde_json"))] /// table! { /// contacts { /// id -> Integer, @@ -480,8 +479,20 @@ pub struct Json; /// address -> Jsonb, /// } /// } +/// # #[cfg(all( +/// # feature = "sqlite", +/// # feature = "serde_json", +/// # feature = "returning_clauses_for_sqlite_3_35" +/// # ))] +/// table! { +/// contacts { +/// id -> Integer, +/// name -> Text, +/// address -> Jsonb, +/// } +/// } /// -/// # #[cfg(feature = "serde_json")] +/// # #[cfg(all(feature = "postgres_backend", feature = "serde_json"))] /// # fn main() -> Result<(), Box> { /// # use diesel::insert_into; /// # use self::contacts::dsl::*; @@ -491,40 +502,11 @@ pub struct Json; /// # name VARCHAR NOT NULL, /// # address JSONB NOT NULL /// # )").execute(connection)?; -/// -/// let santas_address: serde_json::Value = serde_json::from_str(r#"{ -/// "street": "Article Circle Expressway 1", -/// "city": "North Pole", -/// "postcode": "99705", -/// "state": "Alaska" -/// }"#)?; -/// let inserted_address = insert_into(contacts) -/// .values((name.eq("Claus"), address.eq(&santas_address))) -/// .returning(address) -/// .get_result::(connection)?; -/// assert_eq!(santas_address, inserted_address); -/// # Ok(()) -/// # } -/// # #[cfg(not(feature = "serde_json"))] -/// # fn main() {} -/// ``` -/// -/// ### SQLite -/// -/// ```rust -/// # #![allow(dead_code)] -/// # include!("../doctest_setup.rs"); -/// # -/// # #[cfg(all(feature = "sqlite", feature = "serde_json"))] -/// table! { -/// contacts { -/// id -> Integer, -/// name -> Text, -/// address -> Jsonb, -/// } -/// } -/// -/// # #[cfg(all(feature = "sqlite", feature = "serde_json"))] +/// # #[cfg(all( +/// # feature = "sqlite", +/// # feature = "serde_json", +/// # feature = "returning_clauses_for_sqlite_3_35" +/// # ))] /// # fn main() -> Result<(), Box> { /// # use diesel::insert_into; /// # use diesel::query_dsl::RunQueryDsl; @@ -532,11 +514,6 @@ pub struct Json; /// # use diesel::ExpressionMethods; /// # use diesel::dsl::*; /// # let connection = &mut connection(); -/// # diesel::sql_query("CREATE TABLE contacts ( -/// # id INT PRIMARY KEY, -/// # name TEXT NOT NULL, -/// # address BLOB NOT NULL -/// # )").execute(connection)?; /// let santas_address: serde_json::Value = serde_json::from_str(r#"{ /// "street": "Article Circle Expressway 1", /// "city": "North Pole", From ae024947fdb87ffe07f644a530941dd8679e7673 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 23:23:44 +0200 Subject: [PATCH 17/24] add Jsonb scalar fields payload size consistency checks --- diesel/src/sqlite/types/json.rs | 38 +++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index b7385019b5ac..373fdc2a34b1 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -157,8 +157,7 @@ fn read_jsonb_int(bytes: &[u8], payload_size: usize) -> deserialize::Result() @@ -170,7 +169,16 @@ fn read_jsonb_int(bytes: &[u8], payload_size: usize) -> deserialize::Result deserialize::Result { +fn read_jsonb_float(bytes: &[u8], payload_size: usize) -> deserialize::Result { + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } + let float_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB float")?; let float_value = float_str .parse::() @@ -182,14 +190,30 @@ fn read_jsonb_float(bytes: &[u8], _payload_size: usize) -> deserialize::Result deserialize::Result { - let text_bytes = &bytes[..payload_size]; - let text = std::str::from_utf8(text_bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } + + let text = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; Ok(serde_json::Value::String(text.to_string())) } fn read_jsonb_textj(bytes: &[u8], payload_size: usize) -> deserialize::Result { - let text_bytes = &bytes[..payload_size]; - let text = std::str::from_utf8(text_bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } + + let text = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; // Unescape JSON escape sequences (e.g., "\n", "\u0020") let unescaped_text = serde_json::from_str(&format!("\"{}\"", text)) From 2af1253437d3883141a42114e7afaab8b672ab53 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 23:33:21 +0200 Subject: [PATCH 18/24] avoid Vec re-allocations in create_jsonb_header --- diesel/src/sqlite/types/json.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 373fdc2a34b1..9a267cf7e2b3 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -275,28 +275,36 @@ fn create_jsonb_header(element_type: u8, payload_size: usize) -> Result, return Err("Payload size exceeds the maximum allowed size of 2GB".into()); } - let mut header = Vec::new(); - - if payload_size <= 0x0B { - // Small payloads (size fits in 4 bits) - header.push((u8::try_from(payload_size).map_err(|e| e.to_string())?) << 4 | element_type); + let header = if payload_size <= 0x0B { + // Small payloads, 0 additional byte for size + vec![(u8::try_from(payload_size).map_err(|e| e.to_string())?) << 4 | element_type] } else if payload_size <= 0xFF { // Medium payloads, 1 additional byte for size - header.push(0x0C << 4 | element_type); - header.push(u8::try_from(payload_size).map_err(|e| e.to_string())?); + vec![ + 0x0C << 4 | element_type, + u8::try_from(payload_size).map_err(|e| e.to_string())?, + ] } else if payload_size <= 0xFFFF { + let mut header = Vec::with_capacity(3); + // Larger payloads, 2 additional bytes for size header.push(0x0D << 4 | element_type); header.extend_from_slice( &(u16::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), ); + + header } else { + let mut header = Vec::with_capacity(5); + // Very large payloads, 4 additional bytes for size (up to 2 GiB) header.push(0x0E << 4 | element_type); header.extend_from_slice( &(u32::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), ); - } + + header + }; Ok(header) } From 7fe0c670bda981f40ff2b893f6b8179ed69b14b3 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 23:40:02 +0200 Subject: [PATCH 19/24] add tests for FromSql --- diesel/src/sqlite/types/json.rs | 122 ++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 9a267cf7e2b3..9350e61b6c1f 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -848,6 +848,128 @@ mod tests { assert!(res); } + #[test] + fn jsonb_from_sql_null() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('null')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!(null)); + } + + #[test] + fn jsonb_from_sql_true() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('true')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!(true)); + } + + #[test] + fn jsonb_from_sql_false() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('false')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!(false)); + } + + #[test] + fn jsonb_from_sql_int() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('42')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!(42)); + } + + #[test] + fn jsonb_from_sql_float() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('3.14')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!(3.14)); + } + + #[test] + fn jsonb_from_sql_object() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('{\"key\": \"value\"}')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!({"key": "value"})); + } + + #[test] + fn jsonb_from_sql_array() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('[1, 2, 3]')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!([1, 2, 3])); + } + + #[test] + fn jsonb_from_sql_nested_objects() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('{\"outer\": {\"inner\": 42}}')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!({"outer": {"inner": 42}})); + } + + #[test] + fn jsonb_from_sql_nested_arrays() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('[[1, 2], [3, 4]]')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!([[1, 2], [3, 4]])); + } + + #[test] + fn jsonb_from_sql_nested_arrays_in_objects() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('{\"array\": [1, 2, 3]}')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!({"array": [1, 2, 3]})); + } + + #[test] + fn jsonb_from_sql_nested_objects_in_arrays() { + let conn = &mut connection(); + let res = diesel::select(sql::( + "jsonb('[{\"key1\": \"value1\"}, {\"key2\": \"value2\"}]')", + )) + .get_result::(conn) + .unwrap(); + assert_eq!( + res, + serde_json::json!([{"key1": "value1"}, {"key2": "value2"}]) + ); + } + + #[test] + fn jsonb_from_sql_text() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('\"hello\"')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!("hello")); + } + + #[test] + fn jsonb_from_sql_textj() { + let conn = &mut connection(); + let res = diesel::select(sql::("jsonb('\"hello\\nworld\"')")) + .get_result::(conn) + .unwrap(); + assert_eq!(res, serde_json::json!("hello\nworld")); + } + #[test] fn bad_json_from_sql() { let conn = &mut connection(); From 76ac65045a8cc28dd530d67b419c2aefaf09963c Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 23:42:14 +0200 Subject: [PATCH 20/24] update CHANGELOG with SQLite+Json/Jsonb support --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86f3e4537285..bee40b24f51b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Increasing the minimal supported Rust version will always be coupled at least wi * Support for postgres multirange type * Added `diesel::r2d2::TestCustomizer`, which allows users to customize their `diesel::r2d2::Pool`s in a way that makes the pools suitable for use in parallel tests. +* Added `Json` and `Jsonb` support for the SQLite backend. ## [2.2.2] 2024-07-19 From 4dd26d92c3c2d4ea5a6fcd8cb138393b84cc5329 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Fri, 27 Sep 2024 23:54:24 +0200 Subject: [PATCH 21/24] fix clippy error --- diesel/src/sqlite/types/json.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 9350e61b6c1f..91fe45ee0b7f 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -887,10 +887,10 @@ mod tests { #[test] fn jsonb_from_sql_float() { let conn = &mut connection(); - let res = diesel::select(sql::("jsonb('3.14')")) + let res = diesel::select(sql::("jsonb('42.23')")) .get_result::(conn) .unwrap(); - assert_eq!(res, serde_json::json!(3.14)); + assert_eq!(res, serde_json::json!(42.23)); } #[test] From fd1022854d9235fce3b220ce16a9fbbe04ef4d11 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Sat, 28 Sep 2024 00:16:13 +0200 Subject: [PATCH 22/24] factorize the Jsonb SQLite/Pg example --- diesel/src/sql_types/mod.rs | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index d760507f0e7e..31ac664142b7 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -471,19 +471,6 @@ pub struct Json; /// # #![allow(dead_code)] /// # include!("../doctest_setup.rs"); /// # -/// # #[cfg(all(feature = "postgres_backend", feature = "serde_json"))] -/// table! { -/// contacts { -/// id -> Integer, -/// name -> VarChar, -/// address -> Jsonb, -/// } -/// } -/// # #[cfg(all( -/// # feature = "sqlite", -/// # feature = "serde_json", -/// # feature = "returning_clauses_for_sqlite_3_35" -/// # ))] /// table! { /// contacts { /// id -> Integer, @@ -492,11 +479,11 @@ pub struct Json; /// } /// } /// -/// # #[cfg(all(feature = "postgres_backend", feature = "serde_json"))] /// # fn main() -> Result<(), Box> { /// # use diesel::insert_into; /// # use self::contacts::dsl::*; /// # let connection = &mut connection_no_data(); +/// # #[cfg(all(feature = "postgres_backend", feature = "serde_json"))] /// # diesel::sql_query("CREATE TABLE contacts ( /// # id SERIAL PRIMARY KEY, /// # name VARCHAR NOT NULL, @@ -507,13 +494,11 @@ pub struct Json; /// # feature = "serde_json", /// # feature = "returning_clauses_for_sqlite_3_35" /// # ))] -/// # fn main() -> Result<(), Box> { -/// # use diesel::insert_into; -/// # use diesel::query_dsl::RunQueryDsl; -/// # use diesel::test_helpers::connection; -/// # use diesel::ExpressionMethods; -/// # use diesel::dsl::*; -/// # let connection = &mut connection(); +/// # diesel::sql_query("CREATE TABLE contacts ( +/// # id INT PRIMARY KEY, +/// # name TEXT NOT NULL, +/// # address BLOB NOT NULL +/// # )").execute(connection)?; /// let santas_address: serde_json::Value = serde_json::from_str(r#"{ /// "street": "Article Circle Expressway 1", /// "city": "North Pole", From adc5ebf363947d7488e8e23f8b4d4e0d2e449831 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 1 Oct 2024 10:11:28 +0200 Subject: [PATCH 23/24] add a feature gate for SQLite's Json and Jsonb FromSql/ToSql impls --- diesel/src/sqlite/types/json.rs | 747 +++++++++++++++++--------------- 1 file changed, 407 insertions(+), 340 deletions(-) diff --git a/diesel/src/sqlite/types/json.rs b/diesel/src/sqlite/types/json.rs index 91fe45ee0b7f..58511806f306 100644 --- a/diesel/src/sqlite/types/json.rs +++ b/diesel/src/sqlite/types/json.rs @@ -1,30 +1,18 @@ -extern crate serde_json; +//! Support for JSON and JSONB values under PostgreSQL. use crate::deserialize::{self, FromSql}; use crate::serialize::{self, IsNull, Output, ToSql}; use crate::sql_types; use crate::sqlite::{Sqlite, SqliteValue}; -const JSONB_NULL: u8 = 0x00; -const JSONB_TRUE: u8 = 0x01; -const JSONB_FALSE: u8 = 0x02; -const JSONB_INT: u8 = 0x03; -const JSONB_INT5: u8 = 0x04; -const JSONB_FLOAT: u8 = 0x05; -const JSONB_FLOAT5: u8 = 0x06; -const JSONB_TEXT: u8 = 0x07; -const JSONB_TEXTJ: u8 = 0x08; -const JSONB_TEXT5: u8 = 0x09; -const JSONB_TEXTRAW: u8 = 0x0A; -const JSONB_ARRAY: u8 = 0x0B; -const JSONB_OBJECT: u8 = 0x0C; - +#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl FromSql for serde_json::Value { fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { serde_json::from_str(value.read_text()).map_err(|_| "Invalid Json".into()) } } +#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl ToSql for serde_json::Value { fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { out.set_value(serde_json::to_string(self)?); @@ -32,8 +20,11 @@ impl ToSql for serde_json::Value { } } +#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl FromSql for serde_json::Value { fn from_sql(value: SqliteValue<'_, '_, '_>) -> deserialize::Result { + use self::jsonb::*; + let bytes = value.read_blob(); if bytes.is_empty() { @@ -47,8 +38,11 @@ impl FromSql for serde_json::Value { } } +#[cfg(all(feature = "sqlite", feature = "serde_json"))] impl ToSql for serde_json::Value { fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { + use self::jsonb::*; + // Create a buffer to hold the binary JSONB encoding let mut buffer = Vec::new(); @@ -62,396 +56,469 @@ impl ToSql for serde_json::Value { } } -// Helper function to read a JSONB value from the byte stream -fn read_jsonb_value(bytes: &[u8]) -> deserialize::Result<(serde_json::Value, usize)> { - if bytes.is_empty() { - return Err("Empty JSONB data".into()); - } +#[cfg(all(feature = "sqlite", feature = "serde_json"))] +mod jsonb { + extern crate serde_json; - // The first byte contains both the element type and potentially the payload size - let first_byte = bytes[0]; - let element_type = first_byte & 0x0F; - let size_hint = (first_byte & 0xF0) >> 4; + use super::*; - let (payload_size, header_size) = match size_hint { - 0x00..=0x0B => (size_hint as usize, 1), // Payload size is directly in the upper nibble - 0x0C => { - if bytes.len() < 2 { - return Err("Invalid JSONB data: insufficient bytes for payload size".into()); - } - (bytes[1] as usize, 2) // 1 additional byte for payload size + pub(super) const JSONB_NULL: u8 = 0x00; + pub(super) const JSONB_TRUE: u8 = 0x01; + pub(super) const JSONB_FALSE: u8 = 0x02; + pub(super) const JSONB_INT: u8 = 0x03; + pub(super) const JSONB_INT5: u8 = 0x04; + pub(super) const JSONB_FLOAT: u8 = 0x05; + pub(super) const JSONB_FLOAT5: u8 = 0x06; + pub(super) const JSONB_TEXT: u8 = 0x07; + pub(super) const JSONB_TEXTJ: u8 = 0x08; + pub(super) const JSONB_TEXT5: u8 = 0x09; + pub(super) const JSONB_TEXTRAW: u8 = 0x0A; + pub(super) const JSONB_ARRAY: u8 = 0x0B; + pub(super) const JSONB_OBJECT: u8 = 0x0C; + + // Helper function to read a JSONB value from the byte stream + pub(super) fn read_jsonb_value( + bytes: &[u8], + ) -> deserialize::Result<(serde_json::Value, usize)> { + if bytes.is_empty() { + return Err("Empty JSONB data".into()); } - 0x0D => { - if bytes.len() < 3 { - return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + + // The first byte contains both the element type and potentially the payload size + let first_byte = bytes[0]; + let element_type = first_byte & 0x0F; + let size_hint = (first_byte & 0xF0) >> 4; + + let (payload_size, header_size) = match size_hint { + 0x00..=0x0B => (size_hint as usize, 1), // Payload size is directly in the upper nibble + 0x0C => { + if bytes.len() < 2 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + (bytes[1] as usize, 2) // 1 additional byte for payload size } - (u16::from_be_bytes([bytes[1], bytes[2]]) as usize, 3) // 2 additional bytes - } - 0x0E => { - if bytes.len() < 5 { - return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + 0x0D => { + if bytes.len() < 3 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + (u16::from_be_bytes([bytes[1], bytes[2]]) as usize, 3) // 2 additional bytes } - ( - u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize, - 5, - ) // 4 additional bytes - } - 0x0F => { - if bytes.len() < 9 { - return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + 0x0E => { + if bytes.len() < 5 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + ( + u32::from_be_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize, + 5, + ) // 4 additional bytes } - ( - usize::try_from(u64::from_be_bytes([ - bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], - ])) - .map_err(Box::new)?, - 9, - ) // 8 additional bytes + 0x0F => { + if bytes.len() < 9 { + return Err("Invalid JSONB data: insufficient bytes for payload size".into()); + } + ( + usize::try_from(u64::from_be_bytes([ + bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], + ])) + .map_err(Box::new)?, + 9, + ) // 8 additional bytes + } + _ => return Err("Invalid payload size hint".into()), + }; + + let total_size = header_size + payload_size; + if bytes.len() < total_size { + return Err(format!( + "Invalid JSONB data: insufficient bytes for value of type {}, expected {} bytes, got {}", + element_type, + total_size, + bytes.len() + ) + .into()); } - _ => return Err("Invalid payload size hint".into()), - }; - - let total_size = header_size + payload_size; - if bytes.len() < total_size { - return Err(format!( - "Invalid JSONB data: insufficient bytes for value of type {}, expected {} bytes, got {}", - element_type, - total_size, - bytes.len() - ) - .into()); - } - - let payload_bytes = &bytes[header_size..total_size]; - - let value = match element_type { - JSONB_NULL => Ok(serde_json::Value::Null), - JSONB_TRUE => Ok(serde_json::Value::Bool(true)), - JSONB_FALSE => Ok(serde_json::Value::Bool(false)), - JSONB_INT => read_jsonb_int(payload_bytes, payload_size), - JSONB_INT5 => Err("INT5 is not supported".into()), - JSONB_FLOAT => read_jsonb_float(payload_bytes, payload_size), - JSONB_FLOAT5 => Err("FLOAT5 is not supported".into()), - JSONB_TEXT => read_jsonb_text(payload_bytes, payload_size), - JSONB_TEXTJ => read_jsonb_textj(payload_bytes, payload_size), - JSONB_TEXTRAW => Err("TEXTRAW is not supported".into()), - JSONB_TEXT5 => Err("TEXT5 is not supported".into()), - JSONB_ARRAY => read_jsonb_array(payload_bytes, payload_size), - JSONB_OBJECT => read_jsonb_object(payload_bytes, payload_size), - _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), - }?; - - Ok((value, total_size)) -} - -// Read a JSON integer in canonical format (INT) -fn read_jsonb_int(bytes: &[u8], payload_size: usize) -> deserialize::Result { - // Ensure the bytes are at least as large as the payload size - if bytes.len() < payload_size { - return Err(format!( - "Expected payload of size {}, but got {}", - payload_size, - bytes.len() - ) - .into()); - } - // Read only the number of bytes specified by the payload size - let int_str = std::str::from_utf8(bytes).map_err(|_| "Invalid ASCII in JSONB integer")?; - // Parse the integer string into an i64 - let int_value = int_str - .parse::() - .map_err(|_| "Failed to parse JSONB integer")?; + let payload_bytes = &bytes[header_size..total_size]; + + let value = match element_type { + JSONB_NULL => Ok(serde_json::Value::Null), + JSONB_TRUE => Ok(serde_json::Value::Bool(true)), + JSONB_FALSE => Ok(serde_json::Value::Bool(false)), + JSONB_INT => read_jsonb_int(payload_bytes, payload_size), + JSONB_INT5 => Err("INT5 is not supported".into()), + JSONB_FLOAT => read_jsonb_float(payload_bytes, payload_size), + JSONB_FLOAT5 => Err("FLOAT5 is not supported".into()), + JSONB_TEXT => read_jsonb_text(payload_bytes, payload_size), + JSONB_TEXTJ => read_jsonb_textj(payload_bytes, payload_size), + JSONB_TEXTRAW => Err("TEXTRAW is not supported".into()), + JSONB_TEXT5 => Err("TEXT5 is not supported".into()), + JSONB_ARRAY => read_jsonb_array(payload_bytes, payload_size), + JSONB_OBJECT => read_jsonb_object(payload_bytes, payload_size), + _ => Err(format!("Unsupported or reserved JSONB type: {}", element_type).into()), + }?; + + Ok((value, total_size)) + } + + // Read a JSON integer in canonical format (INT) + pub(super) fn read_jsonb_int( + bytes: &[u8], + payload_size: usize, + ) -> deserialize::Result { + // Ensure the bytes are at least as large as the payload size + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } - Ok(serde_json::Value::Number(serde_json::Number::from( - int_value, - ))) -} + // Read only the number of bytes specified by the payload size + let int_str = std::str::from_utf8(bytes).map_err(|_| "Invalid ASCII in JSONB integer")?; + let int_value = serde_json::from_str(int_str) + .map_err(|_| "Failed to parse JSONB") + .and_then(|v: serde_json::Value| { + v.is_i64() + .then_some(v) + .ok_or("Failed to parse JSONB integer") + })?; + + Ok(int_value) + } + + // Read a JSON float in canonical format (FLOAT) + pub(super) fn read_jsonb_float( + bytes: &[u8], + payload_size: usize, + ) -> deserialize::Result { + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } -// Read a JSON float in canonical format (FLOAT) -fn read_jsonb_float(bytes: &[u8], payload_size: usize) -> deserialize::Result { - if bytes.len() < payload_size { - return Err(format!( - "Expected payload of size {}, but got {}", - payload_size, - bytes.len() - ) - .into()); - } + let float_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB float")?; + let float_value = serde_json::from_str(float_str) + .map_err(|_| "Failed to parse JSONB") + .and_then(|v: serde_json::Value| { + v.is_f64() + .then_some(v) + .ok_or("Failed to parse JSONB number") + })?; + + Ok(float_value) + } + + // Read a JSON string + pub(super) fn read_jsonb_text( + bytes: &[u8], + payload_size: usize, + ) -> deserialize::Result { + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } - let float_str = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB float")?; - let float_value = float_str - .parse::() - .map_err(|_| "Failed to parse JSONB float")?; - Ok(serde_json::Value::Number( - serde_json::Number::from_f64(float_value).ok_or("Invalid float value")?, - )) -} + let text = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; + Ok(serde_json::Value::String(text.to_string())) + } + + pub(super) fn read_jsonb_textj( + bytes: &[u8], + payload_size: usize, + ) -> deserialize::Result { + if bytes.len() < payload_size { + return Err(format!( + "Expected payload of size {}, but got {}", + payload_size, + bytes.len() + ) + .into()); + } -// Read a JSON string -fn read_jsonb_text(bytes: &[u8], payload_size: usize) -> deserialize::Result { - if bytes.len() < payload_size { - return Err(format!( - "Expected payload of size {}, but got {}", - payload_size, - bytes.len() - ) - .into()); - } + let text = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; - let text = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; - Ok(serde_json::Value::String(text.to_string())) -} + // Unescape JSON escape sequences (e.g., "\n", "\u0020") + let unescaped_text = serde_json::from_str(&format!("\"{}\"", text)) + .map_err(|_| "Failed to parse JSON-escaped text in TEXTJ")?; -fn read_jsonb_textj(bytes: &[u8], payload_size: usize) -> deserialize::Result { - if bytes.len() < payload_size { - return Err(format!( - "Expected payload of size {}, but got {}", - payload_size, - bytes.len() - ) - .into()); + Ok(unescaped_text) } - let text = std::str::from_utf8(bytes).map_err(|_| "Invalid UTF-8 in JSONB string")?; + // Read a JSON array + pub(super) fn read_jsonb_array( + bytes: &[u8], + payload_size: usize, + ) -> deserialize::Result { + let mut elements = Vec::new(); + let mut total_read = 0; - // Unescape JSON escape sequences (e.g., "\n", "\u0020") - let unescaped_text = serde_json::from_str(&format!("\"{}\"", text)) - .map_err(|_| "Failed to parse JSON-escaped text in TEXTJ")?; + while total_read < payload_size { + let (element, consumed) = read_jsonb_value(&bytes[total_read..payload_size])?; - Ok(unescaped_text) -} + elements.push(element); + total_read += consumed; + } -// Read a JSON array -fn read_jsonb_array(bytes: &[u8], payload_size: usize) -> deserialize::Result { - let mut elements = Vec::new(); - let mut total_read = 0; + if total_read != payload_size { + return Err("Array payload size mismatch".into()); + } - while total_read < payload_size { - let (element, consumed) = read_jsonb_value(&bytes[total_read..payload_size])?; + Ok(serde_json::Value::Array(elements)) + } + + pub(super) fn read_jsonb_object( + bytes: &[u8], + payload_size: usize, + ) -> deserialize::Result { + let mut object = serde_json::Map::new(); + let mut total_read = 0; + + while total_read < payload_size { + // Read the key (must be a valid JSONB text type) + let (key_value, key_consumed) = read_jsonb_value(&bytes[total_read..])?; + let key_str = key_value + .as_str() + .ok_or("Invalid object key in JSONB, must be a string")? + .to_string(); + total_read += key_consumed; + + // Read the value associated with the key + let (value, value_consumed) = read_jsonb_value(&bytes[total_read..])?; + object.insert(key_str, value); + total_read += value_consumed; + } - elements.push(element); - total_read += consumed; - } + // Ensure the total bytes read match the payload size + if total_read != payload_size { + return Err("Object payload size mismatch".into()); + } - if total_read != payload_size { - return Err("Array payload size mismatch".into()); + Ok(serde_json::Value::Object(object)) } - Ok(serde_json::Value::Array(elements)) -} - -fn read_jsonb_object(bytes: &[u8], payload_size: usize) -> deserialize::Result { - let mut object = serde_json::Map::new(); - let mut total_read = 0; - - while total_read < payload_size { - // Read the key (must be a valid JSONB text type) - let (key_value, key_consumed) = read_jsonb_value(&bytes[total_read..])?; - let key_str = key_value - .as_str() - .ok_or("Invalid object key in JSONB, must be a string")? - .to_string(); - total_read += key_consumed; + // Helper function to create the correct JsonbHeader based on the payload size + pub(super) fn create_jsonb_header( + element_type: u8, + payload_size: usize, + ) -> Result, String> { + // Check if payload size exceeds the maximum allowed size + if payload_size > 2_147_483_647 { + return Err("Payload size exceeds the maximum allowed size of 2GB".into()); + } - // Read the value associated with the key - let (value, value_consumed) = read_jsonb_value(&bytes[total_read..])?; - object.insert(key_str, value); - total_read += value_consumed; + let header = if payload_size <= 0x0B { + // Small payloads, 0 additional byte for size + vec![(u8::try_from(payload_size).map_err(|e| e.to_string())?) << 4 | element_type] + } else if payload_size <= 0xFF { + // Medium payloads, 1 additional byte for size + vec![ + 0x0C << 4 | element_type, + u8::try_from(payload_size).map_err(|e| e.to_string())?, + ] + } else if payload_size <= 0xFFFF { + let mut header = Vec::with_capacity(3); + + // Larger payloads, 2 additional bytes for size + header.push(0x0D << 4 | element_type); + header.extend_from_slice( + &(u16::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), + ); + + header + } else { + let mut header = Vec::with_capacity(5); + + // Very large payloads, 4 additional bytes for size (up to 2 GiB) + header.push(0x0E << 4 | element_type); + header.extend_from_slice( + &(u32::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), + ); + + header + }; + + Ok(header) + } + + pub(super) fn write_jsonb_header( + buffer: &mut Vec, + element_type: u8, + payload_size: usize, + ) -> serialize::Result { + // Create the header and append it to the buffer + let header = create_jsonb_header(element_type, payload_size)?; + buffer.extend(header); + Ok(IsNull::No) } - // Ensure the total bytes read match the payload size - if total_read != payload_size { - return Err("Object payload size mismatch".into()); + // Helper function to write a JSON value into a JSONB binary format + pub(super) fn write_jsonb_value( + value: &serde_json::Value, + buffer: &mut Vec, + ) -> serialize::Result { + if value.is_null() { + write_jsonb_null(buffer) + } else if value.is_boolean() { + write_jsonb_bool(value.as_bool().ok_or("Failed to read JSONB value")?, buffer) + } else if value.is_number() { + write_jsonb_number(value, buffer) + } else if value.is_string() { + write_jsonb_string(value.as_str().ok_or("Failed to read JSONB value")?, buffer) + } else if value.is_array() { + write_jsonb_array( + value.as_array().ok_or("Failed to read JSONB value")?, + buffer, + ) + } else if value.is_object() { + write_jsonb_object( + value.as_object().ok_or("Failed to read JSONB value")?, + buffer, + ) + } else { + Err("Unsupported JSONB value type".into()) + } } - Ok(serde_json::Value::Object(object)) -} - -// Helper function to create the correct JsonbHeader based on the payload size -fn create_jsonb_header(element_type: u8, payload_size: usize) -> Result, String> { - // Check if payload size exceeds the maximum allowed size - if payload_size > 2_147_483_647 { - return Err("Payload size exceeds the maximum allowed size of 2GB".into()); - } - - let header = if payload_size <= 0x0B { - // Small payloads, 0 additional byte for size - vec![(u8::try_from(payload_size).map_err(|e| e.to_string())?) << 4 | element_type] - } else if payload_size <= 0xFF { - // Medium payloads, 1 additional byte for size - vec![ - 0x0C << 4 | element_type, - u8::try_from(payload_size).map_err(|e| e.to_string())?, - ] - } else if payload_size <= 0xFFFF { - let mut header = Vec::with_capacity(3); - - // Larger payloads, 2 additional bytes for size - header.push(0x0D << 4 | element_type); - header.extend_from_slice( - &(u16::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), - ); + // Write a JSON null + pub(super) fn write_jsonb_null(buffer: &mut Vec) -> serialize::Result { + write_jsonb_header(buffer, JSONB_NULL, 0x0)?; + Ok(IsNull::No) + } - header - } else { - let mut header = Vec::with_capacity(5); + // Write a JSON boolean + pub(super) fn write_jsonb_bool(b: bool, buffer: &mut Vec) -> serialize::Result { + // Use the constants for true and false + write_jsonb_header(buffer, if b { JSONB_TRUE } else { JSONB_FALSE }, 0x0)?; + Ok(IsNull::No) + } - // Very large payloads, 4 additional bytes for size (up to 2 GiB) - header.push(0x0E << 4 | element_type); - header.extend_from_slice( - &(u32::try_from(payload_size).map_err(|e| e.to_string())?).to_be_bytes(), - ); + // Write a JSON number (integers and floats) + pub(super) fn write_jsonb_number( + n: &serde_json::Value, + buffer: &mut Vec, + ) -> serialize::Result { + if let Some(i) = n.as_i64() { + // Write an integer (INT type) + write_jsonb_int(i, buffer) + } else if let Some(f) = n.as_f64() { + // Write a float (FLOAT type) + write_jsonb_float(f, buffer) + } else { + Err("Invalid JSONB number type".into()) + } + } - header - }; + // Write an integer in JSONB format + pub(super) fn write_jsonb_int(i: i64, buffer: &mut Vec) -> serialize::Result { + let int_str = i.to_string(); - Ok(header) -} + write_jsonb_header(buffer, JSONB_INT, int_str.len())?; -fn write_jsonb_header( - buffer: &mut Vec, - element_type: u8, - payload_size: usize, -) -> serialize::Result { - // Create the header and append it to the buffer - let header = create_jsonb_header(element_type, payload_size)?; - buffer.extend(header); - Ok(IsNull::No) -} + // Write the ASCII text representation of the integer as the payload + buffer.extend_from_slice(int_str.as_bytes()); -// Helper function to write a JSON value into a JSONB binary format -fn write_jsonb_value(value: &serde_json::Value, buffer: &mut Vec) -> serialize::Result { - match value { - serde_json::Value::Null => write_jsonb_null(buffer), - serde_json::Value::Bool(b) => write_jsonb_bool(*b, buffer), - serde_json::Value::Number(n) => write_jsonb_number(n, buffer), - serde_json::Value::String(s) => write_jsonb_string(s, buffer), - serde_json::Value::Array(arr) => write_jsonb_array(arr, buffer), - serde_json::Value::Object(obj) => write_jsonb_object(obj, buffer), + Ok(IsNull::No) } -} -// Write a JSON null -fn write_jsonb_null(buffer: &mut Vec) -> serialize::Result { - write_jsonb_header(buffer, JSONB_NULL, 0x0)?; - Ok(IsNull::No) -} - -// Write a JSON boolean -fn write_jsonb_bool(b: bool, buffer: &mut Vec) -> serialize::Result { - // Use the constants for true and false - write_jsonb_header(buffer, if b { JSONB_TRUE } else { JSONB_FALSE }, 0x0)?; - Ok(IsNull::No) -} + // Write a floating-point number in JSONB format + pub(super) fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { + let float_str = f.to_string(); -// Write a JSON number (integers and floats) -fn write_jsonb_number(n: &serde_json::Number, buffer: &mut Vec) -> serialize::Result { - if let Some(i) = n.as_i64() { - // Write an integer (INT type) - write_jsonb_int(i, buffer) - } else if let Some(f) = n.as_f64() { - // Write a float (FLOAT type) - write_jsonb_float(f, buffer) - } else { - Err("Invalid number type".into()) - } -} + write_jsonb_header(buffer, JSONB_FLOAT, float_str.len())?; -// Write an integer in JSONB format -fn write_jsonb_int(i: i64, buffer: &mut Vec) -> serialize::Result { - let int_str = i.to_string(); + // Write the ASCII text representation of the float as the payload + buffer.extend_from_slice(float_str.as_bytes()); - write_jsonb_header(buffer, JSONB_INT, int_str.len())?; - - // Write the ASCII text representation of the integer as the payload - buffer.extend_from_slice(int_str.as_bytes()); + Ok(IsNull::No) + } - Ok(IsNull::No) -} + pub(super) fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { + if s.chars().any(|c| c.is_control()) { + // If the string contains control characters, treat it as TEXTJ (escaped JSON) + write_jsonb_textj(s, buffer) + } else { + write_jsonb_header(buffer, JSONB_TEXT, s.len())?; + // Write the UTF-8 text of the string as the payload (no delimiters) + buffer.extend_from_slice(s.as_bytes()); + Ok(IsNull::No) + } + } -// Write a floating-point number in JSONB format -fn write_jsonb_float(f: f64, buffer: &mut Vec) -> serialize::Result { - let float_str = f.to_string(); + pub(super) fn write_jsonb_textj(s: &str, buffer: &mut Vec) -> serialize::Result { + // Escaping the string for JSON (e.g., \n, \uXXXX) + let escaped_string = serde_json::to_string(&String::from(s)) + .map_err(|_| "Failed to serialize string for TEXTJ")?; - write_jsonb_header(buffer, JSONB_FLOAT, float_str.len())?; + // Remove the surrounding quotes from serde_json::to_string result + let escaped_string = &escaped_string[1..escaped_string.len() - 1]; - // Write the ASCII text representation of the float as the payload - buffer.extend_from_slice(float_str.as_bytes()); + // Write the header (JSONB_TEXTJ) and the length of the escaped string + write_jsonb_header(buffer, JSONB_TEXTJ, escaped_string.len())?; - Ok(IsNull::No) -} + // Write the escaped string as the payload + buffer.extend_from_slice(escaped_string.as_bytes()); -fn write_jsonb_string(s: &str, buffer: &mut Vec) -> serialize::Result { - if s.chars().any(|c| c.is_control()) { - // If the string contains control characters, treat it as TEXTJ (escaped JSON) - write_jsonb_textj(s, buffer) - } else { - write_jsonb_header(buffer, JSONB_TEXT, s.len())?; - // Write the UTF-8 text of the string as the payload (no delimiters) - buffer.extend_from_slice(s.as_bytes()); Ok(IsNull::No) } -} - -fn write_jsonb_textj(s: &str, buffer: &mut Vec) -> serialize::Result { - // Escaping the string for JSON (e.g., \n, \uXXXX) - let escaped_string = - serde_json::to_string(s).map_err(|_| "Failed to serialize string for TEXTJ")?; - - // Remove the surrounding quotes from serde_json::to_string result - let escaped_string = &escaped_string[1..escaped_string.len() - 1]; - // Write the header (JSONB_TEXTJ) and the length of the escaped string - write_jsonb_header(buffer, JSONB_TEXTJ, escaped_string.len())?; + // Write a JSON array + pub(super) fn write_jsonb_array( + arr: &[serde_json::Value], + buffer: &mut Vec, + ) -> serialize::Result { + let mut tmp_buffer = Vec::new(); - // Write the escaped string as the payload - buffer.extend_from_slice(escaped_string.as_bytes()); + // Recursively write each element of the array + for element in arr { + write_jsonb_value(element, &mut tmp_buffer)?; + } - Ok(IsNull::No) -} + write_jsonb_header(buffer, JSONB_ARRAY, tmp_buffer.len())?; -// Write a JSON array -fn write_jsonb_array(arr: &[serde_json::Value], buffer: &mut Vec) -> serialize::Result { - let mut tmp_buffer = Vec::new(); + buffer.extend_from_slice(&tmp_buffer); - // Recursively write each element of the array - for element in arr { - write_jsonb_value(element, &mut tmp_buffer)?; + Ok(IsNull::No) } - write_jsonb_header(buffer, JSONB_ARRAY, tmp_buffer.len())?; + // Write a JSON object + pub(super) fn write_jsonb_object( + obj: &serde_json::Map, + buffer: &mut Vec, + ) -> serialize::Result { + let mut tmp_buffer = Vec::new(); - buffer.extend_from_slice(&tmp_buffer); + // Recursively write each key-value pair of the object + for (key, value) in obj { + // Write the key (which must be a string) + write_jsonb_string(key, &mut tmp_buffer)?; - Ok(IsNull::No) -} + // Write the value + write_jsonb_value(value, &mut tmp_buffer)?; + } -// Write a JSON object -fn write_jsonb_object( - obj: &serde_json::Map, - buffer: &mut Vec, -) -> serialize::Result { - let mut tmp_buffer = Vec::new(); + write_jsonb_header(buffer, JSONB_OBJECT, tmp_buffer.len())?; - // Recursively write each key-value pair of the object - for (key, value) in obj { - // Write the key (which must be a string) - write_jsonb_string(key, &mut tmp_buffer)?; + buffer.extend_from_slice(&tmp_buffer); - // Write the value - write_jsonb_value(value, &mut tmp_buffer)?; + Ok(IsNull::No) } - - write_jsonb_header(buffer, JSONB_OBJECT, tmp_buffer.len())?; - - buffer.extend_from_slice(&tmp_buffer); - - Ok(IsNull::No) } #[cfg(test)] +#[cfg(all(feature = "sqlite", feature = "serde_json"))] mod tests { + use super::jsonb::*; use super::*; use crate::query_dsl::RunQueryDsl; use crate::test_helpers::connection; From de6b84713d834e706637436716bf8d463351c814 Mon Sep 17 00:00:00 2001 From: Jean-Marc Le Roux Date: Tue, 1 Oct 2024 21:59:30 +0200 Subject: [PATCH 24/24] remove the serde_json feature gate on the Json and Jsonb types --- diesel/src/sql_types/mod.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/diesel/src/sql_types/mod.rs b/diesel/src/sql_types/mod.rs index 31ac664142b7..1aece629c942 100644 --- a/diesel/src/sql_types/mod.rs +++ b/diesel/src/sql_types/mod.rs @@ -395,11 +395,6 @@ pub struct Timestamp; /// [`ToSql`]: /serialize/trait.ToSql.html /// [`FromSql`]: /deserialize/trait.FromSql.html /// [`serde_json::Value`]: /../serde_json/value/enum.Value.html -#[cfg(any( - feature = "postgres_backend", - feature = "mysql_backend", - all(feature = "sqlite", feature = "serde_json") -))] #[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] #[diesel(postgres_type(oid = 114, array_oid = 199))] #[diesel(mysql_type(name = "String"))] @@ -479,21 +474,24 @@ pub struct Json; /// } /// } /// +/// # #[cfg(all( +/// # feature = "serde_json", +/// # any( +/// # feature = "postgres_backend", +/// # all(feature = "sqlite", feature = "returning_clauses_for_sqlite_3_35"), +/// # ) +/// # ))] /// # fn main() -> Result<(), Box> { /// # use diesel::insert_into; /// # use self::contacts::dsl::*; /// # let connection = &mut connection_no_data(); -/// # #[cfg(all(feature = "postgres_backend", feature = "serde_json"))] +/// # #[cfg(feature = "postgres_backend")] /// # diesel::sql_query("CREATE TABLE contacts ( /// # id SERIAL PRIMARY KEY, /// # name VARCHAR NOT NULL, /// # address JSONB NOT NULL /// # )").execute(connection)?; -/// # #[cfg(all( -/// # feature = "sqlite", -/// # feature = "serde_json", -/// # feature = "returning_clauses_for_sqlite_3_35" -/// # ))] +/// # #[cfg(feature = "sqlite")] /// # diesel::sql_query("CREATE TABLE contacts ( /// # id INT PRIMARY KEY, /// # name TEXT NOT NULL, @@ -512,13 +510,15 @@ pub struct Json; /// assert_eq!(santas_address, inserted_address); /// # Ok(()) /// # } -/// # #[cfg(not(feature = "serde_json"))] +/// # #[cfg(not(all( +/// # feature = "serde_json", +/// # any( +/// # feature = "postgres_backend", +/// # all(feature = "sqlite", feature = "returning_clauses_for_sqlite_3_35"), +/// # ) +/// # )))] /// # fn main() {} /// ``` -#[cfg(any( - feature = "postgres_backend", - all(feature = "sqlite", feature = "serde_json") -))] #[derive(Debug, Clone, Copy, Default, QueryId, SqlType)] #[diesel(postgres_type(oid = 3802, array_oid = 3807))] #[diesel(sqlite_type(name = "Binary"))]