From 1ecef46e718ca0656629f298be772ea2f8b0d6b7 Mon Sep 17 00:00:00 2001 From: Semyon Uchvatov Date: Fri, 13 Dec 2024 12:19:33 +0300 Subject: [PATCH] Move json_type_def to separate module --- Cargo.lock | 17 +++++-- src/stdlib/json_utils/json_type_def.rs | 24 +++++++++ src/stdlib/json_utils/mod.rs | 1 + src/stdlib/mod.rs | 1 + src/stdlib/parse_cbor.rs | 65 +++++++++++++----------- src/stdlib/parse_json.rs | 68 +++++++------------------- src/stdlib/parse_proto.rs | 28 ++--------- 7 files changed, 96 insertions(+), 108 deletions(-) create mode 100644 src/stdlib/json_utils/json_type_def.rs create mode 100644 src/stdlib/json_utils/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 887664913a..c802b136b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,15 +124,16 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.7" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] @@ -1284,9 +1285,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -1529,7 +1530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.2", ] [[package]] @@ -1584,6 +1585,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" diff --git a/src/stdlib/json_utils/json_type_def.rs b/src/stdlib/json_utils/json_type_def.rs new file mode 100644 index 0000000000..77ffa5cd4e --- /dev/null +++ b/src/stdlib/json_utils/json_type_def.rs @@ -0,0 +1,24 @@ +use crate::prelude::{Collection, TypeDef}; +use crate::value::Kind; + +pub(crate) fn json_inner_kind() -> Kind { + Kind::null() + | Kind::bytes() + | Kind::integer() + | Kind::float() + | Kind::boolean() + | Kind::array(Collection::any()) + | Kind::object(Collection::any()) +} + +pub(crate) fn json_type_def() -> TypeDef { + TypeDef::bytes() + .fallible() + .or_boolean() + .or_integer() + .or_float() + .add_null() + .or_null() + .or_array(Collection::from_unknown(json_inner_kind())) + .or_object(Collection::from_unknown(json_inner_kind())) +} diff --git a/src/stdlib/json_utils/mod.rs b/src/stdlib/json_utils/mod.rs new file mode 100644 index 0000000000..472564627e --- /dev/null +++ b/src/stdlib/json_utils/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_type_def; diff --git a/src/stdlib/mod.rs b/src/stdlib/mod.rs index c4dc5b6ae0..9a55968520 100644 --- a/src/stdlib/mod.rs +++ b/src/stdlib/mod.rs @@ -31,6 +31,7 @@ pub use wasm_unsupported_function::WasmUnsupportedFunction; use crate::compiler::Function; +mod json_utils; mod string_utils; mod util; mod wasm_unsupported_function; diff --git a/src/stdlib/parse_cbor.rs b/src/stdlib/parse_cbor.rs index 4f958c9ee3..c384d776cb 100644 --- a/src/stdlib/parse_cbor.rs +++ b/src/stdlib/parse_cbor.rs @@ -1,4 +1,5 @@ use crate::compiler::prelude::*; +use crate::stdlib::json_utils::json_type_def::json_type_def; use ciborium::de::from_reader; use zstd::zstd_safe::WriteBuf; @@ -27,11 +28,38 @@ impl Function for ParseCbor { } fn examples(&self) -> &'static [Example] { - &[Example { - title: "object", - source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#, - result: Ok(r#"{ "field": "value" }"#), - }] + &[ + Example { + title: "object", + source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#, + result: Ok(r#"{ "field": "value" }"#), + }, + Example { + title: "array", + source: r#"parse_cbor!(decode_base64!("gvUA"))"#, + result: Ok("[true, 0]"), + }, + Example { + title: "string", + source: r#"parse_cbor!(decode_base64!("ZWhlbGxv"))"#, + result: Ok("hello"), + }, + Example { + title: "integer", + source: r#"parse_cbor!(decode_base64!("GCo="))"#, + result: Ok("42"), + }, + Example { + title: "float", + source: r#"parse_cbor!(decode_base64!("+0BFEKPXCj1x"))"#, + result: Ok("42.13"), + }, + Example { + title: "boolean", + source: r#"parse_cbor!(decode_base64!("9A=="))"#, + result: Ok("false"), + }, + ] } fn compile( @@ -65,31 +93,10 @@ impl FunctionExpression for ParseCborFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } -fn inner_kind() -> Kind { - Kind::null() - | Kind::bytes() - | Kind::integer() - | Kind::float() - | Kind::boolean() - | Kind::array(Collection::any()) - | Kind::object(Collection::any()) -} - -fn type_def() -> TypeDef { - TypeDef::bytes() - .fallible() - .or_boolean() - .or_integer() - .or_float() - .add_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())) -} - #[cfg(test)] mod tests { use super::*; @@ -113,13 +120,13 @@ mod tests { parses { args: func_args![ value: value!(read_cbor_file("simple.cbor").as_bytes()) ], want: Ok(value!({ field: "value" })), - tdef: type_def(), + tdef: json_type_def(), } complex_cbor { args: func_args![ value: value!(read_cbor_file("complex.cbor").as_bytes()) ], want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })), - tdef: type_def(), + tdef: json_type_def(), } ]; } diff --git a/src/stdlib/parse_json.rs b/src/stdlib/parse_json.rs index a26316c48f..06ef79b59f 100644 --- a/src/stdlib/parse_json.rs +++ b/src/stdlib/parse_json.rs @@ -6,6 +6,7 @@ use serde_json::{ }; use crate::compiler::prelude::*; +use crate::stdlib::json_utils::json_type_def::json_type_def; fn parse_json(value: Value, lossy: Option) -> Resolved { let lossy = lossy.map(Value::try_boolean).transpose()?.unwrap_or(true); @@ -226,7 +227,7 @@ impl FunctionExpression for ParseJsonFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } @@ -250,31 +251,10 @@ impl FunctionExpression for ParseJsonMaxDepthFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } -fn inner_kind() -> Kind { - Kind::null() - | Kind::bytes() - | Kind::integer() - | Kind::float() - | Kind::boolean() - | Kind::array(Collection::any()) - | Kind::object(Collection::any()) -} - -fn type_def() -> TypeDef { - TypeDef::bytes() - .fallible() - .or_boolean() - .or_integer() - .or_float() - .add_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())) -} - #[cfg(test)] mod tests { use super::*; @@ -286,74 +266,62 @@ mod tests { parses { args: func_args![ value: r#"{"field": "value"}"# ], want: Ok(value!({ field: "value" })), - tdef: type_def(), + tdef: json_type_def(), } complex_json { args: func_args![ value: r#"{"object": {"string":"value","number":42,"array":["hello","world"],"boolean":false}}"# ], want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })), - tdef: type_def(), + tdef: json_type_def(), } invalid_json_errors { args: func_args![ value: r#"{"field": "value"# ], want: Err("unable to parse json: EOF while parsing a string at line 1 column 16"), - tdef: TypeDef::bytes().fallible() - .or_boolean() - .or_integer() - .or_float() - .or_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())), + tdef: json_type_def(), } max_depth { args: func_args![ value: r#"{"top_layer": {"layer_one": "finish", "layer_two": 2}}"#, max_depth: 1], want: Ok(value!({ top_layer: r#"{"layer_one": "finish", "layer_two": 2}"# })), - tdef: type_def(), + tdef: json_type_def(), } max_depth_array { args: func_args![ value: r#"[{"top_layer": {"next_layer": ["finish"]}}]"#, max_depth: 2], want: Ok(value!([{ top_layer: r#"{"next_layer": ["finish"]}"# }])), - tdef: type_def(), + tdef: json_type_def(), } max_depth_exceeds_layers { args: func_args![ value: r#"{"top_layer": {"layer_one": "finish", "layer_two": 2}}"#, max_depth: 10], want: Ok(value!({ top_layer: {layer_one: "finish", layer_two: 2} })), - tdef: type_def(), + tdef: json_type_def(), } invalid_json_with_max_depth { args: func_args![ value: r#"{"field": "value"#, max_depth: 3 ], want: Err("unable to read json: EOF while parsing a string at line 1 column 16"), - tdef: TypeDef::bytes().fallible() - .or_boolean() - .or_integer() - .or_float() - .or_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())), + tdef: json_type_def(), } invalid_input_max_depth { args: func_args![ value: r#"{"top_layer": "finish"}"#, max_depth: 129], want: Err("max_depth value should be greater than 0 and less than 128, got 129"), - tdef: type_def(), + tdef: json_type_def(), } // // TODO: provide a function version of the `test_function!` macro. max_int { args: func_args![ value: format!("{{\"num\": {}}}", i64::MAX - 1)], want: Ok(value!({"num": 9_223_372_036_854_775_806_i64})), - tdef: type_def(), + tdef: json_type_def(), } lossy_float_conversion { args: func_args![ value: r#"{"num": 9223372036854775808}"#], want: Ok(value!({"num": 9.223_372_036_854_776e18})), - tdef: type_def(), + tdef: json_type_def(), } // Checks that the parsing uses the default lossy argument value @@ -362,7 +330,7 @@ mod tests { // 0xf5 is out of the range of valid UTF-8 bytes args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22])], want: Ok(value!(std::char::REPLACEMENT_CHARACTER.to_string())), - tdef: type_def(), + tdef: json_type_def(), } parse_invalid_utf8_lossy_arg_true { @@ -370,13 +338,13 @@ mod tests { args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22]), lossy: true], // U+FFFD is the replacement character for invalid UTF-8 want: Ok(value!(std::char::REPLACEMENT_CHARACTER.to_string())), - tdef: type_def(), + tdef: json_type_def(), } invalid_utf8_json_lossy_arg_false { args: func_args![ value: Bytes::from_static(&[0x22,0xf5,0x22]), lossy: false], want: Err("unable to parse json: invalid unicode code point at line 1 column 3"), - tdef: type_def(), + tdef: json_type_def(), } ]; @@ -387,7 +355,7 @@ mod tests { no_roundtrip_float_conversion { args: func_args![ value: r#"{"num": 1626175065.5934923}"#], want: Ok(value!({"num": 1_626_175_065.593_492_5})), - tdef: type_def(), + tdef: json_type_def(), } ]; @@ -398,7 +366,7 @@ mod tests { roundtrip_float_conversion { args: func_args![ value: r#"{"num": 1626175065.5934923}"#], want: Ok(value!({"num": 1_626_175_065.593_492_3})), - tdef: type_def(), + tdef: json_type_def(), } ]; } diff --git a/src/stdlib/parse_proto.rs b/src/stdlib/parse_proto.rs index 0c55f29307..aacbe199e1 100644 --- a/src/stdlib/parse_proto.rs +++ b/src/stdlib/parse_proto.rs @@ -1,6 +1,7 @@ use crate::compiler::prelude::*; use crate::protobuf::get_message_descriptor; use crate::protobuf::parse_proto; +use crate::stdlib::json_utils::json_type_def::json_type_def; use once_cell::sync::Lazy; use prost_reflect::MessageDescriptor; use std::env; @@ -112,31 +113,10 @@ impl FunctionExpression for ParseProtoFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - type_def() + json_type_def() } } -fn inner_kind() -> Kind { - Kind::null() - | Kind::bytes() - | Kind::integer() - | Kind::float() - | Kind::boolean() - | Kind::array(Collection::any()) - | Kind::object(Collection::any()) -} - -fn type_def() -> TypeDef { - TypeDef::bytes() - .fallible() - .or_boolean() - .or_integer() - .or_float() - .add_null() - .or_array(Collection::from_unknown(inner_kind())) - .or_object(Collection::from_unknown(inner_kind())) -} - #[cfg(test)] mod tests { use super::*; @@ -159,7 +139,7 @@ mod tests { desc_file: test_data_dir().join("test_protobuf.desc").to_str().unwrap().to_owned(), message_type: "test_protobuf.Person"], want: Ok(value!({ name: "someone", phones: [{number: "123456"}] })), - tdef: type_def(), + tdef: json_type_def(), } parses_proto3 { @@ -167,7 +147,7 @@ mod tests { desc_file: test_data_dir().join("test_protobuf3.desc").to_str().unwrap().to_owned(), message_type: "test_protobuf3.Person"], want: Ok(value!({ data: {data_phone: "HOME"}, name: "someone", phones: [{number: "1234", type: "MOBILE"}] })), - tdef: type_def(), + tdef: json_type_def(), } ]; }