Skip to content

Commit

Permalink
feat: Support casting Utf8 to Interval
Browse files Browse the repository at this point in the history
  • Loading branch information
@MazterQyou
MazterQyou committed Dec 5, 2023
1 parent 5833202 commit e731f51
Show file tree
Hide file tree
Showing 14 changed files with 55 additions and 46 deletions.
6 changes: 3 additions & 3 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions datafusion-cli/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion datafusion-cli/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ repository = "https://github.com/apache/arrow-datafusion"
rust-version = "1.59"

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e" }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e" }
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "7.0.0" }
dirs = "4.0.0"
Expand Down
2 changes: 1 addition & 1 deletion datafusion-examples/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]

[dev-dependencies]
arrow-flight = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e" }
arrow-flight = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e" }
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
futures = "0.3"
Expand Down
4 changes: 2 additions & 2 deletions datafusion/common/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ jit = ["cranelift-module"]
pyarrow = ["pyo3"]

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
cranelift-module = { version = "0.82.0", optional = true }
ordered-float = "2.10"
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["arrow"], optional = true }
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["arrow"], optional = true }
pyo3 = { version = "0.16", optional = true }
sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "037562f7975fcc92b725efbfbaaf050272668593" }
8 changes: 8 additions & 0 deletions datafusion/common/src/scalar.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1339,6 +1339,14 @@ impl ScalarValue {
DataType::Interval(IntervalUnit::YearMonth) => {
typed_cast!(array, index, IntervalYearMonthArray, IntervalYearMonth)
}
DataType::Interval(IntervalUnit::MonthDayNano) => {
typed_cast!(
array,
index,
IntervalMonthDayNanoArray,
IntervalMonthDayNano
)
}
DataType::Timestamp(TimeUnit::Millisecond, tz_opt) => {
typed_cast_tz!(
array,
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions"]

[dependencies]
ahash = { version = "0.7", default-features = false }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["prettyprint"] }
async-trait = "0.1.41"
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
chrono = { version = "0.4", default-features = false }
Expand All @@ -73,7 +73,7 @@ num-traits = { version = "0.2", optional = true }
num_cpus = "1.13.0"
ordered-float = "2.10"
parking_lot = "0.12"
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["arrow"] }
parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["arrow"] }
paste = "^1.0"
pin-project-lite= "^0.2.7"
pyo3 = { version = "0.16", optional = true }
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/fuzz-utils/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
1 change: 1 addition & 0 deletions datafusion/core/src/sql/planner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2845,6 +2845,7 @@ pub fn convert_data_type(sql_type: &SQLDataType) -> Result<DataType> {
SQLDataType::Timestamp => Ok(DataType::Timestamp(TimeUnit::Nanosecond, None)),
SQLDataType::Date => Ok(DataType::Date32),
SQLDataType::Decimal(precision, scale) => make_decimal_type(*precision, *scale),
SQLDataType::Interval => Ok(DataType::Interval(IntervalUnit::MonthDayNano)),
other => Err(DataFusionError::NotImplemented(format!(
"Unsupported SQL type {:?}",
other
Expand Down
60 changes: 30 additions & 30 deletions datafusion/core/tests/sql/expr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,63 +554,63 @@ async fn test_interval_expressions() -> Result<()> {
// day nano intervals
test_expression!(
"interval '1'",
"0 years 0 mons 0 days 0 hours 0 mins 1.00 secs"
"0 years 0 mons 0 days 0 hours 0 mins 1.000 secs"
);
test_expression!(
"interval '1 second'",
"0 years 0 mons 0 days 0 hours 0 mins 1.00 secs"
"0 years 0 mons 0 days 0 hours 0 mins 1.000 secs"
);
test_expression!(
"interval '500 milliseconds'",
"0 years 0 mons 0 days 0 hours 0 mins 0.500 secs"
);
test_expression!(
"interval '5 second'",
"0 years 0 mons 0 days 0 hours 0 mins 5.00 secs"
"0 years 0 mons 0 days 0 hours 0 mins 5.000 secs"
);
test_expression!(
"interval '0.5 minute'",
"0 years 0 mons 0 days 0 hours 0 mins 30.00 secs"
"0 years 0 mons 0 days 0 hours 0 mins 30.000 secs"
);
test_expression!(
"interval '.5 minute'",
"0 years 0 mons 0 days 0 hours 0 mins 30.00 secs"
"0 years 0 mons 0 days 0 hours 0 mins 30.000 secs"
);
test_expression!(
"interval '5 minute'",
"0 years 0 mons 0 days 0 hours 5 mins 0.00 secs"
"0 years 0 mons 0 days 0 hours 5 mins 0.000 secs"
);
test_expression!(
"interval '5 minute 1 second'",
"0 years 0 mons 0 days 0 hours 5 mins 1.00 secs"
"0 years 0 mons 0 days 0 hours 5 mins 1.000 secs"
);
test_expression!(
"interval '1 hour'",
"0 years 0 mons 0 days 1 hours 0 mins 0.00 secs"
"0 years 0 mons 0 days 1 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '5 hour'",
"0 years 0 mons 0 days 5 hours 0 mins 0.00 secs"
"0 years 0 mons 0 days 5 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '1 day'",
"0 years 0 mons 1 days 0 hours 0 mins 0.00 secs"
"0 years 0 mons 1 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '1 day 1'",
"0 years 0 mons 1 days 0 hours 0 mins 1.00 secs"
"0 years 0 mons 1 days 0 hours 0 mins 1.000 secs"
);
test_expression!(
"interval '0.5'",
"0 years 0 mons 0 days 0 hours 0 mins 0.500 secs"
);
test_expression!(
"interval '0.5 day 1'",
"0 years 0 mons 0 days 12 hours 0 mins 1.00 secs"
"0 years 0 mons 0 days 12 hours 0 mins 1.000 secs"
);
test_expression!(
"interval '0.49 day'",
"0 years 0 mons 0 days 11 hours 45 mins 36.00 secs"
"0 years 0 mons 0 days 11 hours 45 mins 36.000 secs"
);
test_expression!(
"interval '0.499 day'",
Expand All @@ -626,16 +626,16 @@ async fn test_interval_expressions() -> Result<()> {
);
test_expression!(
"interval '0.49999999999 day'",
"0 years 0 mons 0 days 12 hours 0 mins 0.00 secs"
"0 years 0 mons 0 days 12 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '5 day'",
"0 years 0 mons 5 days 0 hours 0 mins 0.00 secs"
"0 years 0 mons 5 days 0 hours 0 mins 0.000 secs"
);
// Hour is ignored, this matches PostgreSQL
test_expression!(
"interval '5 day' hour",
"0 years 0 mons 5 days 0 hours 0 mins 0.00 secs"
"0 years 0 mons 5 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '5 day 4 hours 3 minutes 2 seconds 100 milliseconds'",
Expand All @@ -644,60 +644,60 @@ async fn test_interval_expressions() -> Result<()> {
// month intervals
test_expression!(
"interval '0.5 month'",
"0 years 0 mons 15 days 0 hours 0 mins 0.00 secs"
"0 years 0 mons 15 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '0.5' month",
"0 years 0 mons 15 days 0 hours 0 mins 0.00 secs"
"0 years 0 mons 15 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '1 month'",
"0 years 1 mons 0 days 0 hours 0 mins 0.00 secs"
"0 years 1 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '1' MONTH",
"0 years 1 mons 0 days 0 hours 0 mins 0.00 secs"
"0 years 1 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '5 month'",
"0 years 5 mons 0 days 0 hours 0 mins 0.00 secs"
"0 years 5 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '13 month'",
"1 years 1 mons 0 days 0 hours 0 mins 0.00 secs"
"1 years 1 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '0.5 year'",
"0 years 6 mons 0 days 0 hours 0 mins 0.00 secs"
"0 years 6 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '1 year'",
"1 years 0 mons 0 days 0 hours 0 mins 0.00 secs"
"1 years 0 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '2 year'",
"2 years 0 mons 0 days 0 hours 0 mins 0.00 secs"
"2 years 0 mons 0 days 0 hours 0 mins 0.000 secs"
);
test_expression!(
"interval '2' year",
"2 years 0 mons 0 days 0 hours 0 mins 0.00 secs"
"2 years 0 mons 0 days 0 hours 0 mins 0.000 secs"
);
// complex
test_expression!(
"interval '1 year 1 day'",
"0 years 12 mons 1 days 0 hours 0 mins 0.00 secs"
"0 years 12 mons 1 days 0 hours 0 mins 0.000000000 secs"
);
test_expression!(
"interval '1 year 1 day 1 hour'",
"0 years 12 mons 1 days 1 hours 0 mins 0.00 secs"
"0 years 12 mons 1 days 1 hours 0 mins 0.000000000 secs"
);
test_expression!(
"interval '1 year 1 day 1 hour 1 minute'",
"0 years 12 mons 1 days 1 hours 1 mins 0.00 secs"
"0 years 12 mons 1 days 1 hours 1 mins 0.000000000 secs"
);
test_expression!(
"interval '1 year 1 day 1 hour 1 minute 1 second'",
"0 years 12 mons 1 days 1 hours 1 mins 1.00 secs"
"0 years 12 mons 1 days 1 hours 1 mins 1.000000000 secs"
);

Ok(())
Expand Down
2 changes: 1 addition & 1 deletion datafusion/cube_ext/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ name = "cube_ext"
path = "src/lib.rs"

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["prettyprint"] }
chrono = { version = "0.4.16", package = "chrono", default-features = false, features = ["clock"] }
datafusion-common = { path = "../common", version = "7.0.0" }
datafusion-expr = { path = "../expr", version = "7.0.0" }
2 changes: 1 addition & 1 deletion datafusion/expr/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,6 @@ path = "src/lib.rs"

[dependencies]
ahash = { version = "0.7", default-features = false }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "7.0.0" }
sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "037562f7975fcc92b725efbfbaaf050272668593" }
2 changes: 1 addition & 1 deletion datafusion/jit/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []

[dependencies]
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e" }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e" }
cranelift = "0.82.0"
cranelift-jit = "0.82.0"
cranelift-module = "0.82.0"
Expand Down
2 changes: 1 addition & 1 deletion datafusion/physical-expr/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]

[dependencies]
ahash = { version = "0.7", default-features = false }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "096ef28dde6b1ae43ce89ba2c3a9d98295f2972e", features = ["prettyprint"] }
arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "b13189d2dcfa83886f58e8eaffabd50c7ac0259e", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4", default-features = false }
Expand Down

0 comments on commit e731f51

Please sign in to comment.