From 6dcdafd6a2a5e3af27349d73db8d721f1554953f Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Thu, 7 Mar 2024 10:31:21 +1300 Subject: [PATCH] Update latest chrono --- arrow-array/src/array/primitive_array.rs | 5 ++- arrow-array/src/temporal_conversions.rs | 46 ++++++++++++---------- arrow-array/src/types.rs | 49 +++++++++++++----------- arrow-cast/src/parse.rs | 37 ++++++++++-------- arrow-json/src/writer.rs | 3 ++ arrow/benches/cast_kernels.rs | 5 +-- object_store/src/lib.rs | 10 +++-- parquet_derive_test/src/lib.rs | 6 +-- 8 files changed, 90 insertions(+), 71 deletions(-) diff --git a/arrow-array/src/array/primitive_array.rs b/arrow-array/src/array/primitive_array.rs index a800aa6bf924..03a48609fbab 100644 --- a/arrow-array/src/array/primitive_array.rs +++ b/arrow-array/src/array/primitive_array.rs @@ -1557,7 +1557,10 @@ mod tests { // roundtrip to and from datetime assert_eq!( 1550902545147, - arr.value_as_datetime(i).unwrap().timestamp_millis() + arr.value_as_datetime(i) + .unwrap() + .and_utc() + .timestamp_millis() ); } else { assert!(arr.is_null(i)); diff --git a/arrow-array/src/temporal_conversions.rs b/arrow-array/src/temporal_conversions.rs index e0edcc9bc182..8d238b3a196c 100644 --- a/arrow-array/src/temporal_conversions.rs +++ b/arrow-array/src/temporal_conversions.rs @@ -43,7 +43,7 @@ pub const EPOCH_DAYS_FROM_CE: i32 = 719_163; /// converts a `i32` representing a `date32` to [`NaiveDateTime`] #[inline] pub fn date32_to_datetime(v: i32) -> Option { - NaiveDateTime::from_timestamp_opt(v as i64 * SECONDS_IN_DAY, 0) + Some(DateTime::from_timestamp(v as i64 * SECONDS_IN_DAY, 0)?.naive_utc()) } /// converts a `i64` representing a `date64` to [`NaiveDateTime`] @@ -51,12 +51,13 @@ pub fn date32_to_datetime(v: i32) -> Option { pub fn date64_to_datetime(v: i64) -> Option { let (sec, milli_sec) = split_second(v, MILLISECONDS); - NaiveDateTime::from_timestamp_opt( + let datetime = DateTime::from_timestamp( // extract seconds from milliseconds sec, // discard extracted seconds and convert milliseconds to nanoseconds milli_sec * MICROSECONDS as u32, - ) + )?; + Some(datetime.naive_utc()) } /// converts a `i32` representing a `time32(s)` to [`NaiveDateTime`] @@ -130,7 +131,7 @@ pub fn time_to_time64ns(v: NaiveTime) -> i64 { /// converts a `i64` representing a `timestamp(s)` to [`NaiveDateTime`] #[inline] pub fn timestamp_s_to_datetime(v: i64) -> Option { - NaiveDateTime::from_timestamp_opt(v, 0) + Some(DateTime::from_timestamp(v, 0)?.naive_utc()) } /// converts a `i64` representing a `timestamp(ms)` to [`NaiveDateTime`] @@ -138,12 +139,13 @@ pub fn timestamp_s_to_datetime(v: i64) -> Option { pub fn timestamp_ms_to_datetime(v: i64) -> Option { let (sec, milli_sec) = split_second(v, MILLISECONDS); - NaiveDateTime::from_timestamp_opt( + let datetime = DateTime::from_timestamp( // extract seconds from milliseconds sec, // discard extracted seconds and convert milliseconds to nanoseconds milli_sec * MICROSECONDS as u32, - ) + )?; + Some(datetime.naive_utc()) } /// converts a `i64` representing a `timestamp(us)` to [`NaiveDateTime`] @@ -151,12 +153,13 @@ pub fn timestamp_ms_to_datetime(v: i64) -> Option { pub fn timestamp_us_to_datetime(v: i64) -> Option { let (sec, micro_sec) = split_second(v, MICROSECONDS); - NaiveDateTime::from_timestamp_opt( + let datetime = DateTime::from_timestamp( // extract seconds from microseconds sec, // discard extracted seconds and convert microseconds to nanoseconds micro_sec * MILLISECONDS as u32, - ) + )?; + Some(datetime.naive_utc()) } /// converts a `i64` representing a `timestamp(ns)` to [`NaiveDateTime`] @@ -164,11 +167,12 @@ pub fn timestamp_us_to_datetime(v: i64) -> Option { pub fn timestamp_ns_to_datetime(v: i64) -> Option { let (sec, nano_sec) = split_second(v, NANOSECONDS); - NaiveDateTime::from_timestamp_opt( + let datetime = DateTime::from_timestamp( // extract seconds from nanoseconds sec, // discard extracted seconds nano_sec, - ) + )?; + Some(datetime.naive_utc()) } #[inline] @@ -179,13 +183,13 @@ pub(crate) fn split_second(v: i64, base: i64) -> (i64, u32) { /// converts a `i64` representing a `duration(s)` to [`Duration`] #[inline] pub fn duration_s_to_duration(v: i64) -> Duration { - Duration::seconds(v) + Duration::try_seconds(v).unwrap() } /// converts a `i64` representing a `duration(ms)` to [`Duration`] #[inline] pub fn duration_ms_to_duration(v: i64) -> Duration { - Duration::milliseconds(v) + Duration::try_milliseconds(v).unwrap() } /// converts a `i64` representing a `duration(us)` to [`Duration`] @@ -272,18 +276,18 @@ mod tests { date64_to_datetime, split_second, timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_us_to_datetime, NANOSECONDS, }; - use chrono::NaiveDateTime; + use chrono::DateTime; #[test] fn negative_input_timestamp_ns_to_datetime() { assert_eq!( timestamp_ns_to_datetime(-1), - NaiveDateTime::from_timestamp_opt(-1, 999_999_999) + DateTime::from_timestamp(-1, 999_999_999).map(|x| x.naive_utc()) ); assert_eq!( timestamp_ns_to_datetime(-1_000_000_001), - NaiveDateTime::from_timestamp_opt(-2, 999_999_999) + DateTime::from_timestamp(-2, 999_999_999).map(|x| x.naive_utc()) ); } @@ -291,12 +295,12 @@ mod tests { fn negative_input_timestamp_us_to_datetime() { assert_eq!( timestamp_us_to_datetime(-1), - NaiveDateTime::from_timestamp_opt(-1, 999_999_000) + DateTime::from_timestamp(-1, 999_999_000).map(|x| x.naive_utc()) ); assert_eq!( timestamp_us_to_datetime(-1_000_001), - NaiveDateTime::from_timestamp_opt(-2, 999_999_000) + DateTime::from_timestamp(-2, 999_999_000).map(|x| x.naive_utc()) ); } @@ -304,12 +308,12 @@ mod tests { fn negative_input_timestamp_ms_to_datetime() { assert_eq!( timestamp_ms_to_datetime(-1), - NaiveDateTime::from_timestamp_opt(-1, 999_000_000) + DateTime::from_timestamp(-1, 999_000_000).map(|x| x.naive_utc()) ); assert_eq!( timestamp_ms_to_datetime(-1_001), - NaiveDateTime::from_timestamp_opt(-2, 999_000_000) + DateTime::from_timestamp(-2, 999_000_000).map(|x| x.naive_utc()) ); } @@ -317,12 +321,12 @@ mod tests { fn negative_input_date64_to_datetime() { assert_eq!( date64_to_datetime(-1), - NaiveDateTime::from_timestamp_opt(-1, 999_000_000) + DateTime::from_timestamp(-1, 999_000_000).map(|x| x.naive_utc()) ); assert_eq!( date64_to_datetime(-1_001), - NaiveDateTime::from_timestamp_opt(-2, 999_000_000) + DateTime::from_timestamp(-2, 999_000_000).map(|x| x.naive_utc()) ); } diff --git a/arrow-array/src/types.rs b/arrow-array/src/types.rs index 6e177838c4f5..83a229c1da0d 100644 --- a/arrow-array/src/types.rs +++ b/arrow-array/src/types.rs @@ -390,31 +390,34 @@ impl ArrowTimestampType for TimestampSecondType { const UNIT: TimeUnit = TimeUnit::Second; fn make_value(naive: NaiveDateTime) -> Option { - Some(naive.timestamp()) + Some(naive.and_utc().timestamp()) } } impl ArrowTimestampType for TimestampMillisecondType { const UNIT: TimeUnit = TimeUnit::Millisecond; fn make_value(naive: NaiveDateTime) -> Option { - let millis = naive.timestamp().checked_mul(1_000)?; - millis.checked_add(naive.timestamp_subsec_millis() as i64) + let utc = naive.and_utc(); + let millis = utc.timestamp().checked_mul(1_000)?; + millis.checked_add(utc.timestamp_subsec_millis() as i64) } } impl ArrowTimestampType for TimestampMicrosecondType { const UNIT: TimeUnit = TimeUnit::Microsecond; fn make_value(naive: NaiveDateTime) -> Option { - let micros = naive.timestamp().checked_mul(1_000_000)?; - micros.checked_add(naive.timestamp_subsec_micros() as i64) + let utc = naive.and_utc(); + let micros = utc.timestamp().checked_mul(1_000_000)?; + micros.checked_add(utc.timestamp_subsec_micros() as i64) } } impl ArrowTimestampType for TimestampNanosecondType { const UNIT: TimeUnit = TimeUnit::Nanosecond; fn make_value(naive: NaiveDateTime) -> Option { - let nanos = naive.timestamp().checked_mul(1_000_000_000)?; - nanos.checked_add(naive.timestamp_subsec_nanos() as i64) + let utc = naive.and_utc(); + let nanos = utc.timestamp().checked_mul(1_000_000_000)?; + nanos.checked_add(utc.timestamp_subsec_nanos() as i64) } } @@ -438,7 +441,7 @@ fn add_day_time( let (days, ms) = IntervalDayTimeType::to_parts(delta); let res = as_datetime_with_timezone::(timestamp, tz)?; let res = add_days_datetime(res, days)?; - let res = res.checked_add_signed(Duration::milliseconds(ms as i64))?; + let res = res.checked_add_signed(Duration::try_milliseconds(ms as i64)?)?; let res = res.naive_utc(); T::make_value(res) } @@ -477,7 +480,7 @@ fn subtract_day_time( let (days, ms) = IntervalDayTimeType::to_parts(delta); let res = as_datetime_with_timezone::(timestamp, tz)?; let res = sub_days_datetime(res, days)?; - let res = res.checked_sub_signed(Duration::milliseconds(ms as i64))?; + let res = res.checked_sub_signed(Duration::try_milliseconds(ms as i64)?)?; let res = res.naive_utc(); T::make_value(res) } @@ -1001,7 +1004,7 @@ impl Date32Type { /// * `i` - The Date32Type to convert pub fn to_naive_date(i: ::Native) -> NaiveDate { let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); - epoch.add(Duration::days(i as i64)) + epoch.add(Duration::try_days(i as i64).unwrap()) } /// Converts a chrono::NaiveDate into an arrow Date32Type @@ -1042,8 +1045,8 @@ impl Date32Type { ) -> ::Native { let (days, ms) = IntervalDayTimeType::to_parts(delta); let res = Date32Type::to_naive_date(date); - let res = res.add(Duration::days(days as i64)); - let res = res.add(Duration::milliseconds(ms as i64)); + let res = res.add(Duration::try_days(days as i64).unwrap()); + let res = res.add(Duration::try_milliseconds(ms as i64).unwrap()); Date32Type::from_naive_date(res) } @@ -1060,7 +1063,7 @@ impl Date32Type { let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(delta); let res = Date32Type::to_naive_date(date); let res = shift_months(res, months); - let res = res.add(Duration::days(days as i64)); + let res = res.add(Duration::try_days(days as i64).unwrap()); let res = res.add(Duration::nanoseconds(nanos)); Date32Type::from_naive_date(res) } @@ -1093,8 +1096,8 @@ impl Date32Type { ) -> ::Native { let (days, ms) = IntervalDayTimeType::to_parts(delta); let res = Date32Type::to_naive_date(date); - let res = res.sub(Duration::days(days as i64)); - let res = res.sub(Duration::milliseconds(ms as i64)); + let res = res.sub(Duration::try_days(days as i64).unwrap()); + let res = res.sub(Duration::try_milliseconds(ms as i64).unwrap()); Date32Type::from_naive_date(res) } @@ -1111,7 +1114,7 @@ impl Date32Type { let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(delta); let res = Date32Type::to_naive_date(date); let res = shift_months(res, -months); - let res = res.sub(Duration::days(days as i64)); + let res = res.sub(Duration::try_days(days as i64).unwrap()); let res = res.sub(Duration::nanoseconds(nanos)); Date32Type::from_naive_date(res) } @@ -1125,7 +1128,7 @@ impl Date64Type { /// * `i` - The Date64Type to convert pub fn to_naive_date(i: ::Native) -> NaiveDate { let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); - epoch.add(Duration::milliseconds(i)) + epoch.add(Duration::try_milliseconds(i).unwrap()) } /// Converts a chrono::NaiveDate into an arrow Date64Type @@ -1166,8 +1169,8 @@ impl Date64Type { ) -> ::Native { let (days, ms) = IntervalDayTimeType::to_parts(delta); let res = Date64Type::to_naive_date(date); - let res = res.add(Duration::days(days as i64)); - let res = res.add(Duration::milliseconds(ms as i64)); + let res = res.add(Duration::try_days(days as i64).unwrap()); + let res = res.add(Duration::try_milliseconds(ms as i64).unwrap()); Date64Type::from_naive_date(res) } @@ -1184,7 +1187,7 @@ impl Date64Type { let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(delta); let res = Date64Type::to_naive_date(date); let res = shift_months(res, months); - let res = res.add(Duration::days(days as i64)); + let res = res.add(Duration::try_days(days as i64).unwrap()); let res = res.add(Duration::nanoseconds(nanos)); Date64Type::from_naive_date(res) } @@ -1217,8 +1220,8 @@ impl Date64Type { ) -> ::Native { let (days, ms) = IntervalDayTimeType::to_parts(delta); let res = Date64Type::to_naive_date(date); - let res = res.sub(Duration::days(days as i64)); - let res = res.sub(Duration::milliseconds(ms as i64)); + let res = res.sub(Duration::try_days(days as i64).unwrap()); + let res = res.sub(Duration::try_milliseconds(ms as i64).unwrap()); Date64Type::from_naive_date(res) } @@ -1235,7 +1238,7 @@ impl Date64Type { let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(delta); let res = Date64Type::to_naive_date(date); let res = shift_months(res, -months); - let res = res.sub(Duration::days(days as i64)); + let res = res.sub(Duration::try_days(days as i64).unwrap()); let res = res.sub(Duration::nanoseconds(nanos)); Date64Type::from_naive_date(res) } diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 72942af8394a..6214e6d97371 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -273,7 +273,8 @@ pub fn string_to_timestamp_nanos(s: &str) -> Result { /// Fallible conversion of [`NaiveDateTime`] to `i64` nanoseconds #[inline] fn to_timestamp_nanos(dt: NaiveDateTime) -> Result { - dt.timestamp_nanos_opt() + dt.and_utc() + .timestamp_nanos_opt() .ok_or_else(|| ArrowError::ParseError(ERR_NANOSECONDS_NOT_SUPPORTED.to_string())) } @@ -632,8 +633,8 @@ impl Parser for Date32Type { impl Parser for Date64Type { fn parse(string: &str) -> Option { if string.len() <= 10 { - let date = parse_date(string)?; - Some(NaiveDateTime::new(date, NaiveTime::default()).timestamp_millis()) + let datetime = NaiveDateTime::new(parse_date(string)?, NaiveTime::default()); + Some(datetime.and_utc().timestamp_millis()) } else { let date_time = string_to_datetime(&Utc, string).ok()?; Some(date_time.timestamp_millis()) @@ -662,7 +663,7 @@ impl Parser for Date64Type { Some(date_time.timestamp_millis()) } else { let date_time = NaiveDateTime::parse_from_str(string, format).ok()?; - Some(date_time.timestamp_millis()) + Some(date_time.and_utc().timestamp_millis()) } } } @@ -1286,43 +1287,45 @@ mod tests { // Ensure both T and ' ' variants work assert_eq!( - naive_datetime.timestamp_nanos_opt().unwrap(), + naive_datetime.and_utc().timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08T13:42:29.190855").unwrap() ); assert_eq!( - naive_datetime.timestamp_nanos_opt().unwrap(), + naive_datetime.and_utc().timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08 13:42:29.190855").unwrap() ); // Also ensure that parsing timestamps with no fractional // second part works as well - let naive_datetime_whole_secs = NaiveDateTime::new( + let datetime_whole_secs = NaiveDateTime::new( NaiveDate::from_ymd_opt(2020, 9, 8).unwrap(), NaiveTime::from_hms_opt(13, 42, 29).unwrap(), - ); + ) + .and_utc(); // Ensure both T and ' ' variants work assert_eq!( - naive_datetime_whole_secs.timestamp_nanos_opt().unwrap(), + datetime_whole_secs.timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08T13:42:29").unwrap() ); assert_eq!( - naive_datetime_whole_secs.timestamp_nanos_opt().unwrap(), + datetime_whole_secs.timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08 13:42:29").unwrap() ); // ensure without time work // no time, should be the nano second at // 2020-09-08 0:0:0 - let naive_datetime_no_time = NaiveDateTime::new( + let datetime_no_time = NaiveDateTime::new( NaiveDate::from_ymd_opt(2020, 9, 8).unwrap(), NaiveTime::from_hms_opt(0, 0, 0).unwrap(), - ); + ) + .and_utc(); assert_eq!( - naive_datetime_no_time.timestamp_nanos_opt().unwrap(), + datetime_no_time.timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08").unwrap() ) } @@ -1434,12 +1437,12 @@ mod tests { // Ensure both T and ' ' variants work assert_eq!( - naive_datetime.timestamp_nanos_opt().unwrap(), + naive_datetime.and_utc().timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08T13:42:29.190855").unwrap() ); assert_eq!( - naive_datetime.timestamp_nanos_opt().unwrap(), + naive_datetime.and_utc().timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08 13:42:29.190855").unwrap() ); @@ -1450,12 +1453,12 @@ mod tests { // Ensure both T and ' ' variants work assert_eq!( - naive_datetime.timestamp_nanos_opt().unwrap(), + naive_datetime.and_utc().timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08T13:42:29").unwrap() ); assert_eq!( - naive_datetime.timestamp_nanos_opt().unwrap(), + naive_datetime.and_utc().timestamp_nanos_opt().unwrap(), parse_timestamp("2020-09-08 13:42:29").unwrap() ); diff --git a/arrow-json/src/writer.rs b/arrow-json/src/writer.rs index bb494b595064..d8045c330481 100644 --- a/arrow-json/src/writer.rs +++ b/arrow-json/src/writer.rs @@ -1046,6 +1046,7 @@ mod tests { let ts_nanos = ts_string .parse::() .unwrap() + .and_utc() .timestamp_nanos_opt() .unwrap(); let ts_micros = ts_nanos / 1000; @@ -1099,6 +1100,7 @@ mod tests { let ts_nanos = ts_string .parse::() .unwrap() + .and_utc() .timestamp_nanos_opt() .unwrap(); let ts_micros = ts_nanos / 1000; @@ -1159,6 +1161,7 @@ mod tests { let ts_millis = ts_string .parse::() .unwrap() + .and_utc() .timestamp_millis(); let arr_date32 = Date32Array::from(vec![ diff --git a/arrow/benches/cast_kernels.rs b/arrow/benches/cast_kernels.rs index 6632dbc57c56..1877ed0e7899 100644 --- a/arrow/benches/cast_kernels.rs +++ b/arrow/benches/cast_kernels.rs @@ -21,6 +21,7 @@ use criterion::Criterion; use rand::distributions::{Distribution, Standard, Uniform}; use rand::Rng; +use chrono::DateTime; use std::sync::Arc; extern crate arrow; @@ -63,8 +64,6 @@ fn build_utf8_date_array(size: usize, with_nulls: bool) -> ArrayRef { } fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef { - use chrono::NaiveDateTime; - // use random numbers to avoid spurious compiler optimizations wrt to branching let mut rng = seedable_rng(); let mut builder = StringBuilder::new(); @@ -74,7 +73,7 @@ fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef { if with_nulls && rng.gen::() > 0.8 { builder.append_null(); } else { - let string = NaiveDateTime::from_timestamp_opt(rng.sample(range), 0) + let string = DateTime::from_timestamp(rng.sample(range), 0) .unwrap() .format("%Y-%m-%dT%H:%M:%S") .to_string(); diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs index af5676ef5003..8132002b6e01 100644 --- a/object_store/src/lib.rs +++ b/object_store/src/lib.rs @@ -1693,7 +1693,9 @@ mod tests { } let options = GetOptions { - if_unmodified_since: Some(meta.last_modified + chrono::Duration::hours(10)), + if_unmodified_since: Some( + meta.last_modified + chrono::Duration::try_hours(10).unwrap(), + ), ..GetOptions::default() }; match storage.get_opts(&path, options).await { @@ -1702,7 +1704,9 @@ mod tests { } let options = GetOptions { - if_unmodified_since: Some(meta.last_modified - chrono::Duration::hours(10)), + if_unmodified_since: Some( + meta.last_modified - chrono::Duration::try_hours(10).unwrap(), + ), ..GetOptions::default() }; match storage.get_opts(&path, options).await { @@ -1720,7 +1724,7 @@ mod tests { } let options = GetOptions { - if_modified_since: Some(meta.last_modified - chrono::Duration::hours(10)), + if_modified_since: Some(meta.last_modified - chrono::Duration::try_hours(10).unwrap()), ..GetOptions::default() }; match storage.get_opts(&path, options).await { diff --git a/parquet_derive_test/src/lib.rs b/parquet_derive_test/src/lib.rs index a8b631ecc024..25734813a8d8 100644 --- a/parquet_derive_test/src/lib.rs +++ b/parquet_derive_test/src/lib.rs @@ -68,6 +68,7 @@ struct APartiallyCompleteRecord { mod tests { use super::*; + use chrono::SubsecRound; use std::{env, fs, io::Write, sync::Arc}; use parquet::{ @@ -202,9 +203,8 @@ mod tests { out.read_from_row_group(&mut *row_group, 1).unwrap(); // correct for rounding error when writing milliseconds - drs[0].now = - chrono::naive::NaiveDateTime::from_timestamp_millis(drs[0].now.timestamp_millis()) - .unwrap(); + + drs[0].now = drs[0].now.trunc_subsecs(3); assert!(out[0].double.is_nan()); // these three lines are necessary because NAN != NAN out[0].double = 0.;