Audit action field optionality

python/src/lib.rs

@@ -54,7 +54,7 @@ struct RawDeltaTableMetaData {
     #[pyo3(get)]
     partition_columns: Vec<String>,
     #[pyo3(get)]
-    created_time: deltalake::DeltaDataTypeTimestamp,
+    created_time: Option<deltalake::DeltaDataTypeTimestamp>,
     #[pyo3(get)]
     configuration: HashMap<String, Option<String>>,
 }

rust/src/action.rs

@@ -406,7 +406,7 @@ pub struct MetaData {
     /// An array containing the names of columns by which the data should be partitioned
     pub partition_columns: Vec<String>,
     /// The time when this metadata action is created, in milliseconds since the Unix epoch
-    pub created_time: DeltaDataTypeTimestamp,
+    pub created_time: Option<DeltaDataTypeTimestamp>,
     /// A map containing configuration options for the table
     pub configuration: HashMap<String, Option<String>>,
 }
@@ -459,9 +459,10 @@ impl MetaData {
                         .clone();
                 }
                 "createdTime" => {
-                    re.created_time = record
-                        .get_long(i)
-                        .map_err(|_| gen_action_type_error("metaData", "createdTime", "long"))?;
+                    re.created_time =
+                        Some(record.get_long(i).map_err(|_| {
+                            gen_action_type_error("metaData", "createdTime", "long")
+                        })?);
                 }
                 "configuration" => {
                     let configuration_map = record
@@ -537,7 +538,7 @@ pub struct Remove {
     /// The path of the file that is removed from the table.
     pub path: String,
     /// The timestamp when the remove was added to table state.
-    pub deletion_timestamp: DeltaDataTypeTimestamp,
+    pub deletion_timestamp: Option<DeltaDataTypeTimestamp>,
     /// Whether data is changed by the remove. A table optimize will report this as false for
     /// example, since it adds and removes files by combining many files into one.
     pub data_change: bool,
@@ -554,6 +555,8 @@ pub struct Remove {
 impl Remove {
     fn from_parquet_record(record: &parquet::record::Row) -> Result<Self, ActionError> {
         let mut re = Self {
+            data_change: true,
+            extended_file_metadata: Some(false),
             ..Default::default()
         };
 
@@ -576,9 +579,9 @@ impl Remove {
                     })?);
                 }
                 "deletionTimestamp" => {
-                    re.deletion_timestamp = record.get_long(i).map_err(|_| {
+                    re.deletion_timestamp = Some(record.get_long(i).map_err(|_| {
                         gen_action_type_error("remove", "deletionTimestamp", "long")
-                    })?;
+                    })?);
                 }
                 "partitionValues" => match record.get_map(i) {
                     Ok(_) => {

rust/src/delta.rs

@@ -202,7 +202,7 @@ pub struct DeltaTableMetaData {
     /// An array containing the names of columns by which the data should be partitioned
     pub partition_columns: Vec<String>,
     /// The time when this metadata action is created, in milliseconds since the Unix epoch
-    pub created_time: DeltaDataTypeTimestamp,
+    pub created_time: Option<DeltaDataTypeTimestamp>,
     /// table properties
     pub configuration: HashMap<String, Option<String>>,
 }
@@ -226,7 +226,7 @@ impl DeltaTableMetaData {
             format: format.unwrap_or_default(),
             schema,
             partition_columns,
-            created_time: Utc::now().timestamp_millis(),
+            created_time: Some(Utc::now().timestamp_millis()),
             configuration,
         }
     }
@@ -919,7 +919,7 @@ impl DeltaTable {
         Ok(self
             .get_tombstones()
             .iter()
-            .filter(|tombstone| tombstone.deletion_timestamp < delete_before_timestamp)
+            .filter(|tombstone| tombstone.deletion_timestamp.unwrap_or(0) < delete_before_timestamp)
             .map(|tombstone| tombstone.path.as_str())
             .collect::<HashSet<_>>())
     }

rust/tests/azure_test.rs

@@ -36,7 +36,7 @@ mod azure {
             deltalake::action::Remove {
                 path: "part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet"
                     .to_string(),
-                deletion_timestamp: 1587968596250,
+                deletion_timestamp: Some(1587968596250),
                 data_change: true,
                 ..Default::default()
             }

rust/tests/read_delta_test.rs

@@ -28,8 +28,9 @@ async fn read_delta_2_0_table_without_version() {
         tombstones[0],
         deltalake::action::Remove {
             path: "part-00000-512e1537-8aaa-4193-b8b4-bef3de0de409-c000.snappy.parquet".to_string(),
-            deletion_timestamp: 1564524298213,
+            deletion_timestamp: Some(1564524298213),
             data_change: false,
+            extended_file_metadata: Some(false),
             ..Default::default()
         }
     );
@@ -134,7 +135,7 @@ async fn read_delta_8_0_table_without_version() {
         tombstones[0],
         deltalake::action::Remove {
             path: "part-00001-911a94a2-43f6-4acb-8620-5e68c2654989-c000.snappy.parquet".to_string(),
-            deletion_timestamp: 1615043776198,
+            deletion_timestamp: Some(1615043776198),
             data_change: true,
             extended_file_metadata: Some(true),
             partition_values: Some(HashMap::new()),

rust/tests/read_simple_table_test.rs

@@ -30,8 +30,9 @@ async fn read_simple_table() {
         tombstones[0],
         deltalake::action::Remove {
             path: "part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet".to_string(),
-            deletion_timestamp: 1587968596250,
+            deletion_timestamp: Some(1587968596250),
             data_change: true,
+            extended_file_metadata: None,
             ..Default::default()
         }
     );

rust/tests/s3_test.rs

@@ -41,7 +41,7 @@ mod s3 {
             deltalake::action::Remove {
                 path: "part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet"
                     .to_string(),
-                deletion_timestamp: 1587968596250,
+                deletion_timestamp: Some(1587968596250),
                 data_change: true,
                 ..Default::default()
             }
@@ -77,7 +77,7 @@ mod s3 {
             deltalake::action::Remove {
                 path: "part-00006-63ce9deb-bc0f-482d-b9a1-7e717b67f294-c000.snappy.parquet"
                     .to_string(),
-                deletion_timestamp: 1587968596250,
+                deletion_timestamp: Some(1587968596250),
                 data_change: true,
                 ..Default::default()
             }

rust/tests/write_exploration.rs

@@ -311,7 +311,7 @@ pub fn create_remove(path: String) -> Remove {
 
     Remove {
         path,
-        deletion_timestamp: deletion_timestamp,
+        deletion_timestamp: Some(deletion_timestamp),
         data_change: true,
         extended_file_metadata: Some(false),
         ..Default::default()