Implement sort order aware redistribution

apache · Dec 12, 2023 · f2ec70c · f2ec70c
1 parent 8aca272
commit f2ec70c
Show file tree

Hide file tree

Showing 5 changed files with 480 additions and 47 deletions.
diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rs
@@ -193,6 +193,7 @@ impl ExecutionPlan for CsvExec {
 
         let repartitioned_file_groups_option = FileGroupPartitioner::new()
             .with_target_partitions(target_partitions)
+            .with_preserve_order_within_groups(self.output_ordering().is_some())
             .with_repartition_file_min_size(repartition_file_min_size)
             .repartition_file_groups(&self.base_config.file_groups);