Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow writing into Delta tables using deletion vectors #21784

Closed
wants to merge 1 commit into from
Closed

Allow writing into Delta tables using deletion vectors #21784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...a-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeltaLakeSchemaSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ private DeltaLakeSchemaSupport() {}
.add(CHANGE_DATA_FEED_FEATURE_NAME)
.add(COLUMN_MAPPING_FEATURE_NAME)
.add(TIMESTAMP_NTZ_FEATURE_NAME)
.add(DELETION_VECTORS_FEATURE_NAME)
.build();

public enum ColumnMappingMode
Expand Down
25 changes: 23 additions & 2 deletions plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ public class TestDeltaLakeBasic
private static final List<ResourceTable> OTHER_TABLES = ImmutableList.of(
new ResourceTable("stats_with_minmax_nulls", "deltalake/stats_with_minmax_nulls"),
new ResourceTable("no_column_stats", "databricks73/no_column_stats"),
new ResourceTable("deletion_vectors", "databricks122/deletion_vectors"),
new ResourceTable("timestamp_ntz", "databricks131/timestamp_ntz"),
new ResourceTable("timestamp_ntz_partition", "databricks131/timestamp_ntz_partition"));

Expand Down Expand Up @@ -936,8 +935,30 @@ public void testIdentityColumns()
*/
@Test
public void testDeletionVectors()
throws Exception
{
assertQuery("SELECT * FROM deletion_vectors", "VALUES (1, 11)");
String tableName = "deletion_vectors" + randomNameSuffix();

Path tableLocation = Files.createTempFile(tableName, null);
copyDirectoryContents(new File(Resources.getResource("databricks122/deletion_vectors").toURI()).toPath(), tableLocation);
assertUpdate("CALL system.register_table('%s', '%s', '%s')".formatted(getSession().getSchema().orElseThrow(), tableName, tableLocation.toUri()));

assertQuery("SELECT * FROM " + tableName, "VALUES (1, 11)");

assertUpdate("INSERT INTO " + tableName + " VALUES (3, 31), (3, 32)", 2);
assertQuery("SELECT * FROM " + tableName, "VALUES (1, 11), (3, 31), (3, 32)");

assertUpdate("DELETE FROM " + tableName + " WHERE a = 3 AND b = 31", 1);
assertQuery("SELECT * FROM " + tableName, "VALUES (1, 11), (3, 32)");

assertUpdate("UPDATE " + tableName + " SET a = -3 WHERE b = 32", 1);
assertQuery("SELECT * FROM " + tableName, "VALUES (1, 11), (-3, 32)");

assertUpdate("MERGE INTO " + tableName + " t USING " + tableName + " s " +
"ON (t.a = s.a) WHEN MATCHED THEN UPDATE SET b = -1", 2);
assertQuery("SELECT * FROM " + tableName, "VALUES (1, -1), (-3, -1)");
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reading logic for deletion vectors might be broken though I'm not sure if Databricks writes such transaction logs:

java.lang.AssertionError: For query 20240501_092829_00018_hvqds: 
 SELECT * FROM deletion_vectorsg763ca8zs4
not equal
Actual rows (up to 100 of 1 extra rows shown, 3 rows in total):
    [2, 22]
Expected rows (up to 100 of 0 missing rows shown, 2 rows in total):


assertUpdate("DROP TABLE " + tableName);
}

@Test
Expand Down
19 changes: 10 additions & 9 deletions ...ests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,15 +261,16 @@ public void testDeletionVectors(String mode)
.contains(row("a", "integer", "", ""), row("b", "integer", "", ""));

// TODO https://github.com/trinodb/trino/issues/17063 Use Delta Deletion Vectors for row-level deletes
assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (3, 33)"))
.hasMessageContaining("Unsupported writer features: [deletionVectors]");
assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM delta.default." + tableName))
.hasMessageContaining("Unsupported writer features: [deletionVectors]");
assertQueryFailure(() -> onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a = 3"))
.hasMessageContaining("Unsupported writer features: [deletionVectors]");
assertQueryFailure(() -> onTrino().executeQuery("MERGE INTO delta.default." + tableName + " t USING delta.default." + tableName + " s " +
"ON (t.a = s.a) WHEN MATCHED THEN UPDATE SET b = -1"))
.hasMessageContaining("Unsupported writer features: [deletionVectors]");
onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (3, 33)");
onTrino().executeQuery("DELETE FROM delta.default." + tableName + " WHERE a = 1");
onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a = 30 WHERE b = 33");
onTrino().executeQuery("MERGE INTO delta.default." + tableName + " t USING delta.default." + tableName + " s " +
"ON (t.a = s.a) WHEN MATCHED THEN UPDATE SET b = -1");

assertThat(onDelta().executeQuery("SELECT * FROM default." + tableName))
.containsOnly(row(2, -1), row(30, -1));
assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName))
.containsOnly(row(2, -1), row(30, -1));
}
finally {
dropDeltaTableWithRetry("default." + tableName);
Expand Down
4 changes: 2 additions & 2 deletions ...main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,10 +421,10 @@ public void testVacuumUnsupportedWriterFeature()
"(a INT)" +
"USING DELTA " +
"LOCATION '" + ("s3://" + bucketName + "/" + directoryName) + "'" +
"TBLPROPERTIES ('delta.enableDeletionVectors' = true)");
"TBLPROPERTIES ('delta.enableRowTracking' = true)");
try {
assertThatThrownBy(() -> onTrino().executeQuery("CALL delta.system.vacuum('default', '" + tableName + "', '7d')"))
.hasMessageContaining("Cannot execute vacuum procedure with [deletionVectors] writer features");
.hasMessageContaining("Cannot execute vacuum procedure with [rowTracking] writer features");
}
finally {
dropDeltaTableWithRetry("default." + tableName);
Expand Down
Loading