From 1d6776d6e2e0bfbeca7e4529f66213fcd28aa582 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Tue, 1 Aug 2023 09:41:32 -0400 Subject: [PATCH 01/11] add non-plural transform function names --- .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 2bfd0aaf8da7..193dc13dc29b 100644 --- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -374,14 +374,18 @@ public static Term toIcebergTerm(Transform transform) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index cdd8a45af691..15410abb5b73 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -342,14 +342,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month" case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 23a53ea9e8c3..0ed90e9455e2 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -359,14 +359,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index bbd7986b26d1..43faaa2ae4f6 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -368,14 +368,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year" case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month" case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": From c20e7006864df4e61357d0f6ccdc487eec3db51b Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Tue, 1 Aug 2023 13:18:33 -0400 Subject: [PATCH 02/11] typos --- .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 2 +- .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 15410abb5b73..0a6a893f9ebf 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -345,7 +345,7 @@ public static Term toIcebergTerm(Expression expr) { case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); - case "month" + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 43faaa2ae4f6..51f8e5957117 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -368,7 +368,7 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); - case "year" + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); case "month" From d95d4c3be9c1dedf4b3fc20bd444dace359a53e1 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Tue, 1 Aug 2023 13:59:16 -0400 Subject: [PATCH 03/11] typo --- .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 51f8e5957117..becea88bf167 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -371,7 +371,7 @@ public static Term toIcebergTerm(Expression expr) { case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); - case "month" + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": From ef6a352cd36a0c8acc7b85fe9c993e2df32bc1a5 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Tue, 1 Aug 2023 16:21:49 -0400 Subject: [PATCH 04/11] add tests for singular partition transform functions --- .../TestAlterTablePartitionFields.java | 76 +++++++++++++++++++ .../TestAlterTablePartitionFields.java | 76 +++++++++++++++++++ .../TestAlterTablePartitionFields.java | 76 +++++++++++++++++++ .../TestAlterTablePartitionFields.java | 76 +++++++++++++++++++ 4 files changed, 304 insertions(+) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 5102da844242..00841da995b7 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -179,6 +179,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddMonthPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddDayPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddHourPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 9b4bd12ec1bf..7fa59e4700dc 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -179,6 +179,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddMonthPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddDayPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddHourPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..fcfc0e1aef5e 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -179,6 +179,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddMonthPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddDayPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddHourPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..fcfc0e1aef5e 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -179,6 +179,82 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } + @Test + public void testAddYearPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddMonthPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddDayPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + + @Test + public void testAddHourPartition() { + sql( + "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", + tableName); + Table table = validationCatalog.loadTable(tableIdent); + + Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assert.assertEquals("Should have new spec field", expected, table.spec()); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); From deef41a48f1affbc05a25892a1c6710a78782da5 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Wed, 13 Sep 2023 14:34:07 -0400 Subject: [PATCH 05/11] update tests to assertj --- .../TestAlterTablePartitionFields.java | 33 ++++++++++++++----- .../TestAlterTablePartitionFields.java | 33 ++++++++++++++----- .../TestAlterTablePartitionFields.java | 33 ++++++++++++++----- .../TestAlterTablePartitionFields.java | 33 ++++++++++++++----- 4 files changed, 100 insertions(+), 32 deletions(-) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 00841da995b7..2a6150a6b235 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; +import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -186,7 +187,9 @@ public void testAddYearPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); @@ -195,7 +198,9 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -205,7 +210,9 @@ public void testAddMonthPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); @@ -214,7 +221,9 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -224,7 +233,9 @@ public void testAddDayPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); @@ -233,7 +244,9 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -243,7 +256,9 @@ public void testAddHourPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); @@ -252,7 +267,9 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 7fa59e4700dc..8d1753f03fa3 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; +import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -186,7 +187,9 @@ public void testAddYearPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); @@ -195,7 +198,9 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -205,7 +210,9 @@ public void testAddMonthPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); @@ -214,7 +221,9 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -224,7 +233,9 @@ public void testAddDayPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); @@ -233,7 +244,9 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -243,7 +256,9 @@ public void testAddHourPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); @@ -252,7 +267,9 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index fcfc0e1aef5e..e69c87171a51 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; +import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -186,7 +187,9 @@ public void testAddYearPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); @@ -195,7 +198,9 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -205,7 +210,9 @@ public void testAddMonthPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); @@ -214,7 +221,9 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -224,7 +233,9 @@ public void testAddDayPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); @@ -233,7 +244,9 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -243,7 +256,9 @@ public void testAddHourPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); @@ -252,7 +267,9 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index fcfc0e1aef5e..e69c87171a51 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; +import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -186,7 +187,9 @@ public void testAddYearPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); @@ -195,7 +198,9 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -205,7 +210,9 @@ public void testAddMonthPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); @@ -214,7 +221,9 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -224,7 +233,9 @@ public void testAddDayPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); @@ -233,7 +244,9 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test @@ -243,7 +256,9 @@ public void testAddHourPartition() { tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue("Table should start unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); @@ -252,7 +267,9 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assert.assertEquals("Should have new spec field", expected, table.spec()); + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); } @Test From 9a62170fc11951d49d31d24a07d5a34e3189ea02 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Wed, 20 Sep 2023 13:49:25 -0400 Subject: [PATCH 06/11] add spark 3.5 --- .../TestAlterTablePartitionFields.java | 85 +++++++++++++++++++ .../org/apache/iceberg/spark/Spark3Util.java | 4 + 2 files changed, 89 insertions(+) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index a43f2a041b97..15161446a868 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; +import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -179,6 +180,90 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } +@Test + public void testAddYearPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); + + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); + } + + @Test + public void testAddMonthPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); + + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); + } + + @Test + public void testAddDayPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); + + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); + } + + @Test + public void testAddHourPartition() { + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); + Table table = validationCatalog.loadTable(tableIdent); + + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should start unpartitioned") + .isTrue(); + + sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName); + + table.refresh(); + + PartitionSpec expected = + PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); + + Assertions.assertThat(table.spec()) + .as("Should have new spec field") + .isEqualTo(expected); + } + @Test public void testAddNamedPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 3611dd7960ac..0fabffc8950d 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -368,14 +368,18 @@ public static Term toIcebergTerm(Expression expr) { return org.apache.iceberg.expressions.Expressions.ref(colName); case "bucket": return org.apache.iceberg.expressions.Expressions.bucket(colName, findWidth(transform)); + case "year": case "years": return org.apache.iceberg.expressions.Expressions.year(colName); + case "month": case "months": return org.apache.iceberg.expressions.Expressions.month(colName); case "date": + case "day": case "days": return org.apache.iceberg.expressions.Expressions.day(colName); case "date_hour": + case "hour": case "hours": return org.apache.iceberg.expressions.Expressions.hour(colName); case "truncate": From a886a6a70f0e23eaee373d3584bc38a6a2ff1099 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Wed, 20 Sep 2023 13:49:39 -0400 Subject: [PATCH 07/11] use new createTable() in tests --- .../TestAlterTablePartitionFields.java | 16 ++++------------ .../TestAlterTablePartitionFields.java | 16 ++++------------ .../TestAlterTablePartitionFields.java | 16 ++++------------ .../TestAlterTablePartitionFields.java | 16 ++++------------ 4 files changed, 16 insertions(+), 48 deletions(-) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 2a6150a6b235..16fd1d81a05d 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -182,9 +182,7 @@ public void testAddHoursPartition() { @Test public void testAddYearPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -205,9 +203,7 @@ public void testAddYearPartition() { @Test public void testAddMonthPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -228,9 +224,7 @@ public void testAddMonthPartition() { @Test public void testAddDayPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -251,9 +245,7 @@ public void testAddDayPartition() { @Test public void testAddHourPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 8d1753f03fa3..6c8549b240f9 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -182,9 +182,7 @@ public void testAddHoursPartition() { @Test public void testAddYearPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -205,9 +203,7 @@ public void testAddYearPartition() { @Test public void testAddMonthPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -228,9 +224,7 @@ public void testAddMonthPartition() { @Test public void testAddDayPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -251,9 +245,7 @@ public void testAddDayPartition() { @Test public void testAddHourPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index e69c87171a51..dcace78e23b3 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -182,9 +182,7 @@ public void testAddHoursPartition() { @Test public void testAddYearPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -205,9 +203,7 @@ public void testAddYearPartition() { @Test public void testAddMonthPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -228,9 +224,7 @@ public void testAddMonthPartition() { @Test public void testAddDayPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -251,9 +245,7 @@ public void testAddDayPartition() { @Test public void testAddHourPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index e69c87171a51..dcace78e23b3 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -182,9 +182,7 @@ public void testAddHoursPartition() { @Test public void testAddYearPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -205,9 +203,7 @@ public void testAddYearPartition() { @Test public void testAddMonthPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -228,9 +224,7 @@ public void testAddMonthPartition() { @Test public void testAddDayPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) @@ -251,9 +245,7 @@ public void testAddDayPartition() { @Test public void testAddHourPartition() { - sql( - "CREATE TABLE %s (id bigint NOT NULL, category string, ts timestamp, data string) USING iceberg", - tableName); + createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); Assertions.assertThat(table.spec().isUnpartitioned()) From 0b9c76e1fb420abe985478d3cb78a95f2572adb5 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Wed, 20 Sep 2023 13:53:45 -0400 Subject: [PATCH 08/11] formatting --- .../TestAlterTablePartitionFields.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 15161446a868..0e978e52e570 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,11 +27,11 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; -import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -180,7 +180,7 @@ public void testAddHoursPartition() { Assert.assertEquals("Should have new spec field", expected, table.spec()); } -@Test + @Test public void testAddYearPartition() { createTable("id bigint NOT NULL, category string, ts timestamp, data string"); Table table = validationCatalog.loadTable(tableIdent); @@ -196,9 +196,7 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -217,9 +215,7 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -238,9 +234,7 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -259,9 +253,7 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test From cc49a0a4fef81b868500a9694d8db211930a2d13 Mon Sep 17 00:00:00 2001 From: Chris Lettieri Date: Thu, 21 Sep 2023 10:40:48 -0400 Subject: [PATCH 09/11] create table test --- .../org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../iceberg/spark/sql/TestCreateTable.java | 20 +++++++++++++++++++ .../org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../iceberg/spark/sql/TestCreateTable.java | 20 +++++++++++++++++++ .../org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../iceberg/spark/sql/TestCreateTable.java | 20 +++++++++++++++++++ .../org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../iceberg/spark/sql/TestCreateTable.java | 20 +++++++++++++++++++ .../org/apache/iceberg/spark/Spark3Util.java | 4 ++++ .../iceberg/spark/sql/TestCreateTable.java | 20 +++++++++++++++++++ 10 files changed, 120 insertions(+) diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 193dc13dc29b..3e0452d94ac5 100644 --- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -421,17 +421,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 1411c83ddc65..a6256afcdf65 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 0a6a893f9ebf..c5c7ee4d53d7 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -403,17 +403,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 1411c83ddc65..a6256afcdf65 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 0ed90e9455e2..2f04ae374951 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -420,17 +420,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour" case "hours": builder.hour(colName); break; diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 1411c83ddc65..a6256afcdf65 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index becea88bf167..62301e9676b8 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -429,17 +429,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 01b8b99062a0..ecfd6759b900 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 0fabffc8950d..cfcc3941c748 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -429,17 +429,21 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition case "bucket": builder.bucket(colName, findWidth(transform)); break; + case "year": case "years": builder.year(colName); break; + case "month": case "months": builder.month(colName); break; case "date": + case "day": case "days": builder.day(colName); break; case "date_hour": + case "hour": case "hours": builder.hour(colName); break; diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java index 01b8b99062a0..ecfd6759b900 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java @@ -63,6 +63,26 @@ public void testTransformIgnoreCase() { Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent)); } + @Test + public void testTransformSingularForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hour(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + + @Test + public void testTransformPluralForm() { + Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); + sql( + "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " + + "USING iceberg partitioned by (hours(ts))", + tableName); + Assert.assertTrue("Table should exist", validationCatalog.tableExists(tableIdent)); + } + @Test public void testCreateTable() { Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent)); From c13c989fc05c9febd8489fb226653723af48189f Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Sun, 24 Sep 2023 13:29:03 -0700 Subject: [PATCH 10/11] Add missing : to fix syntax error. --- .../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 2f04ae374951..d7717e2bfd49 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -434,7 +434,7 @@ public static PartitionSpec toPartitionSpec(Schema schema, Transform[] partition builder.day(colName); break; case "date_hour": - case "hour" + case "hour": case "hours": builder.hour(colName); break; From 3bcdd3d08a29df326a1196581f1f0938fef0d9a1 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Sun, 24 Sep 2023 13:30:11 -0700 Subject: [PATCH 11/11] Apply spotless. --- .../TestAlterTablePartitionFields.java | 18 +++++------------- .../TestAlterTablePartitionFields.java | 18 +++++------------- .../TestAlterTablePartitionFields.java | 18 +++++------------- .../TestAlterTablePartitionFields.java | 18 +++++------------- 4 files changed, 20 insertions(+), 52 deletions(-) diff --git a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 16fd1d81a05d..5589b1b05c4b 100644 --- a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,11 +27,11 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; -import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -196,9 +196,7 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -217,9 +215,7 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -238,9 +234,7 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -259,9 +253,7 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test diff --git a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 6c8549b240f9..2ecf6b0c4ca7 100644 --- a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,11 +27,11 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; -import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -196,9 +196,7 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -217,9 +215,7 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -238,9 +234,7 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -259,9 +253,7 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test diff --git a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index dcace78e23b3..0e978e52e570 100644 --- a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,11 +27,11 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; -import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -196,9 +196,7 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -217,9 +215,7 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -238,9 +234,7 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -259,9 +253,7 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index dcace78e23b3..0e978e52e570 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -27,11 +27,11 @@ import org.apache.spark.sql.connector.catalog.CatalogManager; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; -import org.assertj.core.api.Assertions; public class TestAlterTablePartitionFields extends SparkExtensionsTestBase { @@ -196,9 +196,7 @@ public void testAddYearPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -217,9 +215,7 @@ public void testAddMonthPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -238,9 +234,7 @@ public void testAddDayPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test @@ -259,9 +253,7 @@ public void testAddHourPartition() { PartitionSpec expected = PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build(); - Assertions.assertThat(table.spec()) - .as("Should have new spec field") - .isEqualTo(expected); + Assertions.assertThat(table.spec()).as("Should have new spec field").isEqualTo(expected); } @Test