From 9ab839b611a444155e19fb536625072fbec611e1 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 30 Jun 2023 11:33:43 +0800 Subject: [PATCH 1/4] ignore --- .../org/apache/spark/sql/CatalogSuite.scala | 19 +++ .../apache/spark/sql/ClientE2ETestSuite.scala | 116 ++++++++++++++++++ .../spark/sql/DataFrameNaFunctionSuite.scala | 32 +++++ .../apache/spark/sql/DataFrameStatSuite.scala | 15 +++ .../KeyValueGroupedDatasetE2ETestSuite.scala | 79 ++++++++++++ .../spark/sql/SparkSessionE2ESuite.scala | 5 + .../sql/UserDefinedFunctionE2ETestSuite.scala | 46 +++++++ .../spark/sql/application/ReplE2ESuite.scala | 47 ++++--- .../client/util/RemoteSparkSession.scala | 70 ++++++----- ...FlatMapGroupsWithStateStreamingSuite.scala | 9 ++ .../sql/streaming/StreamingQuerySuite.scala | 15 +++ 11 files changed, 402 insertions(+), 51 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala index 04b3f4e639a06..42609df8e9649 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala @@ -21,6 +21,7 @@ import java.io.{File, FilenameFilter} import io.grpc.StatusRuntimeException import org.apache.commons.io.FileUtils +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.apache.spark.sql.connect.client.util.RemoteSparkSession import org.apache.spark.sql.types.{DoubleType, LongType, StructType} @@ -29,6 +30,8 @@ import org.apache.spark.storage.StorageLevel class CatalogSuite extends RemoteSparkSession with SQLHelper { test("Database APIs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val currentDb = spark.catalog.currentDatabase assert(currentDb == "default") withTempDatabase { db => @@ -60,6 +63,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("CatalogMetadata APIs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val currentCatalog = spark.catalog.currentCatalog() assert(currentCatalog == "spark_catalog") try { @@ -87,6 +92,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("Table APIs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.catalog.listTables().collect().isEmpty) val parquetTableName = "parquet_table" val orcTableName = "orc_table" @@ -150,6 +157,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("Cache Table APIs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val parquetTableName = "parquet_table" withTable(parquetTableName) { withTempPath { table1Dir => @@ -175,6 +184,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("TempView APIs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val viewName = "view1" val globalViewName = "g_view1" try { @@ -197,6 +208,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("Function API") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val dbName = spark.catalog.currentDatabase val functions1 = spark.catalog.listFunctions().collect() assert(functions1.nonEmpty) @@ -221,6 +234,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("recoverPartitions") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val tableName = "test" withTable(tableName) { withTempPath { dir => @@ -245,6 +260,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("refreshTable") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTempPath { dir => val tableName = "spark_catalog.default.my_table" withTable(tableName) { @@ -270,6 +287,8 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("refreshByPath") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTempPath { dir => val tableName = "spark_catalog.default.my_table" withTable(tableName) { diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala index 0ababaa0af1a6..c30c45f7ecd4c 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala @@ -46,6 +46,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM // Spark Result test("spark result schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.sql("select val from (values ('Hello'), ('World')) as t(val)") df.withResult { result => val schema = result.schema @@ -54,6 +56,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("spark result array") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.sql("select val from (values ('Hello'), ('World')) as t(val)") val result = df.collect() assert(result.length == 2) @@ -62,6 +66,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("eager execution of sql") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assume(IntegrationTestUtils.isSparkHiveJarAvailable) withTable("test_martin") { // Fails, because table does not exist. @@ -80,6 +86,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("simple dataset") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10).limit(3) val result = df.collect() assert(result.length == 3) @@ -89,6 +97,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("read and write") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val testDataPath = java.nio.file.Paths .get( IntegrationTestUtils.sparkHome, @@ -129,6 +139,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("read path collision") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val testDataPath = java.nio.file.Paths .get( IntegrationTestUtils.sparkHome, @@ -159,6 +171,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("textFile") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val testDataPath = java.nio.file.Paths .get( IntegrationTestUtils.sparkHome, @@ -179,6 +193,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("write table") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("myTable") { val df = spark.range(10).limit(3) df.write.mode(SaveMode.Overwrite).saveAsTable("myTable") @@ -194,6 +210,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("different spark session join/union") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10).limit(3) val spark2 = SparkSession @@ -222,11 +240,15 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("write without table or path") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // Should receive no error to write noop spark.range(10).write.format("noop").mode("append").save() } test("write jdbc") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assume(IntegrationTestUtils.isSparkHiveJarAvailable) if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) { val url = "jdbc:derby:memory:1234" @@ -245,6 +267,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("writeTo with create") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")) @@ -259,6 +283,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("writeTo with create and using") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")) @@ -279,6 +305,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("writeTo with create and append") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")) @@ -296,6 +324,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("WriteTo with overwrite") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows1 = (1L to 3L).map { i => @@ -327,6 +357,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("WriteTo with overwritePartitions") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = (4L to 7L).map { i => @@ -351,6 +383,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("write path collision") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) val outputFolderPath = Files.createTempDirectory("output").toAbsolutePath // Failed because the path cannot be provided both via option and save method. @@ -397,6 +431,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM // Dataset tests test("Dataset inspection") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) val local = spark.newDataFrame { builder => builder.getLocalRelationBuilder.setSchema(simpleSchema.catalogString) @@ -409,6 +445,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) assert(df.schema === simpleSchema) assert(df.dtypes === Array(("value", "LongType"))) @@ -418,6 +456,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataframe schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.sql("select * from range(10)") val expectedSchema = new StructType().add("id", "long", nullable = false) assert(df.schema === expectedSchema) @@ -428,6 +468,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset explain") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) val simpleExplainFragments = Seq("== Physical Plan ==") testCapturedStdOut(df.explain(), simpleExplainFragments: _*) @@ -448,6 +490,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset result collection") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) def checkResult(rows: TraversableOnce[java.lang.Long], expectedValues: Long*): Unit = { rows.toIterator.zipAll(expectedValues.iterator, null, null).foreach { case (actual, expected) => assert(actual === expected) @@ -473,6 +517,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset show") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(20) testCapturedStdOut(df.show(), 24, 5, "+---+", "| id|", "| 0|", "| 19|") testCapturedStdOut( @@ -518,6 +564,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset randomSplit") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) implicit val tolerance = TolerantNumerics.tolerantDoubleEquality(0.01) val df = spark.range(100) @@ -546,10 +594,14 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset count") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.range(10).count() === 10) } test("Dataset collect tuple") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val result = session @@ -592,6 +644,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset collect complex type") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val result = session @@ -603,17 +657,23 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset typed select - simple column") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val numRows = spark.range(1000).select(count("id")).first() assert(numRows === 1000) } test("Dataset typed select - multiple columns") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val result = spark.range(1000).select(count("id"), sum("id")).first() assert(result.getLong(0) === 1000) assert(result.getLong(1) === 499500) } test("Dataset typed select - complex column") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val ds = session @@ -623,6 +683,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset typed select - multiple complex columns") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val s = struct(generateMyTypeColumns: _*).as[MyType] @@ -633,6 +695,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("lambda functions") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This test is mostly to validate lambda variables are properly resolved. val result = spark .range(3) @@ -649,6 +713,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("shuffle array") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // We cannot do structural tests for shuffle because its random seed will always change. val result = spark .sql("select 1") @@ -659,6 +725,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("ambiguous joins") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val left = spark.range(100).select(col("id"), rand(10).as("a")) val right = spark.range(100).select(col("id"), rand(12).as("a")) val joined = left.join(right, left("id") === right("id")).select(left("id"), right("a")) @@ -671,6 +739,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("broadcast join") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf("spark.sql.autoBroadcastJoinThreshold" -> "-1") { val left = spark.range(100).select(col("id"), rand(10).as("a")) val right = spark.range(100).select(col("id"), rand(12).as("a")) @@ -682,6 +752,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("test temp view") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) try { spark.range(100).createTempView("test1") assert(spark.sql("SELECT * FROM test1").count() == 100) @@ -698,11 +770,15 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("time") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val timeFragments = Seq("Time taken: ", " ms") testCapturedStdOut(spark.time(spark.sql("select 1").collect()), timeFragments: _*) } test("RuntimeConfig") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) intercept[NoSuchElementException](spark.conf.get("foo.bar")) assert(spark.conf.getOption("foo.bar").isEmpty) spark.conf.set("foo.bar", value = true) @@ -722,6 +798,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkVersion") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.version.nonEmpty) assert(spark.version == SPARK_VERSION) } @@ -733,6 +811,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Local Relation implicit conversion") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -748,12 +828,16 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession.createDataFrame - row") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = java.util.Arrays.asList(Row("bob", 99), Row("Club", 5), Row("Bag", 5)) val schema = new StructType().add("key", "string").add("value", "int") checkSameResult(rows.asScala, spark.createDataFrame(rows, schema)) } test("SparkSession.createDataFrame - bean") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) def bean(v: String): SimpleBean = { val bean = new SimpleBean bean.setValue(v) @@ -766,6 +850,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession typed createDataSet/createDataframe") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val list = java.util.Arrays.asList(KV("bob", 99), KV("Club", 5), KV("Bag", 5)) @@ -776,12 +862,16 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession newSession") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val oldId = spark.sql("SELECT 1").analyze.getSessionId val newId = spark.newSession().sql("SELECT 1").analyze.getSessionId assert(oldId != newId) } test("createDataFrame from complex type schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val schema = new StructType() .add( "c1", @@ -794,12 +884,16 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SameSemantics") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val plan = spark.sql("select 1") val otherPlan = spark.sql("select 1") assert(plan.sameSemantics(otherPlan)) } test("sameSemantics and semanticHash") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df1 = spark.createDataFrame(Seq((1, 2), (4, 5))) val df2 = spark.createDataFrame(Seq((1, 2), (4, 5))) val df3 = spark.createDataFrame(Seq((0, 2), (4, 5))) @@ -815,6 +909,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("toJSON") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val expected = Array( """{"b":0.0,"id":0,"d":"world","a":0}""", """{"b":0.1,"id":1,"d":"world","a":1}""", @@ -828,6 +924,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("json from Dataset[String] inferSchema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val expected = Seq( @@ -840,6 +938,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("json from Dataset[String] with schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val schema = new StructType().add("city", StringType).add("name", StringType) @@ -850,6 +950,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("json from Dataset[String] with invalid schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val message = intercept[ParseException] { spark.read.schema("123").json(spark.createDataset(Seq.empty[String])(StringEncoder)) }.getMessage @@ -857,6 +959,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("csv from Dataset[String] inferSchema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val expected = Seq( @@ -872,6 +976,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("csv from Dataset[String] with schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val schema = new StructType().add("name", StringType).add("age", LongType) @@ -882,6 +988,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("csv from Dataset[String] with invalid schema") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val message = intercept[ParseException] { spark.read.schema("123").csv(spark.createDataset(Seq.empty[String])(StringEncoder)) }.getMessage @@ -889,6 +997,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset result destructive iterator") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // Helper methods for accessing private field `idxToBatches` from SparkResult val _idxToBatches = PrivateMethod[mutable.Map[Int, ColumnarBatch]](Symbol("idxToBatches")) @@ -934,6 +1044,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession.createDataFrame - large data set") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val threshold = 1024 * 1024 withSQLConf(SQLConf.LOCAL_RELATION_CACHE_THRESHOLD.key -> threshold.toString) { val count = 2 @@ -949,6 +1061,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("sql() with positional parameters") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val result0 = spark.sql("select 1", Array.empty).collect() assert(result0.length == 1 && result0(0).getInt(0) === 1) @@ -962,6 +1076,8 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("sql() with named parameters") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val result0 = spark.sql("select 1", Map.empty[String, Any]).collect() assert(result0.length == 1 && result0(0).getInt(0) === 1) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala index 1f6ea879248dc..bd126759e4ef1 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala @@ -19,6 +19,8 @@ package org.apache.spark.sql import scala.collection.JavaConverters._ +import org.apache.commons.lang3.{JavaVersion, SystemUtils} + import org.apache.spark.sql.connect.client.util.QueryTest import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StringType, StructType} @@ -68,6 +70,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("drop") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val rows = input.collect() @@ -95,6 +99,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("drop with how") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val rows = input.collect() @@ -112,6 +118,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("drop with threshold") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val rows = input.collect() @@ -124,6 +132,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sparkSession = spark import sparkSession.implicits._ @@ -225,6 +235,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill with map") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sparkSession = spark import sparkSession.implicits._ @@ -271,12 +283,16 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill with col(*)") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDF() // If columns are specified with "*", they are ignored. checkAnswer(df.na.fill("new name", Seq("*")), df.collect()) } test("drop with col(*)") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDF() val ex = intercept[RuntimeException] { df.na.drop("any", Seq("*")).collect() @@ -285,11 +301,15 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill with nested columns") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDFWithNestedColumns checkAnswer(df.na.fill("a1", Seq("c1.c1-1")), df) } test("drop with nested columns") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDFWithNestedColumns // Rows with the specified nested columns whose null values are dropped. @@ -298,6 +318,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val result1 = input.na @@ -335,6 +357,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace with null") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = spark.sql( "select name, height, married from (values " + "('Bob', 176.5, true), " + @@ -373,6 +397,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace nan with float") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) checkAnswer( createNaNDF().na.replace("*", Map(Float.NaN -> 10.0f)), Row(1, 1L, 1.toShort, 1.toByte, 1.0f, 1.0) :: @@ -380,6 +406,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace nan with double") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) checkAnswer( createNaNDF().na.replace("*", Map(Double.NaN -> 10.0)), Row(1, 1L, 1.toShort, 1.toByte, 1.0f, 1.0) :: @@ -387,6 +415,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace float with nan") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { checkAnswer( createNaNDF().na.replace("*", Map(1.0f -> Float.NaN)), @@ -396,6 +426,8 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace double with nan") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { checkAnswer( createNaNDF().na.replace("*", Map(1.0 -> Double.NaN)), diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala index aea31005f3bd6..4e23b701a6e78 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql import java.util.Random import io.grpc.StatusRuntimeException +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.matchers.must.Matchers._ import org.apache.spark.sql.connect.client.util.RemoteSparkSession @@ -28,6 +29,8 @@ class DataFrameStatSuite extends RemoteSparkSession { private def toLetter(i: Int): String = (i + 97).toChar.toString test("approxQuantile") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -79,6 +82,8 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("covariance") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -96,6 +101,8 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("correlation") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -110,6 +117,8 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("crosstab") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -132,6 +141,8 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("freqItems") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -151,6 +162,8 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("sampleBy") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val df = Seq("Bob", "Alice", "Nico", "Bob", "Alice").toDF("name") @@ -164,6 +177,8 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("countMinSketch") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(1000) val sketch1 = df.stat.countMinSketch("id", depth = 10, width = 20, seed = 42) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala index 79bedabd559b6..e1c83063b9fb9 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala @@ -20,6 +20,7 @@ import java.sql.Timestamp import java.util.Arrays import io.grpc.StatusRuntimeException +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Append import org.apache.spark.sql.connect.client.util.QueryTest @@ -40,6 +41,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { import session.implicits._ test("mapGroups") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val values = spark @@ -51,6 +54,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatGroupMap") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -60,6 +65,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keys") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -69,6 +76,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keyAs - keys") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // It is okay to cast from Long to Double, but not Long to Int. val values = spark .range(10) @@ -80,6 +89,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupByKey, keyAs - duplicates") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val result = spark @@ -93,6 +104,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupByKey, keyAs, keys - duplicates") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val result = spark @@ -106,6 +119,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keyAs - flatGroupMap") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -116,6 +131,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keyAs mapValues - cogroup") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -147,6 +164,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapValues - flatGroupMap") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -157,6 +176,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapValues - keys") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -167,6 +188,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatMapSortedGroups") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -190,6 +213,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("cogroup") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -206,6 +231,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("cogroupSorted") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -233,6 +260,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("agg, keyAs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = spark .range(10) .groupByKey(v => v % 2) @@ -243,6 +272,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() @@ -255,6 +286,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -265,6 +298,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -275,6 +310,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -290,6 +327,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -306,6 +345,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -323,6 +364,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr, expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -341,6 +384,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr, expr, expr, expr") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -360,6 +405,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: Enable top-level Option of Product encoders") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val data = Seq(Some((1, "a")), Some((2, "b")), None) val ds = data.toDS() @@ -394,6 +441,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: Resolving Option[Product] field") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq((1, ("a", 1.0)), (2, ("b", 2.0)), (3, null)) .toDS() .as[(Int, Option[(String, Double)])] @@ -401,6 +450,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: select Option[Product] field") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 1), ("b", 2), ("c", 3)).toDS() val ds1 = ds.select(expr("struct(_2, _2 + 1)").as[Option[(Int, Int)]]) checkDataset(ds1, Some((1, 2)), Some((2, 3)), Some((3, 4))) @@ -410,6 +461,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: typed agg on Option[Product] type") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(Some((1, 2)), Some((2, 3)), Some((1, 3))).toDS() assert(ds.groupByKey(_.get._1).count().collect() === Seq((1, 2), (2, 1))) @@ -419,6 +472,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-25942: typed aggregation on primitive type") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(1, 2, 3).toDS() val agg = ds @@ -428,12 +483,16 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-25942: typed aggregation on product type") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq((1, 2), (2, 3), (3, 4)).toDS() val agg = ds.groupByKey(x => x).agg(sum("_1").as[Long], sum($"_2" + 1).as[Long]) checkDatasetUnorderly(agg, ((1, 2), 1L, 3L), ((2, 3), 2L, 4L), ((3, 4), 3L, 5L)) } test("SPARK-26085: fix key attribute name for atomic type for typed aggregation") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // TODO(SPARK-43416): Recursively rename the position based tuple to the schema name from the // server. val ds = Seq(1, 2, 3).toDS() @@ -446,6 +505,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("reduceGroups") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq("abc", "xyz", "hello").toDS() checkDatasetUnorderly( ds.groupByKey(_.length).reduceGroups(_ + _), @@ -454,6 +515,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupby") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 1, 10), ("a", 2, 20), ("b", 2, 1), ("b", 1, 2), ("c", 1, 1)) .toDF("key", "seq", "value") val grouped = ds.groupBy($"key").as[String, (String, Int, Int)] @@ -473,6 +536,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupby - keyAs, keys") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 1, 10), ("a", 2, 20), ("b", 2, 1), ("b", 1, 2), ("c", 1, 1)) .toDF("key", "seq", "value") val grouped = ds.groupBy($"value").as[String, (String, Int, Int)] @@ -482,6 +547,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatMapGroupsWithState") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { if (state.exists) throw new IllegalArgumentException("state.exists should be false") @@ -505,6 +572,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatMapGroupsWithState - with initial state") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { val currState = state.getOption.getOrElse(ClickState(key, 0)) @@ -532,6 +601,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { if (state.exists) throw new IllegalArgumentException("state.exists should be false") @@ -555,6 +626,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState - with initial state") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { val currState = state.getOption.getOrElse(ClickState(key, 0)) @@ -582,6 +655,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("RowEncoder in udf") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") checkDatasetUnorderly( @@ -592,6 +667,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapGroups with row encoder") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") checkDataset( @@ -606,6 +683,8 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("coGroup with row encoder") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df1 = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") val df2 = Seq(("x", 10), ("x", 20), ("y", 1), ("y", 2), ("a", 1)).toDF("c1", "c2") diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala index fb295c00edc02..62a64f4fc22af 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala @@ -20,6 +20,7 @@ import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} import scala.concurrent.duration._ import scala.util.{Failure, Success} +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.concurrent.Eventually._ import org.apache.spark.sql.connect.client.util.RemoteSparkSession @@ -33,6 +34,8 @@ import org.apache.spark.util.ThreadUtils class SparkSessionE2ESuite extends RemoteSparkSession { test("interrupt all - background queries, foreground interrupt") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ implicit val ec: ExecutionContextExecutor = ExecutionContext.global @@ -73,6 +76,8 @@ class SparkSessionE2ESuite extends RemoteSparkSession { } test("interrupt all - foreground queries, background interrupt") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ implicit val ec: ExecutionContextExecutor = ExecutionContext.global diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala index a4f1a61cf3997..fa17017297856 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala @@ -23,6 +23,8 @@ import java.util.concurrent.atomic.AtomicLong import scala.collection.JavaConverters._ +import org.apache.commons.lang3.{JavaVersion, SystemUtils} + import org.apache.spark.TaskContext import org.apache.spark.api.java.function._ import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{PrimitiveIntEncoder, PrimitiveLongEncoder} @@ -34,11 +36,15 @@ import org.apache.spark.sql.functions.{col, struct, udf} */ class UserDefinedFunctionE2ETestSuite extends QueryTest { test("Dataset typed filter") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark.range(10).filter(n => n % 2 == 0).collectAsList() assert(rows == Arrays.asList[Long](0, 2, 4, 6, 8)) } test("Dataset typed filter - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark .range(10) .filter(new FilterFunction[JLong] { @@ -49,11 +55,15 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark.range(10).map(n => n / 2)(PrimitiveLongEncoder).collectAsList() assert(rows == Arrays.asList[Long](0, 0, 1, 1, 2, 2, 3, 3, 4, 4)) } test("filter with condition") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This should go via `def filter(condition: Column)` rather than // `def filter(func: T => Boolean)` def func(i: Long): Boolean = i < 5 @@ -63,6 +73,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("filter with col(*)") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This should go via `def filter(condition: Column)` but it is executed as // `def filter(func: T => Boolean)`. This is fine as the result is the same. def func(i: Long): Boolean = i < 5 @@ -72,6 +84,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark .range(10) .map( @@ -84,6 +98,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed flat map") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val rows = spark @@ -95,6 +111,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed flat map - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark .range(5) .flatMap( @@ -108,6 +126,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map partition") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = spark.range(0, 100, 1, 50).repartition(4) @@ -117,6 +137,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map partition - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(0, 100, 1, 50).repartition(4) val result = df .mapPartitions( @@ -131,6 +153,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreach") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val func: JLong => Unit = _ => { throw new RuntimeException("Hello foreach") } @@ -141,6 +165,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreach - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val exception = intercept[Exception] { spark .range(2) @@ -154,6 +180,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreachPartition") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sum = new AtomicLong() val func: Iterator[JLong] => Unit = f => { f.foreach(v => sum.addAndGet(v)) @@ -170,6 +198,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreachPartition - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sum = new AtomicLong() val exception = intercept[Exception] { spark @@ -190,6 +220,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreach: change not visible to client") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sum = new AtomicLong() val func: Iterator[JLong] => Unit = f => { f.foreach(v => sum.addAndGet(v)) @@ -199,24 +231,32 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset reduce without null partition inputs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert(spark.range(0, 10, 1, 5).map(_ + 1).reduce(_ + _) == 55) } test("Dataset reduce with null partition inputs") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert(spark.range(0, 10, 1, 16).map(_ + 1).reduce(_ + _) == 55) } test("Dataset reduce with null partition inputs - java to scala long type") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert(spark.range(0, 5, 1, 10).as[Long].reduce(_ + _) == 10) } test("Dataset reduce with null partition inputs - java") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert( @@ -229,6 +269,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("udf with row input encoder") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = Seq((1, 2, 3)).toDF("a", "b", "c") @@ -237,6 +279,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Filter with row input encoder") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") @@ -245,6 +289,8 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("mapPartitions with row input encoder") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala index 61959234c8790..bac141e1e8ef4 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala @@ -20,6 +20,7 @@ import java.io.{PipedInputStream, PipedOutputStream} import java.util.concurrent.{Executors, Semaphore, TimeUnit} import org.apache.commons.io.output.ByteArrayOutputStream +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.BeforeAndAfterEach import org.apache.spark.sql.connect.client.util.RemoteSparkSession @@ -42,26 +43,28 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } override def beforeAll(): Unit = { - super.beforeAll() - ammoniteOut = new ByteArrayOutputStream() - testSuiteOut = new PipedOutputStream() - // Connect the `testSuiteOut` and `ammoniteIn` pipes - ammoniteIn = new PipedInputStream(testSuiteOut) - errorStream = new ByteArrayOutputStream() - - val args = Array("--port", serverPort.toString) - val task = new Runnable { - override def run(): Unit = { - ConnectRepl.doMain( - args = args, - semaphore = Some(semaphore), - inputStream = ammoniteIn, - outputStream = ammoniteOut, - errorStream = errorStream) + if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) { + super.beforeAll() + ammoniteOut = new ByteArrayOutputStream() + testSuiteOut = new PipedOutputStream() + // Connect the `testSuiteOut` and `ammoniteIn` pipes + ammoniteIn = new PipedInputStream(testSuiteOut) + errorStream = new ByteArrayOutputStream() + + val args = Array("--port", serverPort.toString) + val task = new Runnable { + override def run(): Unit = { + ConnectRepl.doMain( + args = args, + semaphore = Some(semaphore), + inputStream = ammoniteIn, + outputStream = ammoniteOut, + errorStream = errorStream) + } } - } - executorService.submit(task) + executorService.submit(task) + } } override def afterAll(): Unit = { @@ -100,6 +103,8 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("Simple query") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // Run simple query to test REPL val input = """ |spark.sql("select 1").collect() @@ -109,6 +114,8 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("UDF containing 'def'") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = """ |class A(x: Int) { def get = x * 5 + 19 } |def dummyUdf(x: Int): Int = new A(x).get @@ -134,6 +141,8 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("UDF containing in-place lambda") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = """ |class A(x: Int) { def get = x * 42 + 5 } |val myUdf = udf((x: Int) => new A(x).get) @@ -144,6 +153,8 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("SPARK-43198: Filter does not throw ammonite-related class initialization exception") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = """ |spark.range(10).filter(n => n % 2 == 0).collect() """.stripMargin diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index e05828606d098..ab87e78beb175 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -21,6 +21,7 @@ import java.util.concurrent.TimeUnit import scala.io.Source +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.BeforeAndAfterAll import sys.process._ @@ -170,41 +171,44 @@ trait RemoteSparkSession extends ConnectFunSuite with BeforeAndAfterAll { protected lazy val serverPort: Int = port override def beforeAll(): Unit = { - super.beforeAll() - SparkConnectServerUtils.start() - spark = SparkSession - .builder() - .client(SparkConnectClient.builder().port(serverPort).build()) - .create() - - // Retry and wait for the server to start - val stop = System.nanoTime() + TimeUnit.MINUTES.toNanos(1) // ~1 min - var sleepInternalMs = TimeUnit.SECONDS.toMillis(1) // 1s with * 2 backoff - var success = false - val error = new RuntimeException(s"Failed to start the test server on port $serverPort.") - - while (!success && System.nanoTime() < stop) { - try { - // Run a simple query to verify the server is really up and ready - val result = spark - .sql("select val from (values ('Hello'), ('World')) as t(val)") - .collect() - assert(result.length == 2) - success = true - debug("Spark Connect Server is up.") - } catch { - // ignored the error - case e: Throwable => - error.addSuppressed(e) - Thread.sleep(sleepInternalMs) - sleepInternalMs *= 2 + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) { + super.beforeAll() + SparkConnectServerUtils.start() + spark = SparkSession + .builder() + .client(SparkConnectClient.builder().port(serverPort).build()) + .create() + + // Retry and wait for the server to start + val stop = System.nanoTime() + TimeUnit.MINUTES.toNanos(1) // ~1 min + var sleepInternalMs = TimeUnit.SECONDS.toMillis(1) // 1s with * 2 backoff + var success = false + val error = new RuntimeException(s"Failed to start the test server on port $serverPort.") + + while (!success && System.nanoTime() < stop) { + try { + // Run a simple query to verify the server is really up and ready + val result = spark + .sql("select val from (values ('Hello'), ('World')) as t(val)") + .collect() + assert(result.length == 2) + success = true + debug("Spark Connect Server is up.") + } catch { + // ignored the error + case e: Throwable => + error.addSuppressed(e) + Thread.sleep(sleepInternalMs) + sleepInternalMs *= 2 + } } - } - // Throw error if failed - if (!success) { - debug(error) - throw error + // Throw error if failed + if (!success) { + debug(error) + throw error + } } } diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala index cdb6b9a2e9c10..ebe3715c6b582 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.streaming import java.sql.Timestamp +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.concurrent.Eventually.eventually import org.scalatest.concurrent.Futures.timeout import org.scalatest.time.SpanSugar._ @@ -49,6 +50,8 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { Seq(ClickState("a", 2), ClickState("b", 1)) test("flatMapGroupsWithState - streaming") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -90,6 +93,8 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { } test("flatMapGroupsWithState - streaming - with initial state") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -135,6 +140,8 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState - streaming") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -177,6 +184,8 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState - streaming - with initial state") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala index 6ddcedf19cb70..84632738315dd 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ +import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.concurrent.Eventually.eventually import org.scalatest.concurrent.Futures.timeout import org.scalatest.time.SpanSugar._ @@ -35,6 +36,8 @@ import org.apache.spark.util.Utils class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { test("Streaming API with windowed aggregate query") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This verifies standard streaming API by starting a streaming query with windowed count. withSQLConf( "spark.sql.shuffle.partitions" -> "1" // Avoid too many reducers. @@ -111,6 +114,8 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("Streaming table API") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf( "spark.sql.shuffle.partitions" -> "1" // Avoid too many reducers. ) { @@ -147,6 +152,8 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("awaitTermination") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf( "spark.sql.shuffle.partitions" -> "1" // Avoid too many reducers. ) { @@ -172,6 +179,8 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("foreach Row") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val writer = new TestForeachWriter[Row] val df = spark.readStream @@ -191,6 +200,8 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("foreach Int") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -216,6 +227,8 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("foreach Custom class") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -240,6 +253,8 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("streaming query manager") { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.streams.active.isEmpty) val q = spark.readStream .format("rate") From 6a1fbd58d7a30028a028bad58217b95daeaab90e Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 30 Jun 2023 12:39:00 +0800 Subject: [PATCH 2/4] override test function in RemoteSparkSession --- .../org/apache/spark/sql/CatalogSuite.scala | 19 --- .../apache/spark/sql/ClientE2ETestSuite.scala | 116 ------------------ .../spark/sql/DataFrameNaFunctionSuite.scala | 32 ----- .../apache/spark/sql/DataFrameStatSuite.scala | 15 --- .../KeyValueGroupedDatasetE2ETestSuite.scala | 79 ------------ .../spark/sql/SparkSessionE2ESuite.scala | 5 - .../sql/UserDefinedFunctionE2ETestSuite.scala | 46 ------- .../spark/sql/application/ReplE2ESuite.scala | 9 +- .../client/util/RemoteSparkSession.scala | 18 ++- ...FlatMapGroupsWithStateStreamingSuite.scala | 9 -- .../sql/streaming/StreamingQuerySuite.scala | 15 --- 11 files changed, 17 insertions(+), 346 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala index 42609df8e9649..04b3f4e639a06 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/CatalogSuite.scala @@ -21,7 +21,6 @@ import java.io.{File, FilenameFilter} import io.grpc.StatusRuntimeException import org.apache.commons.io.FileUtils -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.apache.spark.sql.connect.client.util.RemoteSparkSession import org.apache.spark.sql.types.{DoubleType, LongType, StructType} @@ -30,8 +29,6 @@ import org.apache.spark.storage.StorageLevel class CatalogSuite extends RemoteSparkSession with SQLHelper { test("Database APIs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val currentDb = spark.catalog.currentDatabase assert(currentDb == "default") withTempDatabase { db => @@ -63,8 +60,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("CatalogMetadata APIs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val currentCatalog = spark.catalog.currentCatalog() assert(currentCatalog == "spark_catalog") try { @@ -92,8 +87,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("Table APIs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.catalog.listTables().collect().isEmpty) val parquetTableName = "parquet_table" val orcTableName = "orc_table" @@ -157,8 +150,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("Cache Table APIs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val parquetTableName = "parquet_table" withTable(parquetTableName) { withTempPath { table1Dir => @@ -184,8 +175,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("TempView APIs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val viewName = "view1" val globalViewName = "g_view1" try { @@ -208,8 +197,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("Function API") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val dbName = spark.catalog.currentDatabase val functions1 = spark.catalog.listFunctions().collect() assert(functions1.nonEmpty) @@ -234,8 +221,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("recoverPartitions") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val tableName = "test" withTable(tableName) { withTempPath { dir => @@ -260,8 +245,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("refreshTable") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTempPath { dir => val tableName = "spark_catalog.default.my_table" withTable(tableName) { @@ -287,8 +270,6 @@ class CatalogSuite extends RemoteSparkSession with SQLHelper { } test("refreshByPath") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTempPath { dir => val tableName = "spark_catalog.default.my_table" withTable(tableName) { diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala index c30c45f7ecd4c..0ababaa0af1a6 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala @@ -46,8 +46,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM // Spark Result test("spark result schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.sql("select val from (values ('Hello'), ('World')) as t(val)") df.withResult { result => val schema = result.schema @@ -56,8 +54,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("spark result array") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.sql("select val from (values ('Hello'), ('World')) as t(val)") val result = df.collect() assert(result.length == 2) @@ -66,8 +62,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("eager execution of sql") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assume(IntegrationTestUtils.isSparkHiveJarAvailable) withTable("test_martin") { // Fails, because table does not exist. @@ -86,8 +80,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("simple dataset") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10).limit(3) val result = df.collect() assert(result.length == 3) @@ -97,8 +89,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("read and write") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val testDataPath = java.nio.file.Paths .get( IntegrationTestUtils.sparkHome, @@ -139,8 +129,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("read path collision") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val testDataPath = java.nio.file.Paths .get( IntegrationTestUtils.sparkHome, @@ -171,8 +159,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("textFile") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val testDataPath = java.nio.file.Paths .get( IntegrationTestUtils.sparkHome, @@ -193,8 +179,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("write table") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("myTable") { val df = spark.range(10).limit(3) df.write.mode(SaveMode.Overwrite).saveAsTable("myTable") @@ -210,8 +194,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("different spark session join/union") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10).limit(3) val spark2 = SparkSession @@ -240,15 +222,11 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("write without table or path") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // Should receive no error to write noop spark.range(10).write.format("noop").mode("append").save() } test("write jdbc") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assume(IntegrationTestUtils.isSparkHiveJarAvailable) if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) { val url = "jdbc:derby:memory:1234" @@ -267,8 +245,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("writeTo with create") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")) @@ -283,8 +259,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("writeTo with create and using") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")) @@ -305,8 +279,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("writeTo with create and append") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c")) @@ -324,8 +296,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("WriteTo with overwrite") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows1 = (1L to 3L).map { i => @@ -357,8 +327,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("WriteTo with overwritePartitions") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withTable("testcat.myTableV2") { val rows = (4L to 7L).map { i => @@ -383,8 +351,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("write path collision") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) val outputFolderPath = Files.createTempDirectory("output").toAbsolutePath // Failed because the path cannot be provided both via option and save method. @@ -431,8 +397,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM // Dataset tests test("Dataset inspection") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) val local = spark.newDataFrame { builder => builder.getLocalRelationBuilder.setSchema(simpleSchema.catalogString) @@ -445,8 +409,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) assert(df.schema === simpleSchema) assert(df.dtypes === Array(("value", "LongType"))) @@ -456,8 +418,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataframe schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.sql("select * from range(10)") val expectedSchema = new StructType().add("id", "long", nullable = false) assert(df.schema === expectedSchema) @@ -468,8 +428,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset explain") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(10) val simpleExplainFragments = Seq("== Physical Plan ==") testCapturedStdOut(df.explain(), simpleExplainFragments: _*) @@ -490,8 +448,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset result collection") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) def checkResult(rows: TraversableOnce[java.lang.Long], expectedValues: Long*): Unit = { rows.toIterator.zipAll(expectedValues.iterator, null, null).foreach { case (actual, expected) => assert(actual === expected) @@ -517,8 +473,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset show") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(20) testCapturedStdOut(df.show(), 24, 5, "+---+", "| id|", "| 0|", "| 19|") testCapturedStdOut( @@ -564,8 +518,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset randomSplit") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) implicit val tolerance = TolerantNumerics.tolerantDoubleEquality(0.01) val df = spark.range(100) @@ -594,14 +546,10 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset count") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.range(10).count() === 10) } test("Dataset collect tuple") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val result = session @@ -644,8 +592,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset collect complex type") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val result = session @@ -657,23 +603,17 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset typed select - simple column") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val numRows = spark.range(1000).select(count("id")).first() assert(numRows === 1000) } test("Dataset typed select - multiple columns") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val result = spark.range(1000).select(count("id"), sum("id")).first() assert(result.getLong(0) === 1000) assert(result.getLong(1) === 499500) } test("Dataset typed select - complex column") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val ds = session @@ -683,8 +623,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset typed select - multiple complex columns") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val s = struct(generateMyTypeColumns: _*).as[MyType] @@ -695,8 +633,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("lambda functions") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This test is mostly to validate lambda variables are properly resolved. val result = spark .range(3) @@ -713,8 +649,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("shuffle array") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // We cannot do structural tests for shuffle because its random seed will always change. val result = spark .sql("select 1") @@ -725,8 +659,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("ambiguous joins") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val left = spark.range(100).select(col("id"), rand(10).as("a")) val right = spark.range(100).select(col("id"), rand(12).as("a")) val joined = left.join(right, left("id") === right("id")).select(left("id"), right("a")) @@ -739,8 +671,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("broadcast join") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf("spark.sql.autoBroadcastJoinThreshold" -> "-1") { val left = spark.range(100).select(col("id"), rand(10).as("a")) val right = spark.range(100).select(col("id"), rand(12).as("a")) @@ -752,8 +682,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("test temp view") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) try { spark.range(100).createTempView("test1") assert(spark.sql("SELECT * FROM test1").count() == 100) @@ -770,15 +698,11 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("time") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val timeFragments = Seq("Time taken: ", " ms") testCapturedStdOut(spark.time(spark.sql("select 1").collect()), timeFragments: _*) } test("RuntimeConfig") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) intercept[NoSuchElementException](spark.conf.get("foo.bar")) assert(spark.conf.getOption("foo.bar").isEmpty) spark.conf.set("foo.bar", value = true) @@ -798,8 +722,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkVersion") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.version.nonEmpty) assert(spark.version == SPARK_VERSION) } @@ -811,8 +733,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Local Relation implicit conversion") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -828,16 +748,12 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession.createDataFrame - row") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = java.util.Arrays.asList(Row("bob", 99), Row("Club", 5), Row("Bag", 5)) val schema = new StructType().add("key", "string").add("value", "int") checkSameResult(rows.asScala, spark.createDataFrame(rows, schema)) } test("SparkSession.createDataFrame - bean") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) def bean(v: String): SimpleBean = { val bean = new SimpleBean bean.setValue(v) @@ -850,8 +766,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession typed createDataSet/createDataframe") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val list = java.util.Arrays.asList(KV("bob", 99), KV("Club", 5), KV("Bag", 5)) @@ -862,16 +776,12 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession newSession") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val oldId = spark.sql("SELECT 1").analyze.getSessionId val newId = spark.newSession().sql("SELECT 1").analyze.getSessionId assert(oldId != newId) } test("createDataFrame from complex type schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val schema = new StructType() .add( "c1", @@ -884,16 +794,12 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SameSemantics") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val plan = spark.sql("select 1") val otherPlan = spark.sql("select 1") assert(plan.sameSemantics(otherPlan)) } test("sameSemantics and semanticHash") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df1 = spark.createDataFrame(Seq((1, 2), (4, 5))) val df2 = spark.createDataFrame(Seq((1, 2), (4, 5))) val df3 = spark.createDataFrame(Seq((0, 2), (4, 5))) @@ -909,8 +815,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("toJSON") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val expected = Array( """{"b":0.0,"id":0,"d":"world","a":0}""", """{"b":0.1,"id":1,"d":"world","a":1}""", @@ -924,8 +828,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("json from Dataset[String] inferSchema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val expected = Seq( @@ -938,8 +840,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("json from Dataset[String] with schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val schema = new StructType().add("city", StringType).add("name", StringType) @@ -950,8 +850,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("json from Dataset[String] with invalid schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val message = intercept[ParseException] { spark.read.schema("123").json(spark.createDataset(Seq.empty[String])(StringEncoder)) }.getMessage @@ -959,8 +857,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("csv from Dataset[String] inferSchema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val expected = Seq( @@ -976,8 +872,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("csv from Dataset[String] with schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val schema = new StructType().add("name", StringType).add("age", LongType) @@ -988,8 +882,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("csv from Dataset[String] with invalid schema") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val message = intercept[ParseException] { spark.read.schema("123").csv(spark.createDataset(Seq.empty[String])(StringEncoder)) }.getMessage @@ -997,8 +889,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("Dataset result destructive iterator") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // Helper methods for accessing private field `idxToBatches` from SparkResult val _idxToBatches = PrivateMethod[mutable.Map[Int, ColumnarBatch]](Symbol("idxToBatches")) @@ -1044,8 +934,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("SparkSession.createDataFrame - large data set") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val threshold = 1024 * 1024 withSQLConf(SQLConf.LOCAL_RELATION_CACHE_THRESHOLD.key -> threshold.toString) { val count = 2 @@ -1061,8 +949,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("sql() with positional parameters") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val result0 = spark.sql("select 1", Array.empty).collect() assert(result0.length == 1 && result0(0).getInt(0) === 1) @@ -1076,8 +962,6 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM } test("sql() with named parameters") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val result0 = spark.sql("select 1", Map.empty[String, Any]).collect() assert(result0.length == 1 && result0(0).getInt(0) === 1) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala index bd126759e4ef1..1f6ea879248dc 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionSuite.scala @@ -19,8 +19,6 @@ package org.apache.spark.sql import scala.collection.JavaConverters._ -import org.apache.commons.lang3.{JavaVersion, SystemUtils} - import org.apache.spark.sql.connect.client.util.QueryTest import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StringType, StructType} @@ -70,8 +68,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("drop") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val rows = input.collect() @@ -99,8 +95,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("drop with how") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val rows = input.collect() @@ -118,8 +112,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("drop with threshold") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val rows = input.collect() @@ -132,8 +124,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sparkSession = spark import sparkSession.implicits._ @@ -235,8 +225,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill with map") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sparkSession = spark import sparkSession.implicits._ @@ -283,16 +271,12 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill with col(*)") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDF() // If columns are specified with "*", they are ignored. checkAnswer(df.na.fill("new name", Seq("*")), df.collect()) } test("drop with col(*)") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDF() val ex = intercept[RuntimeException] { df.na.drop("any", Seq("*")).collect() @@ -301,15 +285,11 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("fill with nested columns") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDFWithNestedColumns checkAnswer(df.na.fill("a1", Seq("c1.c1-1")), df) } test("drop with nested columns") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = createDFWithNestedColumns // Rows with the specified nested columns whose null values are dropped. @@ -318,8 +298,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = createDF() val result1 = input.na @@ -357,8 +335,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace with null") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = spark.sql( "select name, height, married from (values " + "('Bob', 176.5, true), " + @@ -397,8 +373,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace nan with float") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) checkAnswer( createNaNDF().na.replace("*", Map(Float.NaN -> 10.0f)), Row(1, 1L, 1.toShort, 1.toByte, 1.0f, 1.0) :: @@ -406,8 +380,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace nan with double") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) checkAnswer( createNaNDF().na.replace("*", Map(Double.NaN -> 10.0)), Row(1, 1L, 1.toShort, 1.toByte, 1.0f, 1.0) :: @@ -415,8 +387,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace float with nan") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { checkAnswer( createNaNDF().na.replace("*", Map(1.0f -> Float.NaN)), @@ -426,8 +396,6 @@ class DataFrameNaFunctionSuite extends QueryTest with SQLHelper { } test("replace double with nan") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { checkAnswer( createNaNDF().na.replace("*", Map(1.0 -> Double.NaN)), diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala index 4e23b701a6e78..aea31005f3bd6 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql import java.util.Random import io.grpc.StatusRuntimeException -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.matchers.must.Matchers._ import org.apache.spark.sql.connect.client.util.RemoteSparkSession @@ -29,8 +28,6 @@ class DataFrameStatSuite extends RemoteSparkSession { private def toLetter(i: Int): String = (i + 97).toChar.toString test("approxQuantile") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -82,8 +79,6 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("covariance") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -101,8 +96,6 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("correlation") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -117,8 +110,6 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("crosstab") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -141,8 +132,6 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("freqItems") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ @@ -162,8 +151,6 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("sampleBy") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ val df = Seq("Bob", "Alice", "Nico", "Bob", "Alice").toDF("name") @@ -177,8 +164,6 @@ class DataFrameStatSuite extends RemoteSparkSession { } test("countMinSketch") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(1000) val sketch1 = df.stat.countMinSketch("id", depth = 10, width = 20, seed = 42) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala index e1c83063b9fb9..79bedabd559b6 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala @@ -20,7 +20,6 @@ import java.sql.Timestamp import java.util.Arrays import io.grpc.StatusRuntimeException -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Append import org.apache.spark.sql.connect.client.util.QueryTest @@ -41,8 +40,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { import session.implicits._ test("mapGroups") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val values = spark @@ -54,8 +51,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatGroupMap") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -65,8 +60,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keys") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -76,8 +69,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keyAs - keys") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // It is okay to cast from Long to Double, but not Long to Int. val values = spark .range(10) @@ -89,8 +80,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupByKey, keyAs - duplicates") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val result = spark @@ -104,8 +93,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupByKey, keyAs, keys - duplicates") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val result = spark @@ -119,8 +106,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keyAs - flatGroupMap") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -131,8 +116,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("keyAs mapValues - cogroup") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -164,8 +147,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapValues - flatGroupMap") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -176,8 +157,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapValues - keys") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val values = spark .range(10) .groupByKey(v => v % 2) @@ -188,8 +167,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatMapSortedGroups") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -213,8 +190,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("cogroup") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -231,8 +206,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("cogroupSorted") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val grouped = spark .range(10) .groupByKey(v => v % 2) @@ -260,8 +233,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("agg, keyAs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = spark .range(10) .groupByKey(v => v % 2) @@ -272,8 +243,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() @@ -286,8 +255,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -298,8 +265,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -310,8 +275,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -327,8 +290,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -345,8 +306,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -364,8 +323,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr, expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -384,8 +341,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("typed aggregation: expr, expr, expr, expr, expr, expr, expr, expr") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDS() checkDatasetUnorderly( @@ -405,8 +360,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: Enable top-level Option of Product encoders") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val data = Seq(Some((1, "a")), Some((2, "b")), None) val ds = data.toDS() @@ -441,8 +394,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: Resolving Option[Product] field") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq((1, ("a", 1.0)), (2, ("b", 2.0)), (3, null)) .toDS() .as[(Int, Option[(String, Double)])] @@ -450,8 +401,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: select Option[Product] field") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 1), ("b", 2), ("c", 3)).toDS() val ds1 = ds.select(expr("struct(_2, _2 + 1)").as[Option[(Int, Int)]]) checkDataset(ds1, Some((1, 2)), Some((2, 3)), Some((3, 4))) @@ -461,8 +410,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-24762: typed agg on Option[Product] type") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(Some((1, 2)), Some((2, 3)), Some((1, 3))).toDS() assert(ds.groupByKey(_.get._1).count().collect() === Seq((1, 2), (2, 1))) @@ -472,8 +419,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-25942: typed aggregation on primitive type") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(1, 2, 3).toDS() val agg = ds @@ -483,16 +428,12 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("SPARK-25942: typed aggregation on product type") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq((1, 2), (2, 3), (3, 4)).toDS() val agg = ds.groupByKey(x => x).agg(sum("_1").as[Long], sum($"_2" + 1).as[Long]) checkDatasetUnorderly(agg, ((1, 2), 1L, 3L), ((2, 3), 2L, 4L), ((3, 4), 3L, 5L)) } test("SPARK-26085: fix key attribute name for atomic type for typed aggregation") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // TODO(SPARK-43416): Recursively rename the position based tuple to the schema name from the // server. val ds = Seq(1, 2, 3).toDS() @@ -505,8 +446,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("reduceGroups") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq("abc", "xyz", "hello").toDS() checkDatasetUnorderly( ds.groupByKey(_.length).reduceGroups(_ + _), @@ -515,8 +454,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupby") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 1, 10), ("a", 2, 20), ("b", 2, 1), ("b", 1, 2), ("c", 1, 1)) .toDF("key", "seq", "value") val grouped = ds.groupBy($"key").as[String, (String, Int, Int)] @@ -536,8 +473,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("groupby - keyAs, keys") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 1, 10), ("a", 2, 20), ("b", 2, 1), ("b", 1, 2), ("c", 1, 1)) .toDF("key", "seq", "value") val grouped = ds.groupBy($"value").as[String, (String, Int, Int)] @@ -547,8 +482,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatMapGroupsWithState") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { if (state.exists) throw new IllegalArgumentException("state.exists should be false") @@ -572,8 +505,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("flatMapGroupsWithState - with initial state") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { val currState = state.getOption.getOrElse(ClickState(key, 0)) @@ -601,8 +532,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { if (state.exists) throw new IllegalArgumentException("state.exists should be false") @@ -626,8 +555,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState - with initial state") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { val currState = state.getOption.getOrElse(ClickState(key, 0)) @@ -655,8 +582,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("RowEncoder in udf") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") checkDatasetUnorderly( @@ -667,8 +592,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("mapGroups with row encoder") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") checkDataset( @@ -683,8 +606,6 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with SQLHelper { } test("coGroup with row encoder") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df1 = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") val df2 = Seq(("x", 10), ("x", 20), ("y", 1), ("y", 2), ("a", 1)).toDF("c1", "c2") diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala index 62a64f4fc22af..fb295c00edc02 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/SparkSessionE2ESuite.scala @@ -20,7 +20,6 @@ import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} import scala.concurrent.duration._ import scala.util.{Failure, Success} -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.concurrent.Eventually._ import org.apache.spark.sql.connect.client.util.RemoteSparkSession @@ -34,8 +33,6 @@ import org.apache.spark.util.ThreadUtils class SparkSessionE2ESuite extends RemoteSparkSession { test("interrupt all - background queries, foreground interrupt") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ implicit val ec: ExecutionContextExecutor = ExecutionContext.global @@ -76,8 +73,6 @@ class SparkSessionE2ESuite extends RemoteSparkSession { } test("interrupt all - foreground queries, background interrupt") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session = spark import session.implicits._ implicit val ec: ExecutionContextExecutor = ExecutionContext.global diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala index fa17017297856..a4f1a61cf3997 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala @@ -23,8 +23,6 @@ import java.util.concurrent.atomic.AtomicLong import scala.collection.JavaConverters._ -import org.apache.commons.lang3.{JavaVersion, SystemUtils} - import org.apache.spark.TaskContext import org.apache.spark.api.java.function._ import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{PrimitiveIntEncoder, PrimitiveLongEncoder} @@ -36,15 +34,11 @@ import org.apache.spark.sql.functions.{col, struct, udf} */ class UserDefinedFunctionE2ETestSuite extends QueryTest { test("Dataset typed filter") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark.range(10).filter(n => n % 2 == 0).collectAsList() assert(rows == Arrays.asList[Long](0, 2, 4, 6, 8)) } test("Dataset typed filter - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark .range(10) .filter(new FilterFunction[JLong] { @@ -55,15 +49,11 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark.range(10).map(n => n / 2)(PrimitiveLongEncoder).collectAsList() assert(rows == Arrays.asList[Long](0, 0, 1, 1, 2, 2, 3, 3, 4, 4)) } test("filter with condition") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This should go via `def filter(condition: Column)` rather than // `def filter(func: T => Boolean)` def func(i: Long): Boolean = i < 5 @@ -73,8 +63,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("filter with col(*)") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This should go via `def filter(condition: Column)` but it is executed as // `def filter(func: T => Boolean)`. This is fine as the result is the same. def func(i: Long): Boolean = i < 5 @@ -84,8 +72,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark .range(10) .map( @@ -98,8 +84,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed flat map") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val rows = spark @@ -111,8 +95,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed flat map - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val rows = spark .range(5) .flatMap( @@ -126,8 +108,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map partition") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = spark.range(0, 100, 1, 50).repartition(4) @@ -137,8 +117,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset typed map partition - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val df = spark.range(0, 100, 1, 50).repartition(4) val result = df .mapPartitions( @@ -153,8 +131,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreach") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val func: JLong => Unit = _ => { throw new RuntimeException("Hello foreach") } @@ -165,8 +141,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreach - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val exception = intercept[Exception] { spark .range(2) @@ -180,8 +154,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreachPartition") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sum = new AtomicLong() val func: Iterator[JLong] => Unit = f => { f.foreach(v => sum.addAndGet(v)) @@ -198,8 +170,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreachPartition - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sum = new AtomicLong() val exception = intercept[Exception] { spark @@ -220,8 +190,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset foreach: change not visible to client") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val sum = new AtomicLong() val func: Iterator[JLong] => Unit = f => { f.foreach(v => sum.addAndGet(v)) @@ -231,32 +199,24 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Dataset reduce without null partition inputs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert(spark.range(0, 10, 1, 5).map(_ + 1).reduce(_ + _) == 55) } test("Dataset reduce with null partition inputs") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert(spark.range(0, 10, 1, 16).map(_ + 1).reduce(_ + _) == 55) } test("Dataset reduce with null partition inputs - java to scala long type") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert(spark.range(0, 5, 1, 10).as[Long].reduce(_ + _) == 10) } test("Dataset reduce with null partition inputs - java") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ assert( @@ -269,8 +229,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("udf with row input encoder") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = Seq((1, 2, 3)).toDF("a", "b", "c") @@ -279,8 +237,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("Filter with row input encoder") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") @@ -289,8 +245,6 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest { } test("mapPartitions with row input encoder") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala index bac141e1e8ef4..7501a90415467 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala @@ -43,6 +43,7 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } override def beforeAll(): Unit = { + // TODO(SPARK-44121) Remove this check condition if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) { super.beforeAll() ammoniteOut = new ByteArrayOutputStream() @@ -103,8 +104,6 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("Simple query") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // Run simple query to test REPL val input = """ |spark.sql("select 1").collect() @@ -114,8 +113,6 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("UDF containing 'def'") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = """ |class A(x: Int) { def get = x * 5 + 19 } |def dummyUdf(x: Int): Int = new A(x).get @@ -141,8 +138,6 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("UDF containing in-place lambda") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = """ |class A(x: Int) { def get = x * 42 + 5 } |val myUdf = udf((x: Int) => new A(x).get) @@ -153,8 +148,6 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } test("SPARK-43198: Filter does not throw ammonite-related class initialization exception") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val input = """ |spark.range(10).filter(n => n % 2 == 0).collect() """.stripMargin diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index ab87e78beb175..2796288843ba1 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -22,7 +22,8 @@ import java.util.concurrent.TimeUnit import scala.io.Source import org.apache.commons.lang3.{JavaVersion, SystemUtils} -import org.scalatest.BeforeAndAfterAll +import org.scalactic.source.Position +import org.scalatest.{BeforeAndAfterAll, Tag} import sys.process._ import org.apache.spark.sql.SparkSession @@ -171,7 +172,7 @@ trait RemoteSparkSession extends ConnectFunSuite with BeforeAndAfterAll { protected lazy val serverPort: Int = port override def beforeAll(): Unit = { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + // TODO(SPARK-44121) Remove this check condition if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) { super.beforeAll() SparkConnectServerUtils.start() @@ -221,4 +222,17 @@ trait RemoteSparkSession extends ConnectFunSuite with BeforeAndAfterAll { spark = null super.afterAll() } + + /** + * SPARK-44259: override test function to skip `RemoteSparkSession-based` tests as default, + * we should delete this function after SPARK-44121 is completed. + */ + override protected def test(testName: String, testTags: Tag*)(testFun: => Any) + (implicit pos: Position): Unit = { + super.test(testName, testTags: _*) { + // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 + assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) + testFun + } + } } diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala index ebe3715c6b582..cdb6b9a2e9c10 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.streaming import java.sql.Timestamp -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.concurrent.Eventually.eventually import org.scalatest.concurrent.Futures.timeout import org.scalatest.time.SpanSugar._ @@ -50,8 +49,6 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { Seq(ClickState("a", 2), ClickState("b", 1)) test("flatMapGroupsWithState - streaming") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -93,8 +90,6 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { } test("flatMapGroupsWithState - streaming - with initial state") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -140,8 +135,6 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState - streaming") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -184,8 +177,6 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with SQLHelper { } test("mapGroupsWithState - streaming - with initial state") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala index 84632738315dd..6ddcedf19cb70 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala @@ -22,7 +22,6 @@ import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.concurrent.Eventually.eventually import org.scalatest.concurrent.Futures.timeout import org.scalatest.time.SpanSugar._ @@ -36,8 +35,6 @@ import org.apache.spark.util.Utils class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { test("Streaming API with windowed aggregate query") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) // This verifies standard streaming API by starting a streaming query with windowed count. withSQLConf( "spark.sql.shuffle.partitions" -> "1" // Avoid too many reducers. @@ -114,8 +111,6 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("Streaming table API") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf( "spark.sql.shuffle.partitions" -> "1" // Avoid too many reducers. ) { @@ -152,8 +147,6 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("awaitTermination") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) withSQLConf( "spark.sql.shuffle.partitions" -> "1" // Avoid too many reducers. ) { @@ -179,8 +172,6 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("foreach Row") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val writer = new TestForeachWriter[Row] val df = spark.readStream @@ -200,8 +191,6 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("foreach Int") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -227,8 +216,6 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("foreach Custom class") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) val session: SparkSession = spark import session.implicits._ @@ -253,8 +240,6 @@ class StreamingQuerySuite extends RemoteSparkSession with SQLHelper { } test("streaming query manager") { - // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 - assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) assert(spark.streams.active.isEmpty) val q = spark.readStream .format("rate") From 3600238bd1e2e2110744c532a1e7ed4347887d97 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 30 Jun 2023 12:51:27 +0800 Subject: [PATCH 3/4] revert change of ReplE2ESuite --- .../spark/sql/application/ReplE2ESuite.scala | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala index 7501a90415467..61959234c8790 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala @@ -20,7 +20,6 @@ import java.io.{PipedInputStream, PipedOutputStream} import java.util.concurrent.{Executors, Semaphore, TimeUnit} import org.apache.commons.io.output.ByteArrayOutputStream -import org.apache.commons.lang3.{JavaVersion, SystemUtils} import org.scalatest.BeforeAndAfterEach import org.apache.spark.sql.connect.client.util.RemoteSparkSession @@ -43,29 +42,26 @@ class ReplE2ESuite extends RemoteSparkSession with BeforeAndAfterEach { } override def beforeAll(): Unit = { - // TODO(SPARK-44121) Remove this check condition - if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17)) { - super.beforeAll() - ammoniteOut = new ByteArrayOutputStream() - testSuiteOut = new PipedOutputStream() - // Connect the `testSuiteOut` and `ammoniteIn` pipes - ammoniteIn = new PipedInputStream(testSuiteOut) - errorStream = new ByteArrayOutputStream() - - val args = Array("--port", serverPort.toString) - val task = new Runnable { - override def run(): Unit = { - ConnectRepl.doMain( - args = args, - semaphore = Some(semaphore), - inputStream = ammoniteIn, - outputStream = ammoniteOut, - errorStream = errorStream) - } + super.beforeAll() + ammoniteOut = new ByteArrayOutputStream() + testSuiteOut = new PipedOutputStream() + // Connect the `testSuiteOut` and `ammoniteIn` pipes + ammoniteIn = new PipedInputStream(testSuiteOut) + errorStream = new ByteArrayOutputStream() + + val args = Array("--port", serverPort.toString) + val task = new Runnable { + override def run(): Unit = { + ConnectRepl.doMain( + args = args, + semaphore = Some(semaphore), + inputStream = ammoniteIn, + outputStream = ammoniteOut, + errorStream = errorStream) } - - executorService.submit(task) } + + executorService.submit(task) } override def afterAll(): Unit = { From bcce9d2554cd56fb066e31e0197880ded883c2af Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 30 Jun 2023 14:29:16 +0800 Subject: [PATCH 4/4] format code --- .../sql/connect/client/util/RemoteSparkSession.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index 2796288843ba1..8d84dffc9d5bd 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -224,11 +224,11 @@ trait RemoteSparkSession extends ConnectFunSuite with BeforeAndAfterAll { } /** - * SPARK-44259: override test function to skip `RemoteSparkSession-based` tests as default, - * we should delete this function after SPARK-44121 is completed. + * SPARK-44259: override test function to skip `RemoteSparkSession-based` tests as default, we + * should delete this function after SPARK-44121 is completed. */ - override protected def test(testName: String, testTags: Tag*)(testFun: => Any) - (implicit pos: Position): Unit = { + override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { super.test(testName, testTags: _*) { // TODO(SPARK-44121) Re-enable Arrow-based connect tests in Java 21 assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_17))