From 7057952f6bc2c5cf97dd408effd1b18bee1cb8f4 Mon Sep 17 00:00:00 2001 From: dengziming Date: Thu, 19 Oct 2023 11:14:55 +0500 Subject: [PATCH] [SPARK-45569][SQL] Assign name to the error _LEGACY_ERROR_TEMP_2153 ### What changes were proposed in this pull request? Assign the name `UNEXPECTED_SERIALIZER_FOR_CLASS` to the legacy error class `_LEGACY_ERROR_TEMP_2153`. ### Why are the changes needed? To assign proper name as a part of activity in SPARK-37935. ### Does this PR introduce _any_ user-facing change? Yes, the error message will include the error class name ### How was this patch tested? Add a unit test to produce the error from user code. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43414 from dengziming/SPARK-45573. Authored-by: dengziming Signed-off-by: Max Gekk --- .../main/resources/error/error-classes.json | 11 +++---- docs/sql-error-conditions.md | 6 ++++ .../sql/errors/QueryExecutionErrors.scala | 6 ++-- .../encoders/ExpressionEncoderSuite.scala | 29 ++++++++++++++++--- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json index 9e442f4c66634..dc65d7347fece 100644 --- a/common/utils/src/main/resources/error/error-classes.json +++ b/common/utils/src/main/resources/error/error-classes.json @@ -3017,6 +3017,12 @@ ], "sqlState" : "4274K" }, + "UNEXPECTED_SERIALIZER_FOR_CLASS" : { + "message" : [ + "The class has an unexpected expression serializer. Expects \"STRUCT\" or \"IF\" which returns \"STRUCT\" but found ." + ], + "sqlState" : "42846" + }, "UNKNOWN_PROTOBUF_MESSAGE_TYPE" : { "message" : [ "Attempting to treat as a Message, but it was ." @@ -5718,11 +5724,6 @@ "." ] }, - "_LEGACY_ERROR_TEMP_2153" : { - "message" : [ - "class has unexpected serializer: ." - ] - }, "_LEGACY_ERROR_TEMP_2154" : { "message" : [ "Failed to get outer pointer for ." diff --git a/docs/sql-error-conditions.md b/docs/sql-error-conditions.md index d8f5193c9bc57..e555ae7476b33 100644 --- a/docs/sql-error-conditions.md +++ b/docs/sql-error-conditions.md @@ -1941,6 +1941,12 @@ Parameter `` of function `` requires the `` because it contains positional argument(s) following the named argument assigned to ``; please rearrange them so the positional arguments come first and then retry the query again. +### UNEXPECTED_SERIALIZER_FOR_CLASS + +[SQLSTATE: 42846](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation) + +The class `` has an unexpected expression serializer. Expects "STRUCT" or "IF" which returns "STRUCT" but found ``. + ### UNKNOWN_PROTOBUF_MESSAGE_TYPE [SQLSTATE: 42K0G](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 8d7819de052f1..957e87b4d3f5b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1362,10 +1362,10 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def classHasUnexpectedSerializerError( clsName: String, objSerializer: Expression): SparkRuntimeException = { new SparkRuntimeException( - errorClass = "_LEGACY_ERROR_TEMP_2153", + errorClass = "UNEXPECTED_SERIALIZER_FOR_CLASS", messageParameters = Map( - "clsName" -> clsName, - "objSerializer" -> objSerializer.toString())) + "className" -> clsName, + "expr" -> toSQLExpr(objSerializer))) } def unsupportedOperandTypeForSizeFunctionError( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala index 9d2051b01d62e..a3ea3a462b1c5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala @@ -25,19 +25,20 @@ import scala.collection.mutable.ArrayBuffer import scala.reflect.runtime.universe.TypeTag import org.apache.spark.{SPARK_DOC_ROOT, SparkArithmeticException, SparkRuntimeException, SparkUnsupportedOperationException} -import org.apache.spark.sql.{Encoder, Encoders} +import org.apache.spark.sql.{Encoder, Encoders, Row} import org.apache.spark.sql.catalyst.{FooClassWithEnum, FooEnum, OptionalData, PrimitiveData, ScroogeLikeExample} import org.apache.spark.sql.catalyst.analysis.AnalysisTest import org.apache.spark.sql.catalyst.dsl.plans._ -import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, NaNvl} import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.ArrayData +import org.apache.spark.sql.errors.QueryErrorsBase import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} -import org.apache.spark.util.ClosureCleaner +import org.apache.spark.util.{ClosureCleaner, Utils} case class RepeatedStruct(s: Seq[PrimitiveData]) @@ -138,7 +139,8 @@ case class OptionNestedGeneric[T](list: Option[T]) case class MapNestedGenericKey[T](list: Map[T, Int]) case class MapNestedGenericValue[T](list: Map[Int, T]) -class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTest { +class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTest + with QueryErrorsBase { OuterScopes.addOuterScope(this) implicit def encoder[T : TypeTag]: ExpressionEncoder[T] = verifyNotLeakingReflectionObjects { @@ -576,6 +578,25 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes assert(e.getMessage.contains("tuple with more than 22 elements are not supported")) } + test("throw exception for unexpected serializer") { + val schema = new StructType() + .add("key", StringType) + .add("value", BinaryType) + + val encoder = ExpressionEncoder(schema, lenient = true) + val unexpectedSerializer = NaNvl(encoder.objSerializer, encoder.objSerializer) + val exception = intercept[org.apache.spark.SparkRuntimeException] { + new ExpressionEncoder[Row](unexpectedSerializer, encoder.objDeserializer, encoder.clsTag) + } + checkError( + exception = exception, + errorClass = "UNEXPECTED_SERIALIZER_FOR_CLASS", + parameters = Map( + "className" -> Utils.getSimpleName(encoder.clsTag.runtimeClass), + "expr" -> toSQLExpr(unexpectedSerializer)) + ) + } + encodeDecodeTest((1, FooEnum.E1), "Tuple with Int and scala Enum") encodeDecodeTest((null, FooEnum.E1, FooEnum.E2), "Tuple with Null and scala Enum") encodeDecodeTest(Seq(FooEnum.E1, null), "Seq with scala Enum")