[BUG] generate_expr_test FAILED on Dataproc Cluster

[NvTimLiu]

18:58:05 ----------------------------- Captured stdout call -----------------------------
18:58:05 ### CPU RUN ###
18:58:05 ### GPU RUN ###
18:58:05 �[31m�[1m_______________ test_posexplode_nested_array_data[Decimal(12,2)] _______________�[0m
18:58:05
18:58:05 spark_tmp_path = '/tmp/pyspark_tests//653163/'
18:58:05 data_gen = [Integer, Array(Array(Decimal(12,2)))]
18:58:05
18:58:05 �[37m@ignore_order�[39;49;00m(local=�[94mTrue�[39;49;00m)
18:58:05 �[37m@pytest�[39;49;00m.mark.parametrize(�[33m'�[39;49;00m�[33mdata_gen�[39;49;00m�[33m'�[39;49;00m, all_gen, ids=idfn)
18:58:05 �[94mdef�[39;49;00m �[92mtest_posexplode_nested_array_data�[39;49;00m(spark_tmp_path, data_gen):
18:58:05 data_gen = [int_gen, ArrayGen(ArrayGen(data_gen))]
18:58:05 > assert_gpu_and_cpu_are_equal_collect(
18:58:05 �[94mlambda�[39;49;00m spark: two_col_df(spark, *data_gen).selectExpr(
18:58:05 �[33m'�[39;49;00m�[33ma�[39;49;00m�[33m'�[39;49;00m, �[33m'�[39;49;00m�[33mposexplode(b) as (pos, c)�[39;49;00m�[33m'�[39;49;00m).selectExpr(�[33m'�[39;49;00m�[33ma�[39;49;00m�[33m'�[39;49;00m, �[33m'�[39;49;00m�[33mpos�[39;49;00m�[33m'�[39;49;00m, �[33m'�[39;49;00m�[33mposexplode(c)�[39;49;00m�[33m'�[39;49;00m),
18:58:05 conf=conf_to_enforce_split_input)
18:58:05
18:58:05 �[1m�[31m/home/root/integration_tests/src/main/python/generate_expr_test.py�[0m:107:
18:58:05 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
18:58:05 �[1m�[31m/home/root/integration_tests/src/main/python/asserts.py�[0m:341: in assert_gpu_and_cpu_are_equal_collect
18:58:05 _assert_gpu_and_cpu_are_equal(func, �[33m'�[39;49;00m�[33mCOLLECT�[39;49;00m�[33m'�[39;49;00m, conf=conf)
18:58:05 �[1m�[31m/home/root/integration_tests/src/main/python/asserts.py�[0m:324: in _assert_gpu_and_cpu_are_equal
18:58:05 from_gpu = with_gpu_session(bring_back,
18:58:05 �[1m�[31m/home/root/integration_tests/src/main/python/spark_session.py�[0m:95: in with_gpu_session
18:58:05 �[94mreturn�[39;49;00m with_spark_session(func, conf=copy)
18:58:05 �[1m�[31m/home/root/integration_tests/src/main/python/spark_session.py�[0m:68: in with_spark_session
18:58:05 ret = func(_spark)
18:58:05 �[1m�[31m/home/root/integration_tests/src/main/python/asserts.py�[0m:178: in
18:58:05 bring_back = �[94mlambda�[39;49;00m spark: limit_func(spark).collect()
18:58:05 �[1m�[31m/hadoop/yarn/nm-local-dir/usercache/root/appcache/application_1616146249773_0001/container_e01_1616146249773_0001_01_000001/pyspark.zip/pyspark/sql/dataframe.py�[0m:677: in collect
18:58:05 sock_info = �[96mself�[39;49;00m._jdf.collectToPython()
18:58:05 �[1m�[31m/hadoop/yarn/nm-local-dir/usercache/root/appcache/application_1616146249773_0001/container_e01_1616146249773_0001_01_000001/py4j-0.10.9-src.zip/py4j/java_gateway.py�[0m:1304: in call
18:58:05 return_value = get_return_value(
18:58:05 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
18:58:05
18:58:05 a = ('xro416609', <py4j.java_gateway.GatewayClient object at 0x7f229797c250>, 'o416608', 'collectToPython')
18:58:05 kw = {}
18:58:05 converted = IllegalArgumentException('Part of the plan is not columnar class org.apache.spark.sql.execution.FilterExec\nFilter ((s...:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\n', None)
18:58:05
18:58:05 �[94mdef�[39;49;00m �[92mdeco�[39;49;00m(*a, **kw):
18:58:05 �[94mtry�[39;49;00m:
18:58:05 �[94mreturn�[39;49;00m f(*a, **kw)
18:58:05 �[94mexcept�[39;49;00m py4j.protocol.Py4JJavaError �[94mas�[39;49;00m e:
18:58:05 converted = convert_exception(e.java_exception)
18:58:05 �[94mif�[39;49;00m �[95mnot�[39;49;00m �[96misinstance�[39;49;00m(converted, UnknownException):
18:58:05 �[90m# Hide where the exception came from that shows a non-Pythonic�[39;49;00m
18:58:05 �[90m# JVM exception message.�[39;49;00m
18:58:05 > �[94mraise�[39;49;00m converted �[94mfrom�[39;49;00m �[96mNone�[39;49;00m
18:58:05 �[1m�[31mE pyspark.sql.utils.IllegalArgumentException: Part of the plan is not columnar class org.apache.spark.sql.execution.FilterExec�[0m
18:58:05 �[1m�[31mE Filter ((size(c#219594, true) > 0) AND isnotnull(c#219594))�[0m
18:58:05 �[1m�[31mE +- GpuColumnarToRow false�[0m
18:58:05 �[1m�[31mE +- GpuGenerate gpuposexplode(b#219590), [a#219589], false, [pos#219593, c#219594]�[0m
18:58:05 �[1m�[31mE +- GpuRowToColumnar TargetSize(8192)�[0m
18:58:05 �[1m�[31mE +- Filter ((size(b#219590, true) > 0) AND isnotnull(b#219590))�[0m
18:58:05 �[1m�[31mE +- Scan ExistingRDD[a#219589,b#219590]�[0m
18:58:05
18:58:05 �[1m�[31m/hadoop/yarn/nm-local-dir/usercache/root/appcache/application_1616146249773_0001/container_e01_1616146249773_0001_01_000001/pyspark.zip/pyspark/sql/utils.py�[0m:117: IllegalArgumentException
18:58:05 ----------------------------- Captured stdout call -----------------------------

[jlowe]

I believe this will be fixed by #1972