From e1c2f776b5903e1bcaff57b48214ac87331c2d49 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 1 Mar 2024 07:30:02 +0800 Subject: [PATCH 1/4] Update rapids jni and private dependency version to 24.02.1 (#10511) Signed-off-by: Tim Liu --- pom.xml | 4 ++-- scala2.13/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index d3438970309..a238e4cf90c 100644 --- a/pom.xml +++ b/pom.xml @@ -722,8 +722,8 @@ spark${buildver} cuda11 ${cuda.version} - 24.02.0 - 24.02.0 + 24.02.1 + 24.02.1 2.12 2.8.0 incremental diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 8b07b20b4e3..68ded9d2985 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -722,8 +722,8 @@ spark${buildver} cuda11 ${cuda.version} - 24.02.0 - 24.02.0 + 24.02.1 + 24.02.1 2.13 2.8.0 incremental From c4abf14c4190e905eb898d2d30922ee1370adaed Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 1 Mar 2024 17:05:28 +0800 Subject: [PATCH 2/4] Add missed shims for scala2.13 (#10465) * Add missed shims for scala2.13 Signed-off-by: Tim Liu * Add 351 snapshot shim for the scala2.13 version of plugin jar Signed-off-by: Tim Liu * Remove 351 snapshot shim as spark 3.5.1 has been released Signed-off-by: Tim Liu * Remove scala2.13 351 snapshot shim Signed-off-by: Tim Liu * Remove 351 shim's jason string Ran `mvn generate-sources -Dshimplify=true -Dshimplify.move=true -Dshimplify.remove.shim=351` to remove 351 shim's jason string, and fix some unnecessary empty lines that were introduced Signed-off-by: Tim Liu * Update Copyright 2024 Auto copyright by below scripts ``` export SPARK_RAPIDS_AUTO_COPYRIGHTER=ON ./scripts/auto-copyrighter.sh $(git diff --name-only origin/branch-24.04..HEAD) ``` Signed-off-by: Tim Liu * Revert "Update Copyright 2024" This reverts commit 8482847761f07cdea8bba55f3ab720f7201f3b50. * Revert "Remove 351 shim's jason string" This reverts commit 78d1f00c895fdc766a942b7c8fc787568d14e002. * skip 351 from strict checking * Alien scala2.13/pom.xml to scala2.12 one Run the script `bash build/make-scala-version-build-files.sh 2.13` Signed-off-by: Tim Liu * pretend 351 is a snapshot in 24.02 Signed-off-by: Gera Shegalov * pretend 351 is a SNAPSHOT version * Revert change of build/shimplify.py Signed-off-by: Tim Liu --------- Signed-off-by: Tim Liu Signed-off-by: Gera Shegalov Co-authored-by: Raza Jafri Co-authored-by: Gera Shegalov --- pom.xml | 3 +++ scala2.13/pom.xml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index a238e4cf90c..5b66f068bd9 100644 --- a/pom.xml +++ b/pom.xml @@ -881,11 +881,14 @@ 331, 332, 333, + 334, 340, 341, + 342, 350 + 351 ${noSnapshotScala213.buildvers} diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 68ded9d2985..2276bb98de2 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -881,11 +881,14 @@ 331, 332, 333, + 334, 340, 341, + 342, 350 + 351 ${noSnapshotScala213.buildvers} From 23f1bc48673eca126c98d5cf0a0933011546ae14 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 1 Mar 2024 17:48:53 +0800 Subject: [PATCH 3/4] Update changelog for v24.02.0 release (#10525) Signed-off-by: Tim Liu --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb53243cdb..a9851851f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Change log -Generated on 2024-02-17 +Generated on 2024-03-01 ## Release 24.02 @@ -46,6 +46,9 @@ Generated on 2024-02-17 ### Bugs Fixed ||| |:---|:---| +|[#10464](https://github.com/NVIDIA/spark-rapids/issues/10464)|[BUG] spark334 and spark342 shims missed in scala2.13 dist jar| +|[#10473](https://github.com/NVIDIA/spark-rapids/issues/10473)|[BUG] Leak when running RANK query| +|[#10432](https://github.com/NVIDIA/spark-rapids/issues/10432)|Plug-in Build Failing for Databricks 11.3 | |[#9974](https://github.com/NVIDIA/spark-rapids/issues/9974)|[BUG] host memory Leak in MultiFileCoalescingPartitionReaderBase in UTC time zone| |[#10359](https://github.com/NVIDIA/spark-rapids/issues/10359)|[BUG] Build failure on Databricks nightly run with `GpuMapInPandasExecMeta`| |[#10327](https://github.com/NVIDIA/spark-rapids/issues/10327)|[BUG] Unit test FAILED against : SPARK-24957: average with decimal followed by aggregation returning wrong result | @@ -119,6 +122,12 @@ Generated on 2024-02-17 ### PRs ||| |:---|:---| +|[#10465](https://github.com/NVIDIA/spark-rapids/pull/10465)|Add missed shims for scala2.13| +|[#10511](https://github.com/NVIDIA/spark-rapids/pull/10511)|Update rapids jni and private dependency version to 24.02.1| +|[#10475](https://github.com/NVIDIA/spark-rapids/pull/10475)|Fix scalar leak in RankFixer| +|[#10461](https://github.com/NVIDIA/spark-rapids/pull/10461)|Preserve tags on FileSourceScanExec| +|[#10459](https://github.com/NVIDIA/spark-rapids/pull/10459)|[DOC] Fix table rendering issue in github.io download UI page on branch-24.02 [skip ci] | +|[#10443](https://github.com/NVIDIA/spark-rapids/pull/10443)|Update change log for v24.02.0 release [skip ci]| |[#10439](https://github.com/NVIDIA/spark-rapids/pull/10439)|Reverts NVIDIA/spark-rapids#10232 and fixes the plugin build on Databricks 11.3| |[#10380](https://github.com/NVIDIA/spark-rapids/pull/10380)|Init changelog 24.02 [skip ci]| |[#10367](https://github.com/NVIDIA/spark-rapids/pull/10367)|Update rapids JNI and private version to release 24.02.0| From 4d14b7523ed5fb3da40a9ae29971b5b2f81ff421 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 4 Mar 2024 16:34:24 -0600 Subject: [PATCH 4/4] Fix Delta log cache size settings during integration tests (#10541) Signed-off-by: Jason Lowe --- integration_tests/run_pyspark_from_build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh index cc983d49b3c..491fb271151 100755 --- a/integration_tests/run_pyspark_from_build.sh +++ b/integration_tests/run_pyspark_from_build.sh @@ -227,7 +227,9 @@ else TZ=${TZ:-UTC} # Set the Delta log cache size to prevent the driver from caching every Delta log indefinitely - export PYSP_TEST_spark_driver_extraJavaOptions="-ea -Duser.timezone=$TZ -Ddelta.log.cacheSize=10 $COVERAGE_SUBMIT_FLAGS" + export PYSP_TEST_spark_databricks_delta_delta_log_cacheSize=${PYSP_TEST_spark_databricks_delta_delta_log_cacheSize:-10} + deltaCacheSize=$PYSP_TEST_spark_databricks_delta_delta_log_cacheSize + export PYSP_TEST_spark_driver_extraJavaOptions="-ea -Duser.timezone=$TZ -Ddelta.log.cacheSize=$deltaCacheSize $COVERAGE_SUBMIT_FLAGS" export PYSP_TEST_spark_executor_extraJavaOptions="-ea -Duser.timezone=$TZ" export PYSP_TEST_spark_ui_showConsoleProgress='false' export PYSP_TEST_spark_sql_session_timeZone=$TZ @@ -380,6 +382,7 @@ EOF # avoid double processing of variables passed to spark in # spark_conf_init + unset PYSP_TEST_spark_databricks_delta_delta_log_cacheSize unset PYSP_TEST_spark_driver_extraClassPath unset PYSP_TEST_spark_driver_extraJavaOptions unset PYSP_TEST_spark_jars @@ -391,6 +394,7 @@ EOF --driver-java-options "$driverJavaOpts" \ $SPARK_SUBMIT_FLAGS \ --conf 'spark.rapids.memory.gpu.allocSize='"$gpuAllocSize" \ + --conf 'spark.databricks.delta.delta.log.cacheSize='"$deltaCacheSize" \ "${RUN_TESTS_COMMAND[@]}" "${TEST_COMMON_OPTS[@]}" fi fi