Merge pull request #10544 from jlowe/fix-merge

Fix merge conflict from branch-24.02
NVIDIA · Mar 5, 2024 · fb7abc8 · fb7abc8
2 parents dfc18b2 + 82158aa
commit fb7abc8
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
@@ -231,7 +231,9 @@ else
     export PYSP_TEST_spark_ui_enabled=${PYSP_TEST_spark_ui_enabled:-false}
 
     # Set the Delta log cache size to prevent the driver from caching every Delta log indefinitely
-    export PYSP_TEST_spark_driver_extraJavaOptions="-ea -Duser.timezone=$TZ -Ddelta.log.cacheSize=10 $COVERAGE_SUBMIT_FLAGS"
+    export PYSP_TEST_spark_databricks_delta_delta_log_cacheSize=${PYSP_TEST_spark_databricks_delta_delta_log_cacheSize:-10}
+    deltaCacheSize=$PYSP_TEST_spark_databricks_delta_delta_log_cacheSize
+    export PYSP_TEST_spark_driver_extraJavaOptions="-ea -Duser.timezone=$TZ -Ddelta.log.cacheSize=$deltaCacheSize $COVERAGE_SUBMIT_FLAGS"
     export PYSP_TEST_spark_executor_extraJavaOptions="-ea -Duser.timezone=$TZ"
     export PYSP_TEST_spark_ui_showConsoleProgress='false'
     export PYSP_TEST_spark_sql_session_timeZone=$TZ
@@ -384,6 +386,7 @@ EOF
 
         # avoid double processing of variables passed to spark in
         # spark_conf_init
+        unset PYSP_TEST_spark_databricks_delta_delta_log_cacheSize
         unset PYSP_TEST_spark_driver_extraClassPath
         unset PYSP_TEST_spark_driver_extraJavaOptions
         unset PYSP_TEST_spark_jars
@@ -395,6 +398,7 @@ EOF
             --driver-java-options "$driverJavaOpts" \
             $SPARK_SUBMIT_FLAGS \
             --conf 'spark.rapids.memory.gpu.allocSize='"$gpuAllocSize" \
+            --conf 'spark.databricks.delta.delta.log.cacheSize='"$deltaCacheSize" \
             "${RUN_TESTS_COMMAND[@]}" "${TEST_COMMON_OPTS[@]}"
     fi
 fi