From 4e627b1e70c59189ea3e23f2a3ffb6108d75f800 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Thu, 3 Aug 2023 09:20:04 +0800 Subject: [PATCH] Databricks shim version for integration test Signed-off-by: Tim Liu --- jenkins/databricks/common_vars.sh | 3 +++ jenkins/databricks/test.sh | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins/databricks/common_vars.sh b/jenkins/databricks/common_vars.sh index cf09143ba2f..5f02cbd9439 100644 --- a/jenkins/databricks/common_vars.sh +++ b/jenkins/databricks/common_vars.sh @@ -15,6 +15,9 @@ # limitations under the License. # +SPARK_VER=${SPARK_VER:-$(< /databricks/spark/VERSION)} +export SPARK_SHIM_VER=${SPARK_SHIM_VER:-spark${SPARK_VER//.}db} + # Setup SPARK_HOME if need if [[ -z "$SPARK_HOME" ]]; then # Configure spark environment on Databricks diff --git a/jenkins/databricks/test.sh b/jenkins/databricks/test.sh index 5e87398aa4b..6c96e45ff72 100755 --- a/jenkins/databricks/test.sh +++ b/jenkins/databricks/test.sh @@ -49,7 +49,6 @@ source jenkins/databricks/common_vars.sh BASE_SPARK_VERSION=${BASE_SPARK_VERSION:-$(< /databricks/spark/VERSION)} SHUFFLE_SPARK_SHIM=${SHUFFLE_SPARK_SHIM:-spark${BASE_SPARK_VERSION//./}db} SHUFFLE_SPARK_SHIM=${SHUFFLE_SPARK_SHIM//\-SNAPSHOT/} -[[ -z $SPARK_SHIM_VER ]] && export SPARK_SHIM_VER=spark${BASE_SPARK_VERSION//.}db IS_SPARK_321_OR_LATER=0 [[ "$(printf '%s\n' "3.2.1" "$BASE_SPARK_VERSION" | sort -V | head -n1)" = "3.2.1" ]] && IS_SPARK_321_OR_LATER=1