add Spark 3.5 on JDK 17

godatadriven-dockerhub · Nov 28, 2024 · 7d4db76 · 7d4db76
1 parent 94a5cf6
commit 7d4db76
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 6 deletions.
diff --git a/.github/workflows/dockerhub_ci.yml b/.github/workflows/dockerhub_ci.yml
@@ -22,7 +22,13 @@ jobs:
     strategy:
       matrix:
         # Specify which tags to build
-        DOCKER_TAG: ["3.4.0,3.4,latest", "3.3.2,3.3", "3.1.3,3.1", "3.0.3,3.0", "2.4.8,2.4"]
+        DOCKER_TAG:
+          - "3.5.3,3.5,latest"
+          - "3.4.4,3.4"
+          - "3.3.4,3.3"
+          - "3.1.3,3.1"
+          - "3.0.3,3.0"
+          - "2.4.8,2.4"
 
     timeout-minutes: 30
     steps:
@@ -37,7 +43,10 @@ jobs:
             # Manipulate DOCKER_TAG to create build args
             SPARK_MAJOR_VERSION=${DOCKER_TAG:0:1}
             SPARK_MAJOR_MINOR_VERSION=${DOCKER_TAG:0:3}
-            if [[ ${SPARK_MAJOR_MINOR_VERSION} = "3.3" || ${SPARK_MAJOR_MINOR_VERSION} = "3.4" || ${DOCKER_TAG} == "latest" ]]; then
+            if [[ ${SPARK_MAJOR_MINOR_VERSION} = "3.5" || ${DOCKER_TAG} == "latest" ]]; then
+                OPENJDK_VERSION=17
+                HADOOP_VERSION=3
+            elif [[ ${SPARK_MAJOR_MINOR_VERSION} = "3.3" || ${SPARK_MAJOR_MINOR_VERSION} = "3.4" ]]; then
                 OPENJDK_VERSION=11
                 HADOOP_VERSION=3
             elif [ ${SPARK_MAJOR_VERSION} = "3" ]; then
@@ -97,8 +106,8 @@ jobs:
             IMAGE_NAME: ${{ steps.prep.outputs.image_name }}
         run: |
             if [[ -f "docker-compose.test.yml" ]]; then
-                docker-compose --file docker-compose.test.yml build
-                docker-compose --file docker-compose.test.yml run sut
+                docker compose --file docker-compose.test.yml build
+                docker compose --file docker-compose.test.yml run sut
             fi
 
       - name: Login to DockerHub

diff --git a/Dockerfile b/Dockerfile
@@ -9,11 +9,11 @@ LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \
       org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.version=$SPARK_VERSION      
 
-ENV SPARK_HOME /usr/spark
+ENV SPARK_HOME=/usr/spark
 ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
 
 RUN apt-get update && \
-    apt-get install -y wget netcat procps libpostgresql-jdbc-java && \
+    apt-get install -y wget netcat-openbsd procps libpostgresql-jdbc-java && \
     wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \