feat(build): add spark 3.3.1 (#58)

Also update README and the command to download `poetry`.
dsaidgovsg · Nov 22, 2022 · 55e86f0 · 55e86f0
1 parent 8f4b76a
commit 55e86f0
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 3 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -195,6 +195,66 @@ jobs:
           scala:  "2.13"
           java:   "11"
           python: "3.9"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.12"
+          java:   "8"
+          python: "3.7"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.12"
+          java:   "8"
+          python: "3.8"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.12"
+          java:   "8"
+          python: "3.9"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.13"
+          java:   "8"
+          python: "3.7"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.13"
+          java:   "8"
+          python: "3.8"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.13"
+          java:   "8"
+          python: "3.9"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.12"
+          java:   "11"
+          python: "3.7"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.12"
+          java:   "11"
+          python: "3.8"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.12"
+          java:   "11"
+          python: "3.9"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.13"
+          java:   "11"
+          python: "3.7"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.13"
+          java:   "11"
+          python: "3.8"
+        - spark:  "3.3.1"
+          hadoop: "3.3.2"
+          scala:  "2.13"
+          java:   "11"
+          python: "3.9"
     runs-on: ubuntu-20.04
     env:
       IMAGE_NAME: spark-k8s-addons

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,7 @@
 ## v5
 
 - Use non-versioned constant path for `py4j import.
-- Build only for 3.1.3, 3.2.2 and 3.3.0, and their related supported Hadoop
+- Build only for 3.1.3, 3.2.2, 3.3.0 and 3.3.1, and their related supported Hadoop
   variants (dropped all 2.y.z versions).
 - Change base image version to `v3`.
 

diff --git a/Dockerfile b/Dockerfile
@@ -62,7 +62,7 @@ ENV POETRY_HOME=/opt/poetry
 ENV POETRY_SYSTEM_PROJECT_DIR="${POETRY_HOME}/.system"
 ENV PATH="${POETRY_HOME}/bin:${PATH}"
 RUN set -euo pipefail && \
-    curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python; \
+    curl -sSL https://install.python-poetry.org | python3 -; \
     poetry --version; \
     mkdir -p "${POETRY_SYSTEM_PROJECT_DIR}"; \
     cd "${POETRY_SYSTEM_PROJECT_DIR}"; \

diff --git a/README.md b/README.md
@@ -105,3 +105,9 @@ Otherwise, you will need `cargo`, which can be installed via
 [rustup](https://rustup.rs/).
 
 Once `cargo` is installed, simply run `cargo install tera-cli --version=^0.4.0`.
+
+Always make changes in `templates/ci.yml.tmpl` since the template will be
+applied onto `.github/workflows/ci.yml`.
+
+Run `templates/apply-vars.sh` to apply the template once `tera-cli` has been
+installed.
diff --git a/templates/vars.yml b/templates/vars.yml
@@ -14,7 +14,7 @@ versions:
   java:   ["8", "11"]
   python: ["3.7", "3.8", "3.9"]
 
-- spark:  ["3.3.0"]
+- spark:  ["3.3.0", "3.3.1"]
   hadoop: ["3.3.2"]
   scala:  ["2.12", "2.13"]
   java:   ["8", "11"]