Spark2 CI (flyteorg#435)

* CI for spark2 Signed-off-by: Hongxin Liang <[email protected]> * Simplify Makefile Signed-off-by: Hongxin Liang <[email protected]> Signed-off-by: Max Hoffman <[email protected]>
dolthub · Apr 29, 2021 · d8345d2 · d8345d2
1 parent 928e865
commit d8345d2
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 28 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -11,11 +11,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
+        python-version: [3.7, 3.8]
+        spark-version-suffix: ["", "-spark2"]
+        exclude:
         - python-version: 3.8
-          spark-version-suffix: "-spark3"
-        - python-version: 3.7
-          spark-version-suffix: ""
+          spark-version-suffix: "-spark2"
 
     steps:
     - uses: actions/checkout@v2

diff --git a/Makefile b/Makefile
@@ -1,6 +1,4 @@
-define PIP_COMPILE
-pip-compile $(1) --upgrade --verbose
-endef
+PIP_COMPILE = pip-compile --upgrade --verbose
 
 .SILENT: help
 .PHONY: help
@@ -17,9 +15,9 @@ install-piptools:
 setup: install-piptools ## Install requirements
 	pip-sync requirements.txt dev-requirements.txt
 
-.PHONY: setup-spark3
-setup-spark3: install-piptools ## Install requirements
-	pip-sync requirements-spark3.txt dev-requirements.txt
+.PHONY: setup-spark2
+setup-spark2: install-piptools ## Install requirements
+	pip-sync requirements-spark2.txt dev-requirements.txt
 
 .PHONY: fmt
 fmt: ## Format code with black and isort
@@ -47,24 +45,24 @@ unit_test:
 	pytest tests/scripts
 	pytest plugins/tests
 
-requirements-spark3.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark3.txt
-requirements-spark3.txt: requirements-spark3.in install-piptools
-	$(call PIP_COMPILE,requirements-spark3.in)
+requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
+requirements-spark2.txt: requirements-spark2.in install-piptools
+	$(PIP_COMPILE) $<
 
 requirements.txt: export CUSTOM_COMPILE_COMMAND := make requirements.txt
-requirements.txt: install-piptools
-	$(call PIP_COMPILE,requirements.in)
+requirements.txt: requirements.in install-piptools
+	$(PIP_COMPILE) $<
 
 dev-requirements.txt: export CUSTOM_COMPILE_COMMAND := make dev-requirements.txt
-dev-requirements.txt: requirements.txt install-piptools
-	$(call PIP_COMPILE,dev-requirements.in)
+dev-requirements.txt: dev-requirements.in requirements.txt install-piptools
+	$(PIP_COMPILE) $<
 
 doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt
-doc-requirements.txt: dev-requirements.txt install-piptools
-	$(call PIP_COMPILE,doc-requirements.in)
+doc-requirements.txt: doc-requirements.in install-piptools
+	$(PIP_COMPILE) $<
 
 .PHONY: requirements
-requirements: requirements.txt dev-requirements.txt requirements-spark3.txt doc-requirements.txt ## Compile requirements
+requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ## Compile requirements
 
 # TODO: Change this in the future to be all of flytekit
 .PHONY: coverage
@@ -80,6 +78,6 @@ update_version:
 	# it exits with exit code 1 and github actions aborts the build. 
 	grep "$(PLACEHOLDER)" "flytekit/__init__.py"
 	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py"
-	
+
 	grep "$(PLACEHOLDER)" "setup.py"
 	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py"
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -14,7 +14,6 @@ attrs==20.3.0
     #   pytest
 black==20.8b1
     # via
-    #   -c requirements.txt
     #   -r dev-requirements.in
     #   flake8-black
 click==7.1.2

diff --git a/requirements-spark3.in → requirements-spark2.in b/requirements-spark3.in → requirements-spark2.in
@@ -1,2 +1,2 @@
-.[all-spark3]
+.[all-spark2.4]
 -e file:.#egg=flytekit
diff --git a/requirements-spark3.txt → requirements-spark2.txt b/requirements-spark3.txt → requirements-spark2.txt
@@ -2,10 +2,10 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    make requirements-spark3.txt
+#    make requirements-spark2.txt
 #
 -e file:.#egg=flytekit
-    # via -r requirements-spark3.in
+    # via -r requirements-spark2.in
 ansiwrap==0.8.4
     # via papermill
 appdirs==1.4.4
@@ -190,7 +190,7 @@ psutil==5.8.0
     # via sagemaker-training
 ptyprocess==0.7.0
     # via pexpect
-py4j==0.10.9
+py4j==0.10.7
     # via pyspark
 py==1.10.0
     # via retry
@@ -209,7 +209,7 @@ pyparsing==2.4.7
     # via packaging
 pyrsistent==0.17.3
     # via jsonschema
-pyspark==3.1.1
+pyspark==2.4.7
     # via flytekit
 python-dateutil==2.8.1
     # via

diff --git a/setup.py b/setup.py
@@ -31,7 +31,6 @@
     "notebook": notebook,
     "sagemaker": sagemaker,
     "all-spark2.4": spark + all_but_spark,
-    "all-spark3": spark3 + all_but_spark,
     "all": spark3 + all_but_spark,
 }