CI for spark2

Signed-off-by: Hongxin Liang <[email protected]>

.github/workflows/pythonbuild.yml

@@ -11,11 +11,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
+        python-version: [3.7, 3.8]
+        spark-version-suffix: ["", "-spark2"]
+        exclude:
         - python-version: 3.8
-          spark-version-suffix: "-spark3"
-        - python-version: 3.7
-          spark-version-suffix: ""
+          spark-version-suffix: "-spark2"
 
     steps:
     - uses: actions/checkout@v2

Makefile

@@ -17,9 +17,9 @@ install-piptools:
 setup: install-piptools ## Install requirements
 	pip-sync requirements.txt dev-requirements.txt
 
-.PHONY: setup-spark3
-setup-spark3: install-piptools ## Install requirements
-	pip-sync requirements-spark3.txt dev-requirements.txt
+.PHONY: setup-spark2
+setup-spark2: install-piptools ## Install requirements
+	pip-sync requirements-spark2.txt dev-requirements.txt
 
 .PHONY: fmt
 fmt: ## Format code with black and isort
@@ -47,9 +47,9 @@ unit_test:
 	pytest tests/scripts
 	pytest plugins/tests
 
-requirements-spark3.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark3.txt
-requirements-spark3.txt: requirements-spark3.in install-piptools
-	$(call PIP_COMPILE,requirements-spark3.in)
+requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
+requirements-spark2.txt: requirements-spark2.in install-piptools
+	$(call PIP_COMPILE,requirements-spark2.in)
 
 requirements.txt: export CUSTOM_COMPILE_COMMAND := make requirements.txt
 requirements.txt: install-piptools
@@ -64,7 +64,7 @@ doc-requirements.txt: dev-requirements.txt install-piptools
 	$(call PIP_COMPILE,doc-requirements.in)
 
 .PHONY: requirements
-requirements: requirements.txt dev-requirements.txt requirements-spark3.txt doc-requirements.txt ## Compile requirements
+requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ## Compile requirements
 
 # TODO: Change this in the future to be all of flytekit
 .PHONY: coverage
@@ -80,6 +80,6 @@ update_version:
 	# it exits with exit code 1 and github actions aborts the build. 
 	grep "$(PLACEHOLDER)" "flytekit/__init__.py"
 	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py"
-	
+
 	grep "$(PLACEHOLDER)" "setup.py"
 	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py"

dev-requirements.txt

@@ -15,7 +15,6 @@ attrs==20.3.0
     #   pytest
 black==19.10b0
     # via
-    #   -c requirements.txt
     #   -r dev-requirements.in
     #   flake8-black
 click==7.1.2

doc-requirements.txt

@@ -12,10 +12,6 @@ ansiwrap==0.8.4
     # via papermill
 appdirs==1.4.4
     # via black
-appnope==0.1.2
-    # via
-    #   ipykernel
-    #   ipython
 astroid==2.5.1
     # via sphinx-autoapi
 async-generator==1.10
@@ -41,9 +37,9 @@ black==19.10b0
     #   papermill
 bleach==3.3.0
     # via nbconvert
-boto3==1.17.36
+boto3==1.17.39
     # via sagemaker-training
-botocore==1.20.36
+botocore==1.20.39
     # via
     #   boto3
     #   s3transfer
@@ -64,10 +60,11 @@ click==7.1.2
     #   papermill
 croniter==1.0.10
     # via flytekit
-cryptography==3.4.6
+cryptography==3.4.7
     # via
     #   -r doc-requirements.in
     #   paramiko
+    #   secretstorage
 css-html-js-minify==2.5.5
     # via sphinx-material
 dataclasses-json==0.5.2
@@ -120,6 +117,10 @@ ipython==7.21.0
     # via ipykernel
 jedi==0.18.0
     # via ipython
+jeepney==0.6.0
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==2.11.3
     # via
     #   nbconvert
@@ -287,6 +288,8 @@ scantree==0.0.1
     # via dirhash
 scipy==1.6.2
     # via sagemaker-training
+secretstorage==3.3.1
+    # via keyring
 six==1.15.0
     # via
     #   bcrypt

requirements-spark3.in → requirements-spark2.in

@@ -1,2 +1,2 @@
-.[all-spark3]
+.[all-spark2.4]
 -e file:.#egg=flytekit

requirements-spark3.txt → requirements-spark2.txt

@@ -2,18 +2,14 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    make requirements-spark3.txt
+#    make requirements-spark2.txt
 #
 -e file:.#egg=flytekit
-    # via -r requirements-spark3.in
+    # via -r requirements-spark2.in
 ansiwrap==0.8.4
     # via papermill
 appdirs==1.4.4
     # via black
-appnope==0.1.2
-    # via
-    #   ipykernel
-    #   ipython
 async-generator==1.10
     # via nbclient
 attrs==20.3.0
@@ -31,9 +27,9 @@ black==19.10b0
     #   papermill
 bleach==3.3.0
     # via nbconvert
-boto3==1.17.36
+boto3==1.17.39
     # via sagemaker-training
-botocore==1.20.36
+botocore==1.20.39
     # via
     #   boto3
     #   s3transfer
@@ -54,8 +50,10 @@ click==7.1.2
     #   papermill
 croniter==1.0.10
     # via flytekit
-cryptography==3.4.6
-    # via paramiko
+cryptography==3.4.7
+    # via
+    #   paramiko
+    #   secretstorage
 dataclasses-json==0.5.2
     # via flytekit
 decorator==4.4.2
@@ -100,6 +98,10 @@ ipython==7.21.0
     # via ipykernel
 jedi==0.18.0
     # via ipython
+jeepney==0.6.0
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==2.11.3
     # via nbconvert
 jmespath==0.10.0
@@ -189,7 +191,7 @@ psutil==5.8.0
     # via sagemaker-training
 ptyprocess==0.7.0
     # via pexpect
-py4j==0.10.9
+py4j==0.10.7
     # via pyspark
 py==1.10.0
     # via retry
@@ -208,7 +210,7 @@ pyparsing==2.4.7
     # via packaging
 pyrsistent==0.17.3
     # via jsonschema
-pyspark==3.1.1
+pyspark==2.4.7
     # via flytekit
 python-dateutil==2.8.1
     # via
@@ -250,6 +252,8 @@ scantree==0.0.1
     # via dirhash
 scipy==1.6.2
     # via sagemaker-training
+secretstorage==3.3.1
+    # via keyring
 six==1.15.0
     # via
     #   bcrypt

requirements.txt

@@ -10,10 +10,6 @@ ansiwrap==0.8.4
     # via papermill
 appdirs==1.4.4
     # via black
-appnope==0.1.2
-    # via
-    #   ipykernel
-    #   ipython
 async-generator==1.10
     # via nbclient
 attrs==20.3.0
@@ -31,9 +27,9 @@ black==19.10b0
     #   papermill
 bleach==3.3.0
     # via nbconvert
-boto3==1.17.36
+boto3==1.17.39
     # via sagemaker-training
-botocore==1.20.36
+botocore==1.20.39
     # via
     #   boto3
     #   s3transfer
@@ -54,8 +50,10 @@ click==7.1.2
     #   papermill
 croniter==1.0.10
     # via flytekit
-cryptography==3.4.6
-    # via paramiko
+cryptography==3.4.7
+    # via
+    #   paramiko
+    #   secretstorage
 dataclasses-json==0.5.2
     # via flytekit
 decorator==4.4.2
@@ -100,6 +98,10 @@ ipython==7.21.0
     # via ipykernel
 jedi==0.18.0
     # via ipython
+jeepney==0.6.0
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==2.11.3
     # via nbconvert
 jmespath==0.10.0
@@ -250,6 +252,8 @@ scantree==0.0.1
     # via dirhash
 scipy==1.6.2
     # via sagemaker-training
+secretstorage==3.3.1
+    # via keyring
 six==1.15.0
     # via
     #   bcrypt

setup.py

@@ -31,7 +31,6 @@
     "notebook": notebook,
     "sagemaker": sagemaker,
     "all-spark2.4": spark + all_but_spark,
-    "all-spark3": spark3 + all_but_spark,
     "all": spark3 + all_but_spark,
 }