From d8345d27373eb07297a8ad75c6ac95feed3655fb Mon Sep 17 00:00:00 2001 From: Honnix Date: Fri, 26 Mar 2021 22:30:32 +0100 Subject: [PATCH] Spark2 CI (#435) * CI for spark2 Signed-off-by: Hongxin Liang * Simplify Makefile Signed-off-by: Hongxin Liang Signed-off-by: Max Hoffman --- .github/workflows/pythonbuild.yml | 8 ++--- Makefile | 32 +++++++++---------- dev-requirements.txt | 1 - ...ements-spark3.in => requirements-spark2.in | 2 +- ...ents-spark3.txt => requirements-spark2.txt | 8 ++--- setup.py | 1 - 6 files changed, 24 insertions(+), 28 deletions(-) rename requirements-spark3.in => requirements-spark2.in (58%) rename requirements-spark3.txt => requirements-spark2.txt (98%) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 3260c8a473..cc0d7f0a52 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -11,11 +11,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - include: + python-version: [3.7, 3.8] + spark-version-suffix: ["", "-spark2"] + exclude: - python-version: 3.8 - spark-version-suffix: "-spark3" - - python-version: 3.7 - spark-version-suffix: "" + spark-version-suffix: "-spark2" steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 6f6c79f293..821588a9b6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,4 @@ -define PIP_COMPILE -pip-compile $(1) --upgrade --verbose -endef +PIP_COMPILE = pip-compile --upgrade --verbose .SILENT: help .PHONY: help @@ -17,9 +15,9 @@ install-piptools: setup: install-piptools ## Install requirements pip-sync requirements.txt dev-requirements.txt -.PHONY: setup-spark3 -setup-spark3: install-piptools ## Install requirements - pip-sync requirements-spark3.txt dev-requirements.txt +.PHONY: setup-spark2 +setup-spark2: install-piptools ## Install requirements + pip-sync requirements-spark2.txt dev-requirements.txt .PHONY: fmt fmt: ## Format code with black and isort @@ -47,24 +45,24 @@ unit_test: pytest tests/scripts pytest plugins/tests -requirements-spark3.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark3.txt -requirements-spark3.txt: requirements-spark3.in install-piptools - $(call PIP_COMPILE,requirements-spark3.in) +requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt +requirements-spark2.txt: requirements-spark2.in install-piptools + $(PIP_COMPILE) $< requirements.txt: export CUSTOM_COMPILE_COMMAND := make requirements.txt -requirements.txt: install-piptools - $(call PIP_COMPILE,requirements.in) +requirements.txt: requirements.in install-piptools + $(PIP_COMPILE) $< dev-requirements.txt: export CUSTOM_COMPILE_COMMAND := make dev-requirements.txt -dev-requirements.txt: requirements.txt install-piptools - $(call PIP_COMPILE,dev-requirements.in) +dev-requirements.txt: dev-requirements.in requirements.txt install-piptools + $(PIP_COMPILE) $< doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt -doc-requirements.txt: dev-requirements.txt install-piptools - $(call PIP_COMPILE,doc-requirements.in) +doc-requirements.txt: doc-requirements.in install-piptools + $(PIP_COMPILE) $< .PHONY: requirements -requirements: requirements.txt dev-requirements.txt requirements-spark3.txt doc-requirements.txt ## Compile requirements +requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ## Compile requirements # TODO: Change this in the future to be all of flytekit .PHONY: coverage @@ -80,6 +78,6 @@ update_version: # it exits with exit code 1 and github actions aborts the build. grep "$(PLACEHOLDER)" "flytekit/__init__.py" sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py" - + grep "$(PLACEHOLDER)" "setup.py" sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py" diff --git a/dev-requirements.txt b/dev-requirements.txt index 5acc7855b8..ba18954a7b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,7 +14,6 @@ attrs==20.3.0 # pytest black==20.8b1 # via - # -c requirements.txt # -r dev-requirements.in # flake8-black click==7.1.2 diff --git a/requirements-spark3.in b/requirements-spark2.in similarity index 58% rename from requirements-spark3.in rename to requirements-spark2.in index f12785fbdb..d5f2633c7e 100644 --- a/requirements-spark3.in +++ b/requirements-spark2.in @@ -1,2 +1,2 @@ -.[all-spark3] +.[all-spark2.4] -e file:.#egg=flytekit diff --git a/requirements-spark3.txt b/requirements-spark2.txt similarity index 98% rename from requirements-spark3.txt rename to requirements-spark2.txt index 69aec3fb38..c58e52ca25 100644 --- a/requirements-spark3.txt +++ b/requirements-spark2.txt @@ -2,10 +2,10 @@ # This file is autogenerated by pip-compile # To update, run: # -# make requirements-spark3.txt +# make requirements-spark2.txt # -e file:.#egg=flytekit - # via -r requirements-spark3.in + # via -r requirements-spark2.in ansiwrap==0.8.4 # via papermill appdirs==1.4.4 @@ -190,7 +190,7 @@ psutil==5.8.0 # via sagemaker-training ptyprocess==0.7.0 # via pexpect -py4j==0.10.9 +py4j==0.10.7 # via pyspark py==1.10.0 # via retry @@ -209,7 +209,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -pyspark==3.1.1 +pyspark==2.4.7 # via flytekit python-dateutil==2.8.1 # via diff --git a/setup.py b/setup.py index 0ea2a69fa8..cd687e7c53 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ "notebook": notebook, "sagemaker": sagemaker, "all-spark2.4": spark + all_but_spark, - "all-spark3": spark3 + all_but_spark, "all": spark3 + all_but_spark, }