Fixed flytekit-papermill ImportError (flyteorg#818)

Signed-off-by: Kevin Su <[email protected]>
Signed-off-by: Kenny Workman <[email protected]>

.github/workflows/pythonbuild.yml

@@ -102,6 +102,7 @@ jobs:
         make setup
         cd plugins/${{ matrix.plugin-names }}
         pip install -e .
+        if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
         pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
         pip freeze
     - name: Test with coverage

plugins/flytekit-papermill/dev-requirements.in

@@ -0,0 +1 @@
+flytekitplugins-spark>=0.30.0b4

plugins/flytekit-papermill/dev-requirements.txt

@@ -0,0 +1,152 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    pip-compile dev-requirements.in
+#
+arrow==1.2.1
+    # via jinja2-time
+binaryornot==0.4.4
+    # via cookiecutter
+certifi==2021.10.8
+    # via requests
+chardet==4.0.0
+    # via binaryornot
+charset-normalizer==2.0.10
+    # via requests
+checksumdir==1.2.0
+    # via flytekit
+click==7.1.2
+    # via
+    #   cookiecutter
+    #   flytekit
+cloudpickle==2.0.0
+    # via flytekit
+cookiecutter==1.7.3
+    # via flytekit
+croniter==1.2.0
+    # via flytekit
+dataclasses-json==0.5.6
+    # via flytekit
+decorator==5.1.1
+    # via retry
+deprecated==1.2.13
+    # via flytekit
+diskcache==5.4.0
+    # via flytekit
+docker-image-py==0.1.12
+    # via flytekit
+docstring-parser==0.13
+    # via flytekit
+flyteidl==0.21.23
+    # via flytekit
+flytekit==0.26.0
+    # via flytekitplugins-spark
+flytekitplugins-spark==0.30.0b4
+    # via -r dev-requirements.in
+grpcio==1.43.0
+    # via flytekit
+idna==3.3
+    # via requests
+importlib-metadata==4.10.1
+    # via keyring
+jinja2==3.0.3
+    # via
+    #   cookiecutter
+    #   jinja2-time
+jinja2-time==0.2.0
+    # via cookiecutter
+keyring==23.5.0
+    # via flytekit
+markupsafe==2.0.1
+    # via jinja2
+marshmallow==3.14.1
+    # via
+    #   dataclasses-json
+    #   marshmallow-enum
+    #   marshmallow-jsonschema
+marshmallow-enum==1.5.1
+    # via dataclasses-json
+marshmallow-jsonschema==0.13.0
+    # via flytekit
+mypy-extensions==0.4.3
+    # via typing-inspect
+natsort==8.0.2
+    # via flytekit
+numpy==1.22.1
+    # via
+    #   pandas
+    #   pyarrow
+pandas==1.3.5
+    # via flytekit
+poyo==0.5.0
+    # via cookiecutter
+protobuf==3.19.3
+    # via
+    #   flyteidl
+    #   flytekit
+py==1.11.0
+    # via retry
+py4j==0.10.9.2
+    # via pyspark
+pyarrow==6.0.1
+    # via flytekit
+pyspark==3.2.0
+    # via flytekitplugins-spark
+python-dateutil==2.8.1
+    # via
+    #   arrow
+    #   croniter
+    #   flytekit
+    #   pandas
+python-json-logger==2.0.2
+    # via flytekit
+python-slugify==5.0.2
+    # via cookiecutter
+pytimeparse==1.1.8
+    # via flytekit
+pytz==2021.3
+    # via
+    #   flytekit
+    #   pandas
+regex==2021.11.10
+    # via docker-image-py
+requests==2.27.1
+    # via
+    #   cookiecutter
+    #   flytekit
+    #   responses
+responses==0.17.0
+    # via flytekit
+retry==0.9.2
+    # via flytekit
+six==1.16.0
+    # via
+    #   cookiecutter
+    #   flytekit
+    #   grpcio
+    #   python-dateutil
+    #   responses
+sortedcontainers==2.4.0
+    # via flytekit
+statsd==3.3.0
+    # via flytekit
+text-unidecode==1.3
+    # via python-slugify
+typing-extensions==4.0.1
+    # via typing-inspect
+typing-inspect==0.7.1
+    # via dataclasses-json
+urllib3==1.26.8
+    # via
+    #   flytekit
+    #   requests
+    #   responses
+wheel==0.37.1
+    # via flytekit
+wrapt==1.13.3
+    # via
+    #   deprecated
+    #   flytekit
+zipp==3.7.0
+    # via importlib-metadata

plugins/flytekit-papermill/requirements.in

@@ -1,3 +1,2 @@
 .
 -e file:.#egg=flytekitplugins-papermill
-flytekitplugins-spark

plugins/flytekit-papermill/requirements.txt

@@ -8,6 +8,10 @@
     # via -r requirements.in
 ansiwrap==0.8.4
     # via papermill
+appnope==0.1.2
+    # via
+    #   ipykernel
+    #   ipython
 arrow==1.2.1
     # via jinja2-time
 attrs==21.2.0
@@ -22,8 +26,6 @@ bleach==4.1.0
     # via nbconvert
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
 charset-normalizer==2.0.7
@@ -42,8 +44,6 @@ cookiecutter==1.7.3
     # via flytekit
 croniter==1.0.15
     # via flytekit
-cryptography==35.0.0
-    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 debugpy==1.5.1
@@ -67,16 +67,10 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.21.8
+flyteidl==0.21.23
     # via flytekit
 flytekit==0.24.0
-    # via
-    #   flytekitplugins-papermill
-    #   flytekitplugins-spark
-flytekitplugins-spark==0.24.0
-    # via
-    #   -r requirements.in
-    #   flytekitplugins-papermill
+    # via flytekitplugins-papermill
 grpcio==1.41.1
     # via flytekit
 idna==3.3
@@ -91,10 +85,6 @@ ipython-genutils==0.2.0
     # via nbformat
 jedi==0.18.0
     # via ipython
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -189,12 +179,8 @@ ptyprocess==0.7.0
     # via pexpect
 py==1.11.0
     # via retry
-py4j==0.10.9.2
-    # via pyspark
 pyarrow==6.0.0
     # via flytekit
-pycparser==2.21
-    # via cffi
 pygments==2.10.0
     # via
     #   ipython
@@ -204,8 +190,6 @@ pyparsing==2.4.7
     # via packaging
 pyrsistent==0.18.0
     # via jsonschema
-pyspark==3.2.0
-    # via flytekitplugins-spark
 python-dateutil==2.8.1
     # via
     #   arrow
@@ -241,8 +225,6 @@ responses==0.15.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   bleach

plugins/flytekit-papermill/setup.py

@@ -6,7 +6,6 @@
 
 plugin_requires = [
     "flytekit>=0.16.0b0,<1.0.0",
-    "flytekitplugins-spark>=0.16.0b0,<1.0.0,!=0.24.0b0",
     "papermill>=1.2.0",
     "nbconvert>=6.0.7",
     "ipykernel>=5.0.0",