Fixed flytekit-papermill ImportError (flyteorg#818)

Signed-off-by: Kevin Su <[email protected]>

.github/workflows/pythonbuild.yml

@@ -102,6 +102,7 @@ jobs:
         make setup
         cd plugins/${{ matrix.plugin-names }}
         pip install -e .
+        if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
         pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
         pip freeze
     - name: Test with coverage

plugins/flytekit-papermill/dev-requirements.in

@@ -0,0 +1 @@
+flytekitplugins-spark>=0.30.0b4

plugins/flytekit-papermill/dev-requirements.txt

@@ -0,0 +1,152 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    pip-compile dev-requirements.in
+#
+arrow==1.2.1
+    # via jinja2-time
+binaryornot==0.4.4
+    # via cookiecutter
+certifi==2021.10.8
+    # via requests
+chardet==4.0.0
+    # via binaryornot
+charset-normalizer==2.0.10
+    # via requests
+checksumdir==1.2.0
+    # via flytekit
+click==7.1.2
+    # via
+    #   cookiecutter
+    #   flytekit
+cloudpickle==2.0.0
+    # via flytekit
+cookiecutter==1.7.3
+    # via flytekit
+croniter==1.2.0
+    # via flytekit
+dataclasses-json==0.5.6
+    # via flytekit
+decorator==5.1.1
+    # via retry
+deprecated==1.2.13
+    # via flytekit
+diskcache==5.4.0
+    # via flytekit
+docker-image-py==0.1.12
+    # via flytekit
+docstring-parser==0.13
+    # via flytekit
+flyteidl==0.21.23
+    # via flytekit
+flytekit==0.26.0
+    # via flytekitplugins-spark
+flytekitplugins-spark==0.30.0b4
+    # via -r dev-requirements.in
+grpcio==1.43.0
+    # via flytekit
+idna==3.3
+    # via requests
+importlib-metadata==4.10.1
+    # via keyring
+jinja2==3.0.3
+    # via
+    #   cookiecutter
+    #   jinja2-time
+jinja2-time==0.2.0
+    # via cookiecutter
+keyring==23.5.0
+    # via flytekit
+markupsafe==2.0.1
+    # via jinja2
+marshmallow==3.14.1
+    # via
+    #   dataclasses-json
+    #   marshmallow-enum
+    #   marshmallow-jsonschema
+marshmallow-enum==1.5.1
+    # via dataclasses-json
+marshmallow-jsonschema==0.13.0
+    # via flytekit
+mypy-extensions==0.4.3
+    # via typing-inspect
+natsort==8.0.2
+    # via flytekit
+numpy==1.22.1
+    # via
+    #   pandas
+    #   pyarrow
+pandas==1.3.5
+    # via flytekit
+poyo==0.5.0
+    # via cookiecutter
+protobuf==3.19.3
+    # via
+    #   flyteidl
+    #   flytekit
+py==1.11.0
+    # via retry
+py4j==0.10.9.2
+    # via pyspark
+pyarrow==6.0.1
+    # via flytekit
+pyspark==3.2.0
+    # via flytekitplugins-spark
+python-dateutil==2.8.1
+    # via
+    #   arrow
+    #   croniter
+    #   flytekit
+    #   pandas
+python-json-logger==2.0.2
+    # via flytekit
+python-slugify==5.0.2
+    # via cookiecutter
+pytimeparse==1.1.8
+    # via flytekit
+pytz==2021.3
+    # via
+    #   flytekit
+    #   pandas
+regex==2021.11.10
+    # via docker-image-py
+requests==2.27.1
+    # via
+    #   cookiecutter
+    #   flytekit
+    #   responses
+responses==0.17.0
+    # via flytekit
+retry==0.9.2
+    # via flytekit
+six==1.16.0
+    # via
+    #   cookiecutter
+    #   flytekit
+    #   grpcio
+    #   python-dateutil
+    #   responses
+sortedcontainers==2.4.0
+    # via flytekit
+statsd==3.3.0
+    # via flytekit
+text-unidecode==1.3
+    # via python-slugify
+typing-extensions==4.0.1
+    # via typing-inspect
+typing-inspect==0.7.1
+    # via dataclasses-json
+urllib3==1.26.8
+    # via
+    #   flytekit
+    #   requests
+    #   responses
+wheel==0.37.1
+    # via flytekit
+wrapt==1.13.3
+    # via
+    #   deprecated
+    #   flytekit
+zipp==3.7.0
+    # via importlib-metadata

plugins/flytekit-papermill/requirements.in

@@ -1,3 +1,2 @@
 .
 -e file:.#egg=flytekitplugins-papermill
-flytekitplugins-spark

plugins/flytekit-papermill/requirements.txt

@@ -67,17 +67,11 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.21.13
+flyteidl==0.21.23
     # via flytekit
-flytekit==0.25.0
-    # via
-    #   flytekitplugins-papermill
-    #   flytekitplugins-spark
-flytekitplugins-spark==0.25.0
-    # via
-    #   -r requirements.in
-    #   flytekitplugins-papermill
-grpcio==1.42.0
+flytekit==0.24.0
+    # via flytekitplugins-papermill
+grpcio==1.41.1
     # via flytekit
 idna==3.3
     # via requests
@@ -185,9 +179,13 @@ ptyprocess==0.7.0
     # via pexpect
 py==1.11.0
     # via retry
+<<<<<<< HEAD
 py4j==0.10.9.2
     # via pyspark
 pyarrow==6.0.1
+=======
+pyarrow==6.0.0
+>>>>>>> 7fcaedcf (Fixed flytekit-papermill ImportError (#818))
     # via flytekit
 pygments==2.10.0
     # via
@@ -198,8 +196,6 @@ pyparsing==3.0.6
     # via packaging
 pyrsistent==0.18.0
     # via jsonschema
-pyspark==3.2.0
-    # via flytekitplugins-spark
 python-dateutil==2.8.1
     # via
     #   arrow
@@ -296,4 +292,4 @@ zipp==3.6.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+# setuptools

plugins/flytekit-papermill/setup.py

@@ -6,7 +6,6 @@
 
 plugin_requires = [
     "flytekit>=0.16.0b0,<1.0.0",
-    "flytekitplugins-spark>=0.16.0b0,<1.0.0,!=0.24.0b0",
     "papermill>=1.2.0",
     "nbconvert>=6.0.7",
     "ipykernel>=5.0.0",