Merge remote-tracking branch 'origin/master' into onnx-plugin

flyteorg · Jul 12, 2022 · 5626045 · 5626045
2 parents 66d64be + 93122da
commit 5626045
Show file tree

Hide file tree

Showing 138 changed files with 3,917 additions and 1,070 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -51,7 +51,7 @@ jobs:
         run: |
           coverage run -m pytest tests/flytekit/unit -m "not sandbox_test"
       - name: Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v1.5.2
         with:
           fail_ci_if_error: true # optional (default = false)
 
@@ -82,6 +82,7 @@ jobs:
           - flytekit-onnx-tensorflow
           - flytekit-pandera
           - flytekit-papermill
+          - flytekit-polars
           - flytekit-snowflake
           - flytekit-spark
           - flytekit-sqlalchemy
@@ -90,6 +91,10 @@ jobs:
           # Issue tracked in https://github.com/ray-project/ray/issues/19116.
           - python-version: 3.10
             plugin-names: "flytekit-modin"
+          # Great-expectations does not support python 3.10 yet
+          # https://github.com/great-expectations/great_expectations/blob/develop/setup.py#L87-L89
+          - python-version: 3.10
+            plugin-names: "flytekit-greatexpectations"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -107,7 +112,7 @@ jobs:
         run: |
           make setup
           cd plugins/${{ matrix.plugin-names }}
-          pip install -e .
+          pip install -r requirements.txt
           if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
           pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
           pip freeze

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,24 +1,24 @@
 repos:
 - repo: https://github.com/PyCQA/flake8
-  rev:  3.9.2
+  rev:  4.0.1
   hooks:
     - id: flake8
 - repo: https://github.com/psf/black
   rev:  22.3.0
   hooks:
     - id: black
 - repo: https://github.com/PyCQA/isort
-  rev:  5.9.3
+  rev:  5.10.1
   hooks:
     - id: isort
       args: ["--profile", "black"]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.0.1
+  rev: v4.2.0
   hooks:
     - id: check-yaml
     - id: end-of-file-fixer
     - id: trailing-whitespace
 - repo: https://github.com/shellcheck-py/shellcheck-py
-  rev: v0.7.2.1
+  rev: v0.8.0.4
   hooks:
   - id: shellcheck
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,3 +1,3 @@
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence.
-*       @wild-endeavor @kumare3 @eapolinario
+*       @wild-endeavor @kumare3 @eapolinario @pingsutw
diff --git a/README.md b/README.md
@@ -1,19 +1,17 @@
-<html>
-    <p align="center">
-        <img src="https://raw.githubusercontent.com/flyteorg/static-resources/main/common/flyte_circle_gradient_1_4x4.png" alt="Flyte Logo" width="100">
-    </p>
-    <h1 align="center">
-        Flytekit Python
-    </h1>
-    <p align="center">
-        Flytekit Python is the Python SDK built on top of Flyte
-    </p>
-    <h3 align="center">
-        <a href="plugins/README.md">Plugins</a>
-        <span> · </span>
-        <a href="https://docs.flyte.org/projects/flytekit/en/latest/contributing.html">Contribution Guide</a>
-    </h3>
-</html>
+<p align="center">
+    <img src="https://raw.githubusercontent.com/flyteorg/static-resources/main/common/flyte_circle_gradient_1_4x4.png" alt="Flyte Logo" width="100">
+</p>
+<h1 align="center">
+    Flytekit Python
+</h1>
+<p align="center">
+    Flytekit Python is the Python SDK built on top of Flyte
+</p>
+<h3 align="center">
+    <a href="plugins/README.md">Plugins</a>
+    <span> · </span>
+    <a href="https://docs.flyte.org/projects/flytekit/en/latest/contributing.html">Contribution Guide</a>
+</h3>
 
 [![PyPI version fury.io](https://badge.fury.io/py/flytekit.svg)](https://pypi.python.org/pypi/flytekit/)
 [![PyPI download day](https://img.shields.io/pypi/dd/flytekit.svg)](https://pypi.python.org/pypi/flytekit/)

diff --git a/dev-requirements.in b/dev-requirements.in
@@ -11,3 +11,4 @@ codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
+torch
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -28,25 +28,25 @@ binaryornot==0.4.4
     #   cookiecutter
 cached-property==1.5.2
     # via docker-compose
-cachetools==5.1.0
+cachetools==5.2.0
     # via google-auth
-certifi==2022.5.18.1
+certifi==2022.6.15
     # via
     #   -c requirements.txt
     #   requests
-cffi==1.15.0
+cffi==1.15.1
     # via
     #   -c requirements.txt
     #   bcrypt
     #   cryptography
     #   pynacl
 cfgv==3.3.1
     # via pre-commit
-chardet==4.0.0
+chardet==5.0.0
     # via
     #   -c requirements.txt
     #   binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.1.0
     # via
     #   -c requirements.txt
     #   requests
@@ -61,20 +61,21 @@ cloudpickle==2.1.0
     #   flytekit
 codespell==2.1.0
     # via -r dev-requirements.in
-cookiecutter==1.7.3
+cookiecutter==2.1.1
     # via
     #   -c requirements.txt
     #   flytekit
-coverage[toml]==6.3.3
+coverage[toml]==6.4.1
     # via -r dev-requirements.in
 croniter==1.3.5
     # via
     #   -c requirements.txt
     #   flytekit
-cryptography==37.0.2
+cryptography==37.0.4
     # via
     #   -c requirements.txt
     #   paramiko
+    #   pyopenssl
     #   secretstorage
 dataclasses-json==0.5.7
     # via
@@ -118,47 +119,47 @@ docstring-parser==0.14.1
     # via
     #   -c requirements.txt
     #   flytekit
-filelock==3.7.0
+filelock==3.7.1
     # via virtualenv
-flyteidl==1.0.1
+flyteidl==1.1.8
     # via
     #   -c requirements.txt
     #   flytekit
-google-api-core[grpc]==2.8.0
+google-api-core[grpc]==2.8.2
     # via
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-core
-google-auth==2.6.6
+google-auth==2.9.0
     # via
     #   google-api-core
     #   google-cloud-core
-google-cloud-bigquery==3.1.0
+google-cloud-bigquery==3.2.0
     # via -r dev-requirements.in
-google-cloud-bigquery-storage==2.13.1
+google-cloud-bigquery-storage==2.13.2
     # via
     #   -r dev-requirements.in
     #   google-cloud-bigquery
-google-cloud-core==2.3.0
+google-cloud-core==2.3.1
     # via google-cloud-bigquery
 google-crc32c==1.3.0
     # via google-resumable-media
 google-resumable-media==2.3.3
     # via google-cloud-bigquery
-googleapis-common-protos==1.56.1
+googleapis-common-protos==1.56.3
     # via
     #   -c requirements.txt
     #   flyteidl
     #   google-api-core
     #   grpcio-status
-grpcio==1.46.1
+grpcio==1.47.0
     # via
     #   -c requirements.txt
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
     #   grpcio-status
-grpcio-status==1.46.1
+grpcio-status==1.47.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -169,10 +170,11 @@ idna==3.3
     # via
     #   -c requirements.txt
     #   requests
-importlib-metadata==4.11.3
+importlib-metadata==4.12.0
     # via
     #   -c requirements.txt
     #   click
+    #   flytekit
     #   jsonschema
     #   keyring
     #   pluggy
@@ -181,7 +183,7 @@ importlib-metadata==4.11.3
     #   virtualenv
 iniconfig==1.1.1
     # via pytest
-ipython==7.33.0
+ipython==7.34.0
     # via -r dev-requirements.in
 jedi==0.18.1
     # via ipython
@@ -206,15 +208,15 @@ jsonschema==3.2.0
     # via
     #   -c requirements.txt
     #   docker-compose
-keyring==23.5.0
+keyring==23.6.0
     # via
     #   -c requirements.txt
     #   flytekit
 markupsafe==2.1.1
     # via
     #   -c requirements.txt
     #   jinja2
-marshmallow==3.15.0
+marshmallow==3.17.0
     # via
     #   -c requirements.txt
     #   dataclasses-json
@@ -232,7 +234,7 @@ matplotlib-inline==0.1.3
     # via ipython
 mock==4.0.3
     # via -r dev-requirements.in
-mypy==0.950
+mypy==0.961
     # via -r dev-requirements.in
 mypy-extensions==0.4.3
     # via
@@ -243,7 +245,7 @@ natsort==8.1.0
     # via
     #   -c requirements.txt
     #   flytekit
-nodeenv==1.6.0
+nodeenv==1.7.0
     # via pre-commit
 numpy==1.21.6
     # via
@@ -273,15 +275,11 @@ platformdirs==2.5.2
     # via virtualenv
 pluggy==1.0.0
     # via pytest
-poyo==0.5.0
-    # via
-    #   -c requirements.txt
-    #   cookiecutter
 pre-commit==2.19.0
     # via -r dev-requirements.in
-prompt-toolkit==3.0.29
+prompt-toolkit==3.0.30
     # via ipython
-proto-plus==1.20.4
+proto-plus==1.20.6
     # via
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
@@ -292,6 +290,7 @@ protobuf==3.20.1
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
+    #   google-cloud-bigquery-storage
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
@@ -326,6 +325,10 @@ pygments==2.12.0
     # via ipython
 pynacl==1.5.0
     # via paramiko
+pyopenssl==22.0.0
+    # via
+    #   -c requirements.txt
+    #   flytekit
 pyparsing==3.0.9
     # via
     #   -c requirements.txt
@@ -373,14 +376,15 @@ pytz==2022.1
 pyyaml==5.4.1
     # via
     #   -c requirements.txt
+    #   cookiecutter
     #   docker-compose
     #   flytekit
     #   pre-commit
-regex==2022.4.24
+regex==2022.6.2
     # via
     #   -c requirements.txt
     #   docker-image-py
-requests==2.27.1
+requests==2.28.1
     # via
     #   -c requirements.txt
     #   cookiecutter
@@ -390,7 +394,7 @@ requests==2.27.1
     #   google-api-core
     #   google-cloud-bigquery
     #   responses
-responses==0.20.0
+responses==0.21.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -411,7 +415,6 @@ singledispatchmethod==1.0
 six==1.16.0
     # via
     #   -c requirements.txt
-    #   cookiecutter
     #   dockerpty
     #   google-auth
     #   grpcio
@@ -441,19 +444,23 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
-traitlets==5.2.1.post0
+torch==1.11.0
+    # via -r dev-requirements.in
+traitlets==5.3.0
     # via
     #   ipython
     #   matplotlib-inline
-typed-ast==1.5.3
+typed-ast==1.5.4
     # via mypy
-typing-extensions==4.2.0
+typing-extensions==4.3.0
     # via
     #   -c requirements.txt
     #   arrow
     #   flytekit
     #   importlib-metadata
     #   mypy
+    #   responses
+    #   torch
     #   typing-inspect
 typing-inspect==0.7.1
     # via
@@ -465,7 +472,7 @@ urllib3==1.26.9
     #   flytekit
     #   requests
     #   responses
-virtualenv==20.14.1
+virtualenv==20.15.1
     # via pre-commit
 wcwidth==0.2.5
     # via prompt-toolkit

diff --git a/doc-requirements.in b/doc-requirements.in
@@ -33,3 +33,4 @@ papermill                   # papermill
 jupyter                     # papermill
 pyspark                     # spark
 sqlalchemy                  # sqlalchemy
+torch                       # pytorch
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,3 +11,4 @@ codespell @@
     google-cloud-bigquery
     google-cloud-bigquery-storage
     IPython
+    torch