Bring in buf protobufs

.github/workflows/pythonbuild.yml

@@ -49,7 +49,7 @@ jobs:
           pip freeze
       - name: Test with coverage
         run: |
-          pytest tests/flytekit/unit -m "not sandbox_test" --cov=./ --cov-report=xml
+          make unit_test_codecov
       - name: Codecov
         uses: codecov/[email protected]
         with:
@@ -177,4 +177,6 @@ jobs:
           python -m pip install --upgrade pip==21.2.4 setuptools wheel
           pip install -r doc-requirements.txt
       - name: Build the documentation
-        run: make -C docs html
+        run: |
+          # TODO: Remove after buf migration is done and packages updated
+          PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python make -C docs html

Makefile

@@ -49,9 +49,18 @@ spellcheck:  ## Runs a spellchecker over all code and documentation
 .PHONY: test
 test: lint unit_test
 
+.PHONY: unit_test_codecov
+unit_test_codecov:
+	# Ensure coverage file
+	rm coverage.xml || true
+	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test
+
 .PHONY: unit_test
 unit_test:
-	pytest -m "not sandbox_test" tests/flytekit/unit
+	# Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower)
+	# library is used to serialize/deserialize protobufs is used.
+	pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \
+		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
 
 requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
 requirements-spark2.txt: requirements-spark2.in install-piptools

dev-requirements.in

@@ -12,8 +12,7 @@ codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
-tensorflow
-grpcio-status<1.49.0
+tensorflow==2.8.1
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1

dev-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
 #
 #    make dev-requirements.txt
 #
@@ -12,8 +12,6 @@ absl-py==1.3.0
     # via
     #   tensorboard
     #   tensorflow
-appnope==0.1.3
-    # via ipython
 arrow==1.2.3
     # via
     #   -c requirements.txt
@@ -34,9 +32,11 @@ binaryornot==0.4.4
     # via
     #   -c requirements.txt
     #   cookiecutter
+cached-property==1.5.2
+    # via docker-compose
 cachetools==5.2.0
     # via google-auth
-certifi==2022.9.24
+certifi==2022.12.7
     # via
     #   -c requirements.txt
     #   requests
@@ -47,7 +47,7 @@ cffi==1.15.1
     #   pynacl
 cfgv==3.3.1
     # via pre-commit
-chardet==5.0.0
+chardet==5.1.0
     # via
     #   -c requirements.txt
     #   binaryornot
@@ -74,15 +74,16 @@ coverage[toml]==6.5.0
     # via
     #   -r dev-requirements.in
     #   pytest-cov
-croniter==1.3.7
+croniter==1.3.8
     # via
     #   -c requirements.txt
     #   flytekit
-cryptography==38.0.3
+cryptography==38.0.4
     # via
     #   -c requirements.txt
     #   paramiko
     #   pyopenssl
+    #   secretstorage
 dataclasses-json==0.5.7
     # via
     #   -c requirements.txt
@@ -125,20 +126,22 @@ docstring-parser==0.15
     #   flytekit
 exceptiongroup==1.0.4
     # via pytest
-filelock==3.8.0
+filelock==3.8.2
     # via virtualenv
-flyteidl==1.2.5
+flatbuffers==22.12.6
+    # via tensorflow
+flyteidl==1.3.0
     # via
     #   -c requirements.txt
     #   flytekit
 gast==0.5.3
     # via tensorflow
-google-api-core[grpc]==2.10.2
+google-api-core[grpc]==2.11.0
     # via
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-core
-google-auth==2.14.1
+google-auth==2.15.0
     # via
     #   google-api-core
     #   google-auth-oauthlib
@@ -166,7 +169,7 @@ googleapis-common-protos==1.57.0
     #   flyteidl
     #   google-api-core
     #   grpcio-status
-grpcio==1.48.2
+grpcio==1.51.1
     # via
     #   -c requirements.txt
     #   flytekit
@@ -175,10 +178,9 @@ grpcio==1.48.2
     #   grpcio-status
     #   tensorboard
     #   tensorflow
-grpcio-status==1.48.2
+grpcio-status==1.51.1
     # via
     #   -c requirements.txt
-    #   -r dev-requirements.in
     #   flytekit
     #   google-api-core
 h5py==3.7.0
@@ -189,12 +191,18 @@ idna==3.4
     # via
     #   -c requirements.txt
     #   requests
-importlib-metadata==5.0.0
+importlib-metadata==5.1.0
     # via
     #   -c requirements.txt
+    #   click
     #   flytekit
+    #   jsonschema
     #   keyring
     #   markdown
+    #   pluggy
+    #   pre-commit
+    #   pytest
+    #   virtualenv
 iniconfig==1.1.1
     # via pytest
 ipython==7.34.0
@@ -203,8 +211,13 @@ jaraco-classes==3.2.3
     # via
     #   -c requirements.txt
     #   keyring
-jedi==0.18.1
+jedi==0.18.2
     # via ipython
+jeepney==0.8.0
+    # via
+    #   -c requirements.txt
+    #   keyring
+    #   secretstorage
 jinja2==3.1.2
     # via
     #   -c requirements.txt
@@ -280,19 +293,20 @@ nodeenv==1.7.0
 numpy==1.21.6
     # via
     #   -c requirements.txt
+    #   flytekit
     #   h5py
     #   keras-preprocessing
     #   opt-einsum
     #   pandas
     #   pyarrow
+    #   scikit-learn
+    #   scipy
     #   tensorboard
     #   tensorflow
 oauthlib==3.2.2
     # via requests-oauthlib
 opt-einsum==3.3.0
     # via tensorflow
-    #   scikit-learn
-    #   scipy
 packaging==21.3
     # via
     #   -c requirements.txt
@@ -312,23 +326,22 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.5.4
+platformdirs==2.6.0
     # via virtualenv
 pluggy==1.0.0
     # via pytest
 pre-commit==2.20.0
     # via -r dev-requirements.in
-prompt-toolkit==3.0.32
+prompt-toolkit==3.0.36
     # via ipython
 proto-plus==1.22.1
     # via
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
-protobuf==3.20.3
+protobuf==4.21.10
     # via
     #   -c requirements.txt
     #   flyteidl
-    #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
@@ -348,7 +361,7 @@ py==1.11.0
     # via
     #   -c requirements.txt
     #   retry
-pyarrow==10.0.0
+pyarrow==10.0.1
     # via
     #   -c requirements.txt
     #   flytekit
@@ -457,6 +470,10 @@ scikit-learn==1.0.2
     # via -r dev-requirements.in
 scipy==1.7.3
     # via scikit-learn
+secretstorage==3.3.3
+    # via
+    #   -c requirements.txt
+    #   keyring
 singledispatchmethod==1.0
     # via
     #   -c requirements.txt
@@ -468,7 +485,6 @@ six==1.16.0
     #   dockerpty
     #   google-auth
     #   google-pasta
-    #   grpcio
     #   jsonschema
     #   keras-preprocessing
     #   paramiko
@@ -493,15 +509,15 @@ tensorflow==2.8.1
     # via -r dev-requirements.in
 tensorflow-estimator==2.8.0
     # via tensorflow
-tensorflow-io-gcs-filesystem==0.27.0
+tensorflow-io-gcs-filesystem==0.28.0
     # via tensorflow
-termcolor==2.0.1
+termcolor==2.1.1
     # via tensorflow
 text-unidecode==1.3
     # via
     #   -c requirements.txt
     #   python-slugify
-texttable==1.6.4
+texttable==1.6.7
     # via docker-compose
 threadpoolctl==3.1.0
     # via scikit-learn
@@ -517,7 +533,7 @@ tomli==2.0.1
     #   pytest
 torch==1.12.1
     # via -r dev-requirements.in
-traitlets==5.5.0
+traitlets==5.6.0
     # via
     #   ipython
     #   matplotlib-inline
@@ -530,23 +546,26 @@ types-toml==0.10.8.1
 typing-extensions==4.4.0
     # via
     #   -c requirements.txt
+    #   arrow
     #   flytekit
+    #   importlib-metadata
     #   mypy
+    #   responses
     #   tensorflow
     #   torch
     #   typing-inspect
 typing-inspect==0.8.0
     # via
     #   -c requirements.txt
     #   dataclasses-json
-urllib3==1.26.12
+urllib3==1.26.13
     # via
     #   -c requirements.txt
     #   docker
     #   flytekit
     #   requests
     #   responses
-virtualenv==20.16.7
+virtualenv==20.17.1
     # via pre-commit
 wcwidth==0.2.5
     # via prompt-toolkit
@@ -555,6 +574,8 @@ websocket-client==0.59.0
     #   -c requirements.txt
     #   docker
     #   docker-compose
+werkzeug==2.2.2
+    # via tensorboard
 wheel==0.38.4
     # via
     #   -c requirements.txt
@@ -567,7 +588,7 @@ wrapt==1.14.1
     #   deprecated
     #   flytekit
     #   tensorflow
-zipp==3.10.0
+zipp==3.11.0
     # via
     #   -c requirements.txt
     #   importlib-metadata

doc-requirements.in

@@ -12,19 +12,17 @@ sphinx-copybutton
 sphinx_fontawesome
 sphinx-panels
 sphinxcontrib-yt
-grpcio==1.43.0
-grpcio-status==1.43.0
 cryptography
-google-api-core[grpc]==2.8.2
-scikit-learn==1.1.1
+google-api-core[grpc]
+scikit-learn
 sphinx-tags
 
 # Packages for Plugin docs
 # Package name              Plugin needing it
 botocore                    # fsspec
 fsspec                      # fsspec
 google-cloud                # bigquery
-google-cloud-bigquery==3.1.0 # bigquery
+google-cloud-bigquery       # bigquery
 markdown                    # deck
 plotly                      # deck
 pandas_profiling            # deck
@@ -38,9 +36,10 @@ jupyter                     # papermill
 pyspark                     # spark
 sqlalchemy                  # sqlalchemy
 torch                       # pytorch
-skl2onnx                    # onnxscikitlearn
-tf2onnx                     # onnxtensorflow
-tensorflow==2.9.0           # onnxtensorflow
+# TODO: Remove after buf migration is done and packages updated
+# skl2onnx                    # onnxscikitlearn
+# tf2onnx                     # onnxtensorflow
+tensorflow==2.8.1           # onnxtensorflow
 whylogs                     # whylogs
 whylabs-client              # whylogs
 ray                         # ray