chore(mlmd): Upgrade ML Metadata to 1.14.0.

kubeflow · Aug 10, 2023 · 96fa73a · 96fa73a
1 parent b350ac4
commit 96fa73a
Show file tree

Hide file tree

Showing 26 changed files with 26,171 additions and 11,580 deletions.
diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
@@ -158,7 +158,7 @@ steps:
 # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
 # * test/tag_for_hosted.sh
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0']
+  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0']
   id:   'pullMetadataServer'
 - name: 'gcr.io/cloud-builders/docker'
   args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']

diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml
@@ -259,14 +259,14 @@ steps:
 # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
 # * test/tag_for_hosted.sh
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0']
+  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0']
   id:   'pullMetadataServer'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME']
+  args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME']
   id:   'tagMetadataServerForMarketplace'
   waitFor: ['pullMetadataServer']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0', 'gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$TAG_NAME']
+  args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0', 'gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$TAG_NAME']
   id:   'tagMetadataServerForMarketplaceTest'
   waitFor: ['pullMetadataServer']
 - id:   'tagMetadataServerForMarketplaceMajorMinor'
@@ -276,8 +276,8 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver)
-    docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver)
+    docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver)
+    docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver)
 

diff --git a/backend/metadata_writer/requirements.in b/backend/metadata_writer/requirements.in
@@ -1,3 +1,3 @@
 kubernetes>=8.0.0,<11.0.0
-ml-metadata==1.5.0
+ml-metadata==1.14.0
 lru-dict>=1.1.7,<2.0.0
diff --git a/backend/src/apiserver/visualization/requirements.txt b/backend/src/apiserver/visualization/requirements.txt
@@ -1,153 +0,0 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file=- -
-#
-absl-py==0.12.0           # via tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tfx-bsl
-apache-beam[gcp]==2.34.0  # via tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl
-argon2-cffi-bindings==21.2.0  # via argon2-cffi
-argon2-cffi==21.3.0       # via notebook
-astunparse==1.6.3         # via tensorflow
-attrs==21.2.0             # via jsonschema
-avro-python3==1.9.2.1     # via apache-beam
-backcall==0.2.0           # via ipython
-bleach==4.1.0             # via nbconvert
-bokeh==1.2.0              # via -r -
-cached-property==1.5.2    # via h5py
-cachetools==4.2.4         # via apache-beam, google-auth
-certifi==2021.10.8        # via requests
-cffi==1.15.0              # via argon2-cffi-bindings
-charset-normalizer==2.0.9  # via requests
-crcmod==1.7               # via apache-beam
-dataclasses==0.8          # via apache-beam, argon2-cffi, libcst, werkzeug
-decorator==5.1.0          # via gcsfs, ipython, traitlets
-defusedxml==0.7.1         # via nbconvert
-dill==0.3.1.1             # via apache-beam
-docopt==0.6.2             # via hdfs
-entrypoints==0.3          # via nbconvert
-fastavro==1.4.7           # via apache-beam
-fasteners==0.16.3         # via google-apitools
-flatbuffers==1.12         # via tensorflow
-future==0.18.2            # via apache-beam
-gast==0.4.0               # via tensorflow
-gcsfs==0.2.3              # via -r -
-google-api-core[grpc,grpcgcp]==1.31.5  # via google-cloud-bigquery, google-cloud-bigquery-storage, google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-dlp, google-cloud-language, google-cloud-pubsub, google-cloud-recommendations-ai, google-cloud-spanner, google-cloud-videointelligence, google-cloud-vision
-google-api-python-client==1.7.12  # via -r -, tfx-bsl
-google-apitools==0.5.31   # via apache-beam
-google-auth-httplib2==0.1.0  # via google-api-python-client
-google-auth-oauthlib==0.4.6  # via gcsfs, tensorboard
-google-auth==1.35.0       # via apache-beam, gcsfs, google-api-core, google-api-python-client, google-auth-httplib2, google-auth-oauthlib, google-cloud-core, tensorboard
-google-cloud-bigquery-storage==2.10.1  # via apache-beam
-google-cloud-bigquery==2.20.0  # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl
-google-cloud-bigtable==1.7.0  # via apache-beam
-google-cloud-core==1.7.2  # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore, google-cloud-spanner
-google-cloud-datastore==1.15.3  # via apache-beam
-google-cloud-dlp==1.0.0   # via apache-beam
-google-cloud-language==1.3.0  # via apache-beam
-google-cloud-pubsub==1.7.0  # via apache-beam
-google-cloud-recommendations-ai==0.2.0  # via apache-beam
-google-cloud-spanner==1.19.1  # via apache-beam
-google-cloud-videointelligence==1.16.1  # via apache-beam
-google-cloud-vision==1.0.0  # via apache-beam
-google-crc32c==1.3.0      # via google-resumable-media
-google-pasta==0.2.0       # via tensorflow
-google-resumable-media==1.3.3  # via google-cloud-bigquery
-googleapis-common-protos[grpc]==1.54.0  # via google-api-core, grpc-google-iam-v1, tensorflow-metadata
-grpc-google-iam-v1==0.12.3  # via google-cloud-bigtable, google-cloud-pubsub, google-cloud-spanner
-grpcio-gcp==0.2.2         # via apache-beam, google-api-core
-grpcio==1.34.1            # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, grpcio-gcp, tensorboard, tensorflow, tensorflow-serving-api
-h5py==3.1.0               # via tensorflow
-hdfs==2.6.0               # via apache-beam
-httplib2==0.19.1          # via apache-beam, google-api-python-client, google-apitools, google-auth-httplib2, oauth2client
-idna==3.3                 # via requests
-importlib-metadata==4.8.3  # via jsonschema, markdown
-ipykernel==5.1.1          # via -r -, ipywidgets, notebook
-ipython-genutils==0.2.0   # via ipywidgets, nbformat, notebook, traitlets
-ipython==7.12.0           # via -r -, ipykernel, ipywidgets, tensorflow-model-analysis
-ipywidgets==7.6.5         # via tensorflow-model-analysis
-itables==0.1.0            # via -r -
-jedi==0.18.1              # via ipython
-jinja2==3.0.3             # via bokeh, nbconvert, notebook
-joblib==0.14.1            # via scikit-learn, tensorflow-data-validation
-jsonschema==3.2.0         # via nbformat
-jupyter-client==5.3.5     # via -r -, ipykernel, notebook
-jupyter-core==4.9.1       # via jupyter-client, nbconvert, nbformat, notebook
-jupyterlab-widgets==1.0.2  # via ipywidgets
-keras-nightly==2.5.0.dev2021032900  # via tensorflow
-keras-preprocessing==1.1.2  # via tensorflow
-libcst==0.3.23            # via google-cloud-bigquery-storage
-markdown==3.3.6           # via tensorboard
-markupsafe==2.0.1         # via jinja2
-mistune==0.8.4            # via -r -, nbconvert
-mypy-extensions==0.4.3    # via typing-inspect
-nbconvert==5.5.0          # via -r -, notebook
-nbformat==4.4.0           # via -r -, ipywidgets, nbconvert, notebook
-nest-asyncio==1.5.4       # via notebook
-notebook==6.4.6           # via widgetsnbextension
-numpy==1.19.5             # via apache-beam, bokeh, h5py, keras-preprocessing, opt-einsum, pandas, pyarrow, scikit-learn, scipy, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl
-oauth2client==4.1.3       # via apache-beam, google-apitools
-oauthlib==3.1.1           # via requests-oauthlib
-opt-einsum==3.3.0         # via tensorflow
-orjson==3.6.1             # via apache-beam
-packaging==21.3           # via bleach, bokeh, google-api-core, google-cloud-bigquery
-pandas==1.1.5             # via itables, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl
-pandocfilters==1.5.0      # via nbconvert
-parso==0.8.3              # via jedi
-pexpect==4.8.0            # via ipython
-pickleshare==0.7.5        # via ipython
-pillow==8.4.0             # via bokeh
-prometheus-client==0.12.0  # via notebook
-prompt-toolkit==3.0.24    # via ipython
-proto-plus==1.19.8        # via google-cloud-bigquery, google-cloud-bigquery-storage, google-cloud-recommendations-ai
-protobuf==3.19.1          # via apache-beam, google-api-core, google-cloud-bigquery, googleapis-common-protos, proto-plus, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tensorflow-serving-api, tfx-bsl
-ptyprocess==0.7.0         # via pexpect, terminado
-pyarrow==2.0.0            # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl
-pyasn1-modules==0.2.8     # via google-auth, oauth2client
-pyasn1==0.4.8             # via oauth2client, pyasn1-modules, rsa
-pycparser==2.21           # via cffi
-pydot==1.4.2              # via apache-beam
-pygments==2.10.0          # via ipython, nbconvert
-pymongo==3.12.3           # via apache-beam
-pyparsing==2.4.7          # via httplib2, packaging, pydot
-pyrsistent==0.18.0        # via jsonschema
-python-dateutil==2.8.2    # via apache-beam, bokeh, jupyter-client, pandas
-pytz==2021.3              # via apache-beam, google-api-core, pandas
-pyyaml==6.0               # via bokeh, libcst
-pyzmq==22.3.0             # via jupyter-client, notebook
-requests-oauthlib==1.3.0  # via google-auth-oauthlib
-requests==2.26.0          # via apache-beam, gcsfs, google-api-core, google-cloud-bigquery, hdfs, requests-oauthlib, tensorboard
-rsa==4.8                  # via google-auth, oauth2client
-scikit_learn==0.21.2      # via -r -
-scipy==1.5.4              # via scikit-learn, tensorflow-model-analysis
-send2trash==1.8.0         # via notebook
-six==1.15.0               # via absl-py, astunparse, bleach, bokeh, fasteners, google-api-core, google-api-python-client, google-apitools, google-auth, google-auth-httplib2, google-cloud-core, google-pasta, google-resumable-media, grpcio, hdfs, jsonschema, keras-preprocessing, oauth2client, python-dateutil, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, traitlets
-tensorboard-data-server==0.6.1  # via tensorboard
-tensorboard-plugin-wit==1.8.0  # via tensorboard
-tensorboard==2.7.0        # via tensorflow
-tensorflow-data-validation==1.2.0  # via -r -
-tensorflow-estimator==2.5.0  # via tensorflow
-tensorflow-metadata==1.2.0  # via -r -, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl
-tensorflow-model-analysis==0.33.0  # via -r -
-tensorflow-serving-api==2.5.1  # via -r -, tfx-bsl
-tensorflow==2.5.1         # via -r -, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-serving-api, tfx-bsl
-termcolor==1.1.0          # via tensorflow
-terminado==0.12.1         # via notebook
-testpath==0.5.0           # via nbconvert
-tfx-bsl==1.2.0            # via tensorflow-data-validation, tensorflow-model-analysis
-tornado==6.1              # via -r -, bokeh, ipykernel, jupyter-client, notebook, terminado
-traitlets==4.3.3          # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook
-typing-extensions==3.7.4.3  # via apache-beam, argon2-cffi, importlib-metadata, libcst, tensorflow, typing-inspect
-typing-inspect==0.7.1     # via libcst
-uritemplate==3.0.1        # via google-api-python-client
-urllib3==1.26.7           # via requests
-wcwidth==0.2.5            # via prompt-toolkit
-webencodings==0.5.1       # via bleach
-werkzeug==2.0.2           # via tensorboard
-wheel==0.37.1             # via astunparse, tensorboard, tensorflow
-widgetsnbextension==3.5.2  # via ipywidgets
-wrapt==1.12.1             # via tensorflow
-zipp==3.6.0               # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

diff --git a/backend/src/apiserver/visualization/update_requirements.sh b/backend/src/apiserver/visualization/update_requirements.sh
@@ -1,9 +1,15 @@
 #!/bin/bash
+set -euo pipefail
 
 # This image should be in sync with Dockerfile.visualization.
 IMAGE="tensorflow/tensorflow:2.5.1"
+
 # tensorflow/tfx default entrypoint is Apache BEAM, because Apache BEAM doesn't
 # support custom entrypoint for now. We need to override with --entrypoint ""
 # for other `docker run` usecase.
 # https://github.com/tensorflow/tfx/blob/master/tfx/tools/docker/Dockerfile#L71
-../../../update_requirements.sh $IMAGE <requirements.in >requirements.txt
+docker run -i --rm  --entrypoint "" "$IMAGE" sh -c '
+  python3 -m pip install pip setuptools --upgrade --quiet
+  python3 -m pip install pip-tools==5.4.0 --quiet
+  pip-compile --verbose --output-file - -
+' <requirements.in >requirements.txt
diff --git a/backend/src/v2/test/requirements.txt b/backend/src/v2/test/requirements.txt
@@ -2,7 +2,7 @@
 -e ../../../../sdk/python
 # TODO(chensun): remove the deprecated dependencies once migrated tests.
 -r ../../../../sdk/python/requirements-deprecated.txt
-ml-metadata==1.5.0
+ml-metadata==1.14.0
 minio==7.0.4
 google-cloud-storage
 fire

diff --git a/frontend/README.md b/frontend/README.md
@@ -176,8 +176,22 @@ If a file in [pipelines/third_party/ml-metadata/ml_metadata/proto](third_party/m
 
 * Add `protoc` ([download](https://github.com/protocolbuffers/protobuf/releases)) to your system
   PATH
+
+  ```bash
+  # Example:
+  apt install -y protobuf-compiler=3.15.8
+  ```
+
 * Add `protoc-gen-grpc-web` ([download](https://github.com/grpc/grpc-web/releases)) to your system
   PATH
+
+  ```bash
+  # Example:
+  curl -LO https://github.com/grpc/grpc-web/releases/download/1.4.2/protoc-gen-grpc-web-1.4.2-linux-x86_64
+  mv protoc-gen-grpc-web-1.4.2-linux-x86_64 /usr/local/bin/protoc-gen-grpc-web
+  chmod +x /usr/local/bin/protoc-gen-grpc-web
+  ```
+
 * Replace `metadata_store.proto` and `metadata_store_service.proto` proto files with target mlmd version by running
 
   ```bash