AIP-4600: [Part 2 of 3] Add in publishing to ZG Artifactory

.gitlab-ci.yml

@@ -0,0 +1,38 @@
+include:
+  - project: 'analytics/artificial-intelligence/ai-platform/aip-infrastructure/ci-templates/ci-cd-template'
+    ref: &include_ref 'v2'
+    file: '/blocks/python.yml'
+
+stages:
+  - build
+
+build:publish:
+  extends: .aip_python_debian_image
+  stage: build
+  variables:
+    MAJOR_VERSION: "1"
+    MINOR_VERSION: "0"
+    PY_LIBRARY_NAME: 'zillow-kfp'
+    KFP_VERSION_PATH: "sdk/python/kfp/__init__.py"
+  script:
+  - KFP_VERSION=$(cat $KFP_VERSION_PATH | sed -nr "s/^__version__ = ['\"]([^'\"]*)['\"]$/\1/p")
+  - |
+    if [ "${CI_COMMIT_BRANCH}" = "${CI_DEFAULT_BRANCH}" ]; then
+      PY_LIBRARY_VERSION="${MAJOR_VERSION}.${MINOR_VERSION}.${CI_PIPELINE_IID}.${KFP_VERSION}"
+    else
+      PY_LIBRARY_VERSION="0.0.${CI_PIPELINE_IID}.${KFP_VERSION}"
+    fi
+  # Now write back the zillow-kfp version back to the original location we found the original.
+  - sed -i "s/\(__version__ = ['\"]\)[^'\"]*\(['\"]\)/\1${PY_LIBRARY_VERSION}\2/" $KFP_VERSION_PATH
+  - python setup.py sdist
+  # Set up the configuration for Artifactory to publish the python package internally.
+  - |
+    cat >~/.pypirc <<EOL
+    [distutils]
+    index-servers = artifactory
+    [artifactory]
+    repository: ${ANALYTICS_PYPI_REPOSITORY}
+    username: ${PYPI_USERNAME}
+    password: ${PYPI_PASSWORD}
+    EOL
+  - python setup.py sdist upload --repository "${ANALYTICS_PYPI_REPOSITORY}"

sdk/RELEASE.md

@@ -13,6 +13,12 @@
 
 ## Bug Fixes and Other Changes
 
+* Remove dead code on importer check in v1. [\#6508](https://github.com/kubeflow/pipelines/pull/6508)
+* Fix issue where dict, list, bool typed input parameters don't accept constant values or pipeline inputs. [\#6523](https://github.com/kubeflow/pipelines/pull/6523)
+* Fix passing in "" to a str parameter causes the parameter to receive it as None instead. [\#6533](https://github.com/kubeflow/pipelines/pull/6533)
+* Depends on `kfp-pipeline-spec>=0.1.10,<0.2.0` [\#6515](https://github.com/kubeflow/pipelines/pull/6515)
+* Depends on kubernetes>=8.0.0,<19. [\#6532](https://github.com/kubeflow/pipelines/pull/6532)
+* Get short name of complex input/output types to ensure we can map to appropriate de|serializer. [\#6504](https://github.com/kubeflow/pipelines/pull/6504)
 
 ## Documentation Updates
 

sdk/python/kfp/compiler/compiler.py

@@ -926,7 +926,7 @@ def _create_workflow(
                 default=default_param_values[param.name]))
 
     op_transformers = [add_pod_env]
-    pod_labels = {_SDK_VERSION_LABEL: kfp.__version__, _SDK_ENV_LABEL:_SDK_ENV_DEFAULT}
+    pod_labels = {_SDK_VERSION_LABEL: kfp.__version__.replace('+', '-'), _SDK_ENV_LABEL:_SDK_ENV_DEFAULT}
     op_transformers.append(add_pod_labels(pod_labels))
     op_transformers.extend(pipeline_conf.op_transformers)
 
@@ -974,7 +974,7 @@ def _create_workflow(
 
 
     # Labels might be logged better than annotations so adding some information here as well
-    labels[_SDK_VERSION_LABEL] = kfp.__version__
+    labels[_SDK_VERSION_LABEL] = kfp.__version__.replace('+', '-')
 
     return workflow
 

sdk/python/kfp/components/_components.py

@@ -28,7 +28,7 @@
 from ._naming import _sanitize_file_name, _sanitize_python_function_name, generate_unique_name_conversion_table
 from ._yaml_utils import load_yaml
 from .structures import *
-from ._data_passing import serialize_value, get_canonical_type_for_type_struct
+from ._data_passing import serialize_value, get_canonical_type_for_type_name
 
 
 _default_component_name = 'Component'
@@ -375,10 +375,11 @@ def component_default_to_func_default(component_default: str, is_optional: bool)
     input_parameters = [
         _dynamic.KwParameter(
             input_name_to_pythonic[port.name],
-            annotation=(get_canonical_type_for_type_struct(str(port.type)) or str(port.type) if port.type else inspect.Parameter.empty),
-            default=component_default_to_func_default(port.default, port.optional),
-        )
-        for port in reordered_input_list
+            annotation=(get_canonical_type_for_type_name(str(port.type)) or str(
+                port.type) if port.type else inspect.Parameter.empty),
+            default=component_default_to_func_default(port.default,
+                                                      port.optional),
+        ) for port in reordered_input_list
     ]
     factory_function_parameters = input_parameters #Outputs are no longer part of the task factory function signature. The paths are always generated by the system.
 

sdk/python/kfp/components/_data_passing.py

@@ -13,23 +13,22 @@
 # limitations under the License.
 
 __all__ = [
-    'get_canonical_type_struct_for_type',
-    'get_canonical_type_for_type_struct',
-    'get_deserializer_code_for_type',
-    'get_deserializer_code_for_type_struct',
-    'get_serializer_func_for_type_struct',
+    'get_canonical_type_name_for_type',
+    'get_canonical_type_for_type_name',
+    'get_deserializer_code_for_type_name',
+    'get_serializer_func_for_type_name',
 ]
 
 
 import inspect
-from typing import Any, Callable, NamedTuple, Sequence
+from typing import Any, Callable, NamedTuple, Optional, Sequence, Type
 import warnings
 
 from kfp.components import type_annotation_utils
 
 
 Converter = NamedTuple('Converter', [
-    ('types', Sequence[str]),
+    ('types', Sequence[Type]),
     ('type_names', Sequence[str]),
     ('serializer', Callable[[Any], str]),
     ('deserializer_code', str),
@@ -125,37 +124,64 @@ def _deserialize_base64_pickle(s):
 type_name_to_serializer = {type_name: converter.serializer for converter in _converters for type_name in converter.type_names}
 
 
-def get_canonical_type_struct_for_type(typ) -> str:
+def get_canonical_type_name_for_type(typ: Type) -> str:
+    """Find the canonical type name for a given type.
+
+    Args:
+        typ: The type to search for.
+
+    Returns:
+        The canonical name of the type found.
+    """
     try:
         return type_to_type_name.get(typ, None)
     except:
         return None
 
 
-def get_canonical_type_for_type_struct(type_struct) -> str:
-    try:
-        return type_name_to_type.get(type_struct, None)
-    except:
-        return None
+def get_canonical_type_for_type_name(type_name: str) -> Optional[Type]:
+    """Find the canonical type for a given type name.
 
+    Args:
+        type_name: The type name to search for.
 
-def get_deserializer_code_for_type(typ) -> str:
+    Returns:
+        The canonical type found.
+    """
     try:
-        return type_name_to_deserializer.get(get_canonical_type_struct_for_type[typ], None)
+        return type_name_to_type.get(type_name, None)
     except:
         return None
 
 
-def get_deserializer_code_for_type_struct(type_struct) -> str:
+def get_deserializer_code_for_type_name(type_name: str) -> Optional[str]:
+    """Find the deserializer code for the given type name.
+
+    Args:
+        type_name: The type name to search for.
+
+    Returns:
+        The deserializer code needed to deserialize the type.
+    """
     try:
-        return type_name_to_deserializer.get(type_struct, None)
+        return type_name_to_deserializer.get(
+            type_annotation_utils.get_short_type_name(type_name), None)
     except:
         return None
 
 
-def get_serializer_func_for_type_struct(type_struct) -> str:
+def get_serializer_func_for_type_name(type_name: str) -> Optional[Callable]:
+    """Find the serializer code for the given type name.
+
+    Args:
+        type_name: The type name to search for.
+
+    Returns:
+        The serializer func needed to serialize the type.
+    """
     try:
-        return type_name_to_serializer.get(type_struct, None)
+        return type_name_to_serializer.get(
+            type_annotation_utils.get_short_type_name(type_name), None)
     except:
         return None
 

sdk/python/kfp/components/_python_op.py

@@ -30,7 +30,7 @@
 
 from ._yaml_utils import dump_yaml
 from ._components import _create_task_factory_from_component_spec
-from ._data_passing import serialize_value, get_deserializer_code_for_type_struct, get_serializer_func_for_type_struct, get_canonical_type_struct_for_type
+from ._data_passing import serialize_value, get_deserializer_code_for_type_name, get_serializer_func_for_type_name, get_canonical_type_name_for_type
 from ._naming import _make_name_unique_by_adding_index
 from .structures import *
 from . import _structures as structures
@@ -309,7 +309,7 @@ def annotation_to_type_struct(annotation):
         if isinstance(annotation, dict):
             return annotation
         if isinstance(annotation, type):
-            type_struct = get_canonical_type_struct_for_type(annotation)
+            type_struct = get_canonical_type_name_for_type(annotation)
             if type_struct:
                 return type_struct
             type_name = str(annotation.__name__)
@@ -319,7 +319,7 @@ def annotation_to_type_struct(annotation):
             type_name = str(annotation)
 
         # It's also possible to get the converter by type name
-        type_struct = get_canonical_type_struct_for_type(type_name)
+        type_struct = get_canonical_type_name_for_type(type_name)
         if type_struct:
             return type_struct
         return type_name
@@ -496,7 +496,7 @@ def _func_to_component_spec(func, extra_code='', base_image : str = None, packag
 
     definitions = set()
     def get_deserializer_and_register_definitions(type_name):
-        deserializer_code = get_deserializer_code_for_type_struct(type_name)
+        deserializer_code = get_deserializer_code_for_type_name(type_name)
         if deserializer_code:
             (deserializer_code_str, definition_str) = deserializer_code
             if definition_str:
@@ -532,7 +532,7 @@ def get_argparse_type_for_input_file(passing_style):
         raise NotImplementedError('Unexpected data passing style: "{}".'.format(str(passing_style)))
 
     def get_serializer_and_register_definitions(type_name) -> str:
-        serializer_func = get_serializer_func_for_type_struct(type_name)
+        serializer_func = get_serializer_func_for_type_name(type_name)
         if serializer_func:
             # If serializer is not part of the standard python library, then include its code in the generated program
             if hasattr(serializer_func, '__module__') and not _module_is_builtin_or_standard(serializer_func.__module__):

sdk/python/kfp/v2/components/component_factory.py

@@ -86,8 +86,7 @@ def _annotation_to_type_struct(annotation):
     if isinstance(annotation, dict):
         return annotation
     if isinstance(annotation, type):
-        type_struct = _data_passing.get_canonical_type_struct_for_type(
-            annotation)
+        type_struct = _data_passing.get_canonical_type_name_for_type(annotation)
         if type_struct:
             return type_struct
         type_name = str(annotation.__name__)
@@ -99,7 +98,7 @@ def _annotation_to_type_struct(annotation):
         type_name = str(annotation)
 
     # It's also possible to get the converter by type name
-    type_struct = _data_passing.get_canonical_type_struct_for_type(type_name)
+    type_struct = _data_passing.get_canonical_type_name_for_type(type_name)
     if type_struct:
         return type_struct
     return type_name

sdk/python/requirements.in

@@ -13,7 +13,7 @@ docstring-parser>=0.7.3,<1
 # kfp.dsl
 PyYAML>=5.3,<6
 jsonschema>=3.0.1,<4
-kubernetes>=8.0.0,<13
+kubernetes>=8.0.0,<19
 
 # kfp.Client
 kfp-server-api>=1.1.2,<2.0.0

sdk/python/requirements.txt

@@ -4,6 +4,7 @@
 #
 #    pip-compile requirements.in
 #
+
 absl-py==0.11.0
     # via -r requirements.in
 attrs==20.3.0
@@ -68,7 +69,7 @@ kfp-pipeline-spec==0.1.9
     # via -r requirements.in
 kfp-server-api==1.3.0
     # via -r requirements.in
-kubernetes==12.0.1
+kubernetes==18.20.0
     # via -r requirements.in
 oauthlib==3.1.0
     # via requests-oauthlib

sdk/python/setup.py

@@ -16,7 +16,7 @@
 import re
 from setuptools import setup
 
-NAME = 'kfp'
+NAME = 'zillow-kfp'  # 'kfp'
 #VERSION = .... Change the version in kfp/__init__.py
 
 # NOTICE, after any updates to the following, ./requirements.in should be updated
@@ -27,7 +27,7 @@
     # `Blob.from_string` was introduced in google-cloud-storage 1.20.0
     # https://github.com/googleapis/python-storage/blob/master/CHANGELOG.md#1200
     'google-cloud-storage>=1.20.0,<2',
-    'kubernetes>=8.0.0,<13',
+    'kubernetes>=8.0.0,<19',
     # google-api-python-client v2 doesn't work for private dicovery by default:
     # https://github.com/googleapis/google-api-python-client/issues/1225#issuecomment-791058235
     'google-api-python-client>=1.7.8,<2',