AIP-4600: [Part 2 of 3] Add in publishing to ZG Artifactory

.gitlab-ci.yml

@@ -0,0 +1,36 @@
+include:
+  - project: 'analytics/artificial-intelligence/ai-platform/aip-infrastructure/ci-templates/ci-cd-template'
+    ref: &include_ref 'alexla/AIP-4600'  # TODO AIP-4600 Replace with v2
+    file: '/blocks/python.yml'
+
+stages:
+  - build
+
+build:publish:
+  extends: .aip_python_debian_image
+  stage: build
+  variables:
+    PY_LIBRARY_NAME: 'zillow-kfp'
+    KFP_VERSION_PATH: "sdk/python/kfp/__init__.py"
+  script:
+  - KFP_VERSION=$(cat $KFP_VERSION_PATH | sed -nr "s/^__version__ = ['\"]([^'\"]*)['\"]$/\1/p")
+  - |
+    if [ "${CI_COMMIT_BRANCH}" = "${CI_DEFAULT_BRANCH}" ]; then
+      PY_LIBRARY_VERSION="${KFP_VERSION}.${CI_PIPELINE_IID}"
+    else
+      PY_LIBRARY_VERSION="0.0.${CI_PIPELINE_IID}+${CI_COMMIT_SHORT_SHA}"
+    fi
+  # Now write back the zillow-kfp version back to the original location we found the original.
+  - sed -i "s/\(__version__ = ['\"]\)[^'\"]*\(['\"]\)/\1${PY_LIBRARY_VERSION}\2/" $KFP_VERSION_PATH
+  - python setup.py sdist
+  # Set up the configuration for Artifactory to publish the python package internally.
+  - |
+    cat >~/.pypirc <<EOL
+    [distutils]
+    index-servers = artifactory
+    [artifactory]
+    repository: ${ANALYTICS_PYPI_REPOSITORY}
+    username: ${PYPI_USERNAME}
+    password: ${PYPI_PASSWORD}
+    EOL
+  - python setup.py sdist upload --repository "${ANALYTICS_PYPI_REPOSITORY}"

sdk/python/kfp/compiler/compiler.py

@@ -926,7 +926,7 @@ def _create_workflow(
                 default=default_param_values[param.name]))
 
     op_transformers = [add_pod_env]
-    pod_labels = {_SDK_VERSION_LABEL: kfp.__version__, _SDK_ENV_LABEL:_SDK_ENV_DEFAULT}
+    pod_labels = {_SDK_VERSION_LABEL: kfp.__version__.replace('+', '-'), _SDK_ENV_LABEL:_SDK_ENV_DEFAULT}
     op_transformers.append(add_pod_labels(pod_labels))
     op_transformers.extend(pipeline_conf.op_transformers)
 
@@ -974,7 +974,7 @@ def _create_workflow(
 
 
     # Labels might be logged better than annotations so adding some information here as well
-    labels[_SDK_VERSION_LABEL] = kfp.__version__
+    labels[_SDK_VERSION_LABEL] = kfp.__version__.replace('+', '-')
 
     return workflow
 

sdk/python/setup.py

@@ -16,7 +16,7 @@
 import re
 from setuptools import setup
 
-NAME = 'kfp'
+NAME = 'zillow-kfp'  # 'kfp'
 #VERSION = .... Change the version in kfp/__init__.py
 
 # NOTICE, after any updates to the following, ./requirements.in should be updated