Merge branch 'master' of github.com:ji-yaqi/pipelines into aliasing

.gitignore

@@ -77,3 +77,11 @@ docs/_build
 
 # Copy from kubeflow/frontend
 coverage/
+
+# Python cache
+__pycache__
+.pytest_cache
+
+# Coverage
+.coverage
+.coverage*

api/v2alpha1/pipeline_spec.proto

@@ -74,6 +74,9 @@ message PipelineSpec {
   // The definition of the main pipeline.  Execution of the pipeline is
   // completed upon the completion of this component.
   ComponentSpec root = 9;
+
+  // Optional field. The default root output directory of the pipeline.
+  string default_pipeline_root = 10;
 }
 
 // Definition of a component.
@@ -179,6 +182,9 @@ message ComponentInputsSpec {
 
     // Specifies an input parameter's type.
     ParameterType.ParameterTypeEnum parameter_type = 2;
+
+    // Optional field. Default value of the input parameter.
+    google.protobuf.Value default_value = 3;
   }
   // Name to artifact input.
   map<string, ArtifactSpec> artifacts = 1;

api/v2alpha1/python/setup.py

@@ -24,7 +24,7 @@
     from shutil import which as find_executable
 
 NAME = "kfp-pipeline-spec"
-VERSION = "0.1.11"
+VERSION = "0.1.12"
 
 PROTO_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir))
 

backend/src/apiserver/common/const.go

@@ -16,19 +16,17 @@ package common
 
 import (
 	api "github.com/kubeflow/pipelines/backend/api/go_client"
+	"github.com/kubeflow/pipelines/backend/src/apiserver/model"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 )
 
-type ResourceType string
-type Relationship string
-
 const (
-	Experiment      ResourceType = "Experiment"
-	Job             ResourceType = "Job"
-	Run             ResourceType = "Run"
-	Pipeline        ResourceType = "pipeline"
-	PipelineVersion ResourceType = "PipelineVersion"
-	Namespace       ResourceType = "Namespace"
+	Experiment      model.ResourceType = "Experiment"
+	Job             model.ResourceType = "Job"
+	Run             model.ResourceType = "Run"
+	Pipeline        model.ResourceType = "pipeline"
+	PipelineVersion model.ResourceType = "PipelineVersion"
+	Namespace       model.ResourceType = "Namespace"
 )
 
 const (
@@ -57,8 +55,8 @@ const (
 )
 
 const (
-	Owner   Relationship = "Owner"
-	Creator Relationship = "Creator"
+	Owner   model.Relationship = "Owner"
+	Creator model.Relationship = "Creator"
 )
 
 const (
@@ -70,7 +68,7 @@ const (
 
 const DefaultTokenReviewAudience string = "pipelines.kubeflow.org"
 
-func ToModelResourceType(apiType api.ResourceType) (ResourceType, error) {
+func ToModelResourceType(apiType api.ResourceType) (model.ResourceType, error) {
 	switch apiType {
 	case api.ResourceType_EXPERIMENT:
 		return Experiment, nil
@@ -85,7 +83,7 @@ func ToModelResourceType(apiType api.ResourceType) (ResourceType, error) {
 	}
 }
 
-func ToModelRelationship(r api.Relationship) (Relationship, error) {
+func ToModelRelationship(r api.Relationship) (model.Relationship, error) {
 	switch r {
 	case api.Relationship_CREATOR:
 		return Creator, nil

backend/src/apiserver/common/filter_context.go

@@ -14,8 +14,10 @@
 
 package common
 
+import "github.com/kubeflow/pipelines/backend/src/apiserver/model"
+
 type ReferenceKey struct {
-	Type ResourceType
+	Type model.ResourceType
 	ID   string
 }
 

backend/src/apiserver/list/list.go

@@ -29,6 +29,7 @@ import (
 	api "github.com/kubeflow/pipelines/backend/api/go_client"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/common"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/filter"
+	"github.com/kubeflow/pipelines/backend/src/apiserver/model"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 )
 
@@ -226,7 +227,7 @@ func (o *Options) AddFilterToSelect(sqlBuilder sq.SelectBuilder) sq.SelectBuilde
 
 // FilterOnResourceReference filters the given resource's table by rows from the ResourceReferences
 // table that match an optional given filter, and returns the rebuilt SelectBuilder
-func FilterOnResourceReference(tableName string, columns []string, resourceType common.ResourceType,
+func FilterOnResourceReference(tableName string, columns []string, resourceType model.ResourceType,
 	selectCount bool, filterContext *common.FilterContext) (sq.SelectBuilder, error) {
 	selectBuilder := sq.Select(columns...)
 	if selectCount {

backend/src/apiserver/list/list_test.go

@@ -1,11 +1,12 @@
 package list
 
 import (
-	"google.golang.org/protobuf/testing/protocmp"
 	"reflect"
 	"strings"
 	"testing"
 
+	"google.golang.org/protobuf/testing/protocmp"
+
 	"github.com/kubeflow/pipelines/backend/src/apiserver/common"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/filter"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/model"
@@ -197,7 +198,7 @@ func TestNextPageToken_ValidTokens(t *testing.T) {
 	for _, test := range tests {
 		got, err := test.inOpts.nextPageToken(l)
 
-		if !cmp.Equal(got, test.want, cmpopts.EquateEmpty(), protocmp.Transform(),cmp.AllowUnexported(filter.Filter{})) || err != nil {
+		if !cmp.Equal(got, test.want, cmpopts.EquateEmpty(), protocmp.Transform(), cmp.AllowUnexported(filter.Filter{})) || err != nil {
 			t.Errorf("nextPageToken(%+v, %+v) =\nGot: %+v, %+v\nWant: %+v, <nil>\nDiff:\n%s",
 				test.inOpts, l, got, err, test.want, cmp.Diff(test.want, got))
 		}
@@ -713,7 +714,7 @@ func TestTokenSerialization(t *testing.T) {
 
 		got := &token{}
 		got.unmarshal(s)
-		if !cmp.Equal(got, test.want, cmpopts.EquateEmpty(), protocmp.Transform(),cmp.AllowUnexported(filter.Filter{})) {
+		if !cmp.Equal(got, test.want, cmpopts.EquateEmpty(), protocmp.Transform(), cmp.AllowUnexported(filter.Filter{})) {
 			t.Errorf("token.unmarshal(%q) =\nGot: %+v\nWant: %+v\nDiff:\n%s",
 				s, got, test.want, cmp.Diff(test.want, got, cmp.AllowUnexported(filter.Filter{})))
 		}
@@ -795,7 +796,7 @@ func TestFilterOnResourceReference(t *testing.T) {
 
 	type testIn struct {
 		table        string
-		resourceType common.ResourceType
+		resourceType model.ResourceType
 		count        bool
 		filter       *common.FilterContext
 	}

backend/src/apiserver/model/resource_reference.go

@@ -14,17 +14,19 @@
 
 package model
 
-import (
-	"github.com/kubeflow/pipelines/backend/src/apiserver/common"
-)
+// The type of a resource object.
+type ResourceType string
+
+// The relationship between two resource objects.
+type Relationship string
 
 // Resource reference table models the relationship between resources in a loosely coupled way.
 type ResourceReference struct {
 	// ID of the resource object
 	ResourceUUID string `gorm:"column:ResourceUUID; not null; primary_key"`
 
 	// The type of the resource object
-	ResourceType common.ResourceType `gorm:"column:ResourceType; not null; primary_key; index:referencefilter"`
+	ResourceType ResourceType `gorm:"column:ResourceType; not null; primary_key; index:referencefilter"`
 
 	// The ID of the resource that been referenced to.
 	ReferenceUUID string `gorm:"column:ReferenceUUID; not null; index:referencefilter"`
@@ -33,22 +35,11 @@ type ResourceReference struct {
 	ReferenceName string `gorm:"column:ReferenceName; not null; "`
 
 	// The type of the resource that been referenced to.
-	ReferenceType common.ResourceType `gorm:"column:ReferenceType; not null; primary_key; index:referencefilter"`
+	ReferenceType ResourceType `gorm:"column:ReferenceType; not null; primary_key; index:referencefilter"`
 
 	// The relationship between the resource object and the resource that been referenced to.
-	Relationship common.Relationship `gorm:"column:Relationship; not null; "`
+	Relationship Relationship `gorm:"column:Relationship; not null; "`
 
 	// The json formatted blob of the resource reference.
 	Payload string `gorm:"column:Payload; not null; size:65535 "`
 }
-
-func GetNamespaceFromModelResourceReferences(resourceRefs []*ResourceReference) string {
-	namespace := ""
-	for _, resourceRef := range resourceRefs {
-		if resourceRef.ReferenceType == common.Namespace {
-			namespace = resourceRef.ReferenceUUID
-			break
-		}
-	}
-	return namespace
-}

backend/src/apiserver/resource/model_converter.go

@@ -216,7 +216,7 @@ func toModelParameters(apiParams []*api.Parameter) (string, error) {
 }
 
 func (r *ResourceManager) toModelResourceReferences(
-	resourceId string, resourceType common.ResourceType, apiRefs []*api.ResourceReference) ([]*model.ResourceReference, error) {
+	resourceId string, resourceType model.ResourceType, apiRefs []*api.ResourceReference) ([]*model.ResourceReference, error) {
 	var modelRefs []*model.ResourceReference
 	for _, apiRef := range apiRefs {
 		modelReferenceType, err := common.ToModelResourceType(apiRef.Key.Type)
@@ -246,7 +246,7 @@ func (r *ResourceManager) toModelResourceReferences(
 	return modelRefs, nil
 }
 
-func (r *ResourceManager) getResourceName(resourceType common.ResourceType, resourceId string) (string, error) {
+func (r *ResourceManager) getResourceName(resourceType model.ResourceType, resourceId string) (string, error) {
 	switch resourceType {
 	case common.Experiment:
 		experiment, err := r.GetExperiment(resourceId)

backend/src/apiserver/resource/resource_manager.go

@@ -31,8 +31,8 @@ import (
 	"github.com/kubeflow/pipelines/backend/src/apiserver/archive"
 	kfpauth "github.com/kubeflow/pipelines/backend/src/apiserver/auth"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/client"
-	"github.com/kubeflow/pipelines/backend/src/apiserver/list"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/common"
+	"github.com/kubeflow/pipelines/backend/src/apiserver/list"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/model"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/storage"
 	"github.com/kubeflow/pipelines/backend/src/common/util"

backend/src/apiserver/server/api_converter.go

@@ -287,7 +287,7 @@ func toApiResourceReferences(references []*model.ResourceReference) []*api.Resou
 	return apiReferences
 }
 
-func toApiResourceType(modelType common.ResourceType) api.ResourceType {
+func toApiResourceType(modelType model.ResourceType) api.ResourceType {
 	switch modelType {
 	case common.Experiment:
 		return api.ResourceType_EXPERIMENT
@@ -302,7 +302,7 @@ func toApiResourceType(modelType common.ResourceType) api.ResourceType {
 	}
 }
 
-func toApiRelationship(r common.Relationship) api.Relationship {
+func toApiRelationship(r model.Relationship) api.Relationship {
 	switch r {
 	case common.Creator:
 		return api.Relationship_CREATOR

backend/src/apiserver/storage/resource_reference_store.go

@@ -17,8 +17,8 @@ var resourceReferenceColumns = []string{"ResourceUUID", "ResourceType", "Referen
 
 type ResourceReferenceStoreInterface interface {
 	// Retrieve the resource reference for a given resource id, type and a reference type.
-	GetResourceReference(resourceId string, resourceType common.ResourceType,
-		referenceType common.ResourceType) (*model.ResourceReference, error)
+	GetResourceReference(resourceId string, resourceType model.ResourceType,
+		referenceType model.ResourceType) (*model.ResourceReference, error)
 }
 
 type ResourceReferenceStore struct {
@@ -55,7 +55,7 @@ func (s *ResourceReferenceStore) CreateResourceReferences(tx *sql.Tx, refs []*mo
 	return nil
 }
 
-func (s *ResourceReferenceStore) checkReferenceExist(tx *sql.Tx, referenceId string, referenceType common.ResourceType) bool {
+func (s *ResourceReferenceStore) checkReferenceExist(tx *sql.Tx, referenceId string, referenceType model.ResourceType) bool {
 	var selectBuilder sq.SelectBuilder
 	switch referenceType {
 	case common.Job:
@@ -85,7 +85,7 @@ func (s *ResourceReferenceStore) checkReferenceExist(tx *sql.Tx, referenceId str
 
 // Delete all resource references for a specific resource.
 // This is always in company with creating a parent resource so a transaction is needed as input.
-func (s *ResourceReferenceStore) DeleteResourceReferences(tx *sql.Tx, id string, resourceType common.ResourceType) error {
+func (s *ResourceReferenceStore) DeleteResourceReferences(tx *sql.Tx, id string, resourceType model.ResourceType) error {
 	refSql, refArgs, err := sq.
 		Delete("resource_references").
 		Where(sq.Or{
@@ -99,8 +99,8 @@ func (s *ResourceReferenceStore) DeleteResourceReferences(tx *sql.Tx, id string,
 	return nil
 }
 
-func (s *ResourceReferenceStore) GetResourceReference(resourceId string, resourceType common.ResourceType,
-	referenceType common.ResourceType) (*model.ResourceReference, error) {
+func (s *ResourceReferenceStore) GetResourceReference(resourceId string, resourceType model.ResourceType,
+	referenceType model.ResourceType) (*model.ResourceReference, error) {
 	sql, args, err := sq.Select(resourceReferenceColumns...).
 		From("resource_references").
 		Where(sq.Eq{
@@ -142,11 +142,11 @@ func (s *ResourceReferenceStore) scanRows(r *sql.Rows) ([]model.ResourceReferenc
 		}
 		references = append(references, model.ResourceReference{
 			ResourceUUID:  resourceUUID,
-			ResourceType:  common.ResourceType(resourceType),
+			ResourceType:  model.ResourceType(resourceType),
 			ReferenceUUID: referenceUUID,
 			ReferenceName: referenceName,
-			ReferenceType: common.ResourceType(referenceType),
-			Relationship:  common.Relationship(relationship),
+			ReferenceType: model.ResourceType(referenceType),
+			Relationship:  model.Relationship(relationship),
 			Payload:       payload,
 		})
 	}

components/google-cloud/dependencies.py

@@ -20,8 +20,9 @@ def make_required_install_packages():
         # Explicity add google-api-core as a dependancy to avoid conflict
         # between kfp & aiplatform.
         "google-api-core<2dev,>=1.26.0",
-        "kfp>=1.7.2,<2.0.0",
+        "kfp>=1.8.4,<2.0.0",
         "google-cloud-aiplatform>=1.4.3",
+        "google-cloud-notebooks>=0.4.0",
     ]