Project domain attribute for workflow Execution config (flyteorg#298)

boilerplate/flyte/golang_test_targets/download_tooling.sh

@@ -17,7 +17,7 @@ set -e
 # In the format of "<cli>:<package>" or ":<package>" if no cli
 tools=(
   "github.com/vektra/mockery/cmd/mockery"
-  "github.com/flyteorg/flytestdlib/cli/pflags@latest"
+  "github.com/flyteorg/flytestdlib/cli/pflags"
   "github.com/golangci/golangci-lint/cmd/golangci-lint@latest"
   "github.com/alvaroloes/enumer"
   "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"

cmd/config/subcommand/matchable_attr_file_config_utils.go

@@ -57,4 +57,4 @@ func DumpTaskResourceAttr(matchableAttrConfig interface{}, fileName string) erro
 		fmt.Printf("%v", String(matchableAttrConfig))
 	}
 	return nil
-}
+}

cmd/config/subcommand/workflowexecutionconfig/fetch_config.go

@@ -4,6 +4,7 @@ package workflowexecutionconfig
 
 type AttrFetchConfig struct {
 	AttrFile string `json:"attrFile" pflag:",attribute file name to be used for generating attribute for the resource type."`
+	Gen      bool   `json:"gen" pflag:",generates an empty workflow execution config file with conformance to the api format."`
 }
 
 var DefaultFetchConfig = &AttrFetchConfig{}

cmd/delete/matchable_attribute_util.go

@@ -2,36 +2,36 @@ package delete
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/flyteorg/flytectl/pkg/ext"
 	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
-	"github.com/flyteorg/flytestdlib/logger"
 )
 
 func deleteMatchableAttr(ctx context.Context, project, domain, workflowName string,
 	deleter ext.AdminDeleterExtInterface, rsType admin.MatchableResource, dryRun bool) error {
 	if len(workflowName) > 0 {
 		// Delete the workflow attribute from the admin. If the attribute doesn't exist , admin deesn't return an error and same behavior is followed here
 		if dryRun {
-			logger.Infof(ctx, "skipping DeleteWorkflowAttributes request (dryRun)")
+			fmt.Print("skipping DeleteWorkflowAttributes request (dryRun)\n")
 		} else {
 			err := deleter.DeleteWorkflowAttributes(ctx, project, domain, workflowName, rsType)
 			if err != nil {
 				return err
 			}
 		}
-		logger.Debugf(ctx, "Deleted matchable resources from %v project and domain %v and workflow %v", project, domain, workflowName)
+		fmt.Printf("Deleted matchable resources from %v project and domain %v and workflow %v\n", project, domain, workflowName)
 	} else {
 		// Delete the project domain attribute from the admin. If the attribute doesn't exist , admin deesn't return an error and same behavior is followed here
 		if dryRun {
-			logger.Infof(ctx, "skipping DeleteProjectDomainAttributes request (dryRun)")
+			fmt.Print("skipping DeleteProjectDomainAttributes request (dryRun)\n")
 		} else {
 			err := deleter.DeleteProjectDomainAttributes(ctx, project, domain, rsType)
 			if err != nil {
 				return err
 			}
 		}
-		logger.Debugf(ctx, "Deleted matchable resources from %v project and domain %v", project, domain)
+		fmt.Printf("Deleted matchable resources from %v project and domain %v\n", project, domain)
 	}
 	return nil
 }

cmd/get/matchable_workflow_execution_config.go

@@ -2,13 +2,17 @@ package get
 
 import (
 	"context"
-
-	"github.com/flyteorg/flytectl/cmd/config/subcommand/workflowexecutionconfig"
+	"fmt"
 
 	"github.com/flyteorg/flytectl/cmd/config"
 	sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand"
+	"github.com/flyteorg/flytectl/cmd/config/subcommand/workflowexecutionconfig"
 	cmdCore "github.com/flyteorg/flytectl/cmd/core"
 	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 const (
@@ -88,7 +92,12 @@ func getWorkflowExecutionConfigFunc(ctx context.Context, args []string, cmdCtx c
 	// Updates the workflowExecutionConfigFileConfig with the fetched matchable attribute
 	if err := FetchAndUnDecorateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminFetcherExt(),
 		&workflowExecutionConfigFileConfig, admin.MatchableResource_WORKFLOW_EXECUTION_CONFIG); err != nil {
-		return err
+		if grpcError := status.Code(err); grpcError == codes.NotFound && workflowexecutionconfig.DefaultFetchConfig.Gen {
+			fmt.Println("Generating a sample workflow execution config file")
+			workflowExecutionConfigFileConfig = getSampleWorkflowExecutionFileConfig(project, domain, workflowName)
+		} else {
+			return err
+		}
 	}
 
 	// Write the config to the file which can be used for update
@@ -97,3 +106,29 @@ func getWorkflowExecutionConfigFunc(ctx context.Context, args []string, cmdCtx c
 	}
 	return nil
 }
+
+func getSampleWorkflowExecutionFileConfig(project, domain, workflow string) workflowexecutionconfig.FileConfig {
+	return workflowexecutionconfig.FileConfig{
+		Project:  project,
+		Domain:   domain,
+		Workflow: workflow,
+		WorkflowExecutionConfig: &admin.WorkflowExecutionConfig{
+			MaxParallelism: 10,
+			SecurityContext: &core.SecurityContext{
+				RunAs: &core.Identity{
+					K8SServiceAccount: "default",
+					IamRole:           "",
+				},
+			},
+			Labels: &admin.Labels{
+				Values: map[string]string{"cliLabelKey": "cliLabelValue"},
+			},
+			Annotations: &admin.Annotations{
+				Values: map[string]string{"cliAnnotationKey": "cliAnnotationValue"},
+			},
+			RawOutputDataConfig: &admin.RawOutputDataConfig{
+				OutputLocationPrefix: "cliOutputLocationPrefix",
+			},
+		},
+	}
+}

cmd/update/matchable_attribute_util.go

@@ -2,10 +2,10 @@ package update
 
 import (
 	"context"
+	"fmt"
 
 	sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand"
 	"github.com/flyteorg/flytectl/pkg/ext"
-	"github.com/flyteorg/flytestdlib/logger"
 )
 
 func DecorateAndUpdateMatchableAttr(ctx context.Context, project, domain, workflowName string,
@@ -14,25 +14,25 @@ func DecorateAndUpdateMatchableAttr(ctx context.Context, project, domain, workfl
 	if len(workflowName) > 0 {
 		// Update the workflow attribute using the admin.
 		if dryRun {
-			logger.Infof(ctx, "skipping UpdateWorkflowAttributes request (dryRun)")
+			fmt.Printf("skipping UpdateWorkflowAttributes request (dryRun)\n")
 		} else {
 			err := updater.UpdateWorkflowAttributes(ctx, project, domain, workflowName, matchingAttr)
 			if err != nil {
 				return err
 			}
 		}
-		logger.Debugf(ctx, "Updated attributes from %v project and domain %v and workflow %v", project, domain, workflowName)
+		fmt.Printf("Updated attributes from %v project and domain %v and workflow %v\n", project, domain, workflowName)
 	} else {
 		// Update the project domain attribute using the admin.
 		if dryRun {
-			logger.Infof(ctx, "skipping UpdateProjectDomainAttributes request (dryRun)")
+			fmt.Printf("skipping UpdateProjectDomainAttributes request (dryRun)\n")
 		} else {
 			err := updater.UpdateProjectDomainAttributes(ctx, project, domain, matchingAttr)
 			if err != nil {
 				return err
 			}
 		}
-		logger.Debugf(ctx, "Updated attributes from %v project and domain %v", project, domain)
+		fmt.Printf("Updated attributes from %v project and domain %v\n", project, domain)
 	}
 	return nil
 }

cmd/update/matchable_cluster_resource_attribute_test.go

@@ -36,7 +36,7 @@ func TestUpdateClusterResourceAttributes(t *testing.T) {
 			mock.Anything).Return(nil)
 		err = updateClusterResourceAttributesFunc(ctx, args, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development`)
 	})
 	t.Run("failed to update project domain attribute", func(t *testing.T) {
 		setup()
@@ -59,7 +59,7 @@ func TestUpdateClusterResourceAttributes(t *testing.T) {
 			mock.Anything, mock.Anything).Return(nil)
 		err = updateClusterResourceAttributesFunc(ctx, nil, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development and workflow core.control_flow.run_merge_sort.merge_sort`)
 	})
 	t.Run("failed to update workflow attribute", func(t *testing.T) {
 		setup()

cmd/update/matchable_execution_cluster_label_test.go

@@ -36,7 +36,7 @@ func TestExecutionClusterLabel(t *testing.T) {
 			mock.Anything).Return(nil)
 		err = updateExecutionClusterLabelFunc(ctx, args, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development`)
 	})
 	t.Run("failed update project domain attribute", func(t *testing.T) {
 		setup()
@@ -59,7 +59,7 @@ func TestExecutionClusterLabel(t *testing.T) {
 			mock.Anything, mock.Anything).Return(nil)
 		err = updateExecutionClusterLabelFunc(ctx, nil, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development and workflow core.control_flow.run_merge_sort.merge_sort`)
 	})
 	t.Run("failed update workflow attribute", func(t *testing.T) {
 		setup()

cmd/update/matchable_execution_queue_attribute_test.go

@@ -36,7 +36,7 @@ func TestExecutionQueueAttributes(t *testing.T) {
 			mock.Anything).Return(nil)
 		err = updateExecutionQueueAttributesFunc(ctx, args, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development`)
 	})
 	t.Run("failed update project domain attribute", func(t *testing.T) {
 		setup()
@@ -59,7 +59,7 @@ func TestExecutionQueueAttributes(t *testing.T) {
 			mock.Anything, mock.Anything).Return(nil)
 		err = updateExecutionQueueAttributesFunc(ctx, nil, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development and workflow core.control_flow.run_merge_sort.merge_sort`)
 	})
 	t.Run("failed update workflow attribute", func(t *testing.T) {
 		setup()

cmd/update/matchable_plugin_override_test.go

@@ -36,7 +36,7 @@ func TestPluginOverride(t *testing.T) {
 			mock.Anything).Return(nil)
 		err = updatePluginOverridesFunc(ctx, args, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development`)
 	})
 	t.Run("failed update project domain attribute", func(t *testing.T) {
 		setup()
@@ -59,7 +59,7 @@ func TestPluginOverride(t *testing.T) {
 			mock.Anything, mock.Anything).Return(nil)
 		err = updatePluginOverridesFunc(ctx, nil, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development and workflow core.control_flow.run_merge_sort.merge_sort`)
 	})
 	t.Run("failed update workflow attribute", func(t *testing.T) {
 		setup()

cmd/update/matchable_task_resource_attribute_test.go

@@ -36,7 +36,7 @@ func TestUpdateTaskResourceAttributes(t *testing.T) {
 			mock.Anything).Return(nil)
 		err = updateTaskResourceAttributesFunc(ctx, args, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development`)
 	})
 	t.Run("failed update project domain attribute", func(t *testing.T) {
 		setup()
@@ -59,7 +59,7 @@ func TestUpdateTaskResourceAttributes(t *testing.T) {
 			mock.Anything, mock.Anything).Return(nil)
 		err = updateTaskResourceAttributesFunc(ctx, nil, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development and workflow core.control_flow.run_merge_sort.merge_sort`)
 	})
 	t.Run("failed update workflow attribute", func(t *testing.T) {
 		setup()

cmd/update/matchable_workflow_execution_config_test.go

@@ -37,7 +37,7 @@ func TestWorkflowExecutionConfigs(t *testing.T) {
 			mock.Anything).Return(nil)
 		err = updateWorkflowExecutionConfigFunc(ctx, args, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development`)
 	})
 	t.Run("failed update project domain attribute", func(t *testing.T) {
 		setup()
@@ -60,7 +60,7 @@ func TestWorkflowExecutionConfigs(t *testing.T) {
 			mock.Anything, mock.Anything).Return(nil)
 		err = updateWorkflowExecutionConfigFunc(ctx, nil, cmdCtx)
 		assert.Nil(t, err)
-		tearDownAndVerify(t, ``)
+		tearDownAndVerify(t, `Updated attributes from flytectldemo project and domain development and workflow core.control_flow.run_merge_sort.merge_sort`)
 	})
 	t.Run("failed update workflow attribute", func(t *testing.T) {
 		setup()

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/enescakir/emoji v1.0.0
-	github.com/flyteorg/flyteidl v0.21.24
+	github.com/flyteorg/flyteidl v0.24.6
 	github.com/flyteorg/flytestdlib v0.4.14
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0

go.sum

@@ -356,8 +356,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
-github.com/flyteorg/flyteidl v0.21.24 h1:e2wPBK4aiLE+fw2zmhUDNg39QoJk6Lf5lQRvj8XgtFk=
-github.com/flyteorg/flyteidl v0.21.24/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
+github.com/flyteorg/flyteidl v0.24.6 h1:n2796X9Sw7mNDtXWwsJr84DLQpz8Cptvb7LptfJLxag=
+github.com/flyteorg/flyteidl v0.24.6/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
 github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220=
 github.com/flyteorg/flytestdlib v0.4.14 h1:qpPwvJ+DqM1fI/y5uPKAP8p8VONz8oRp9Fz0jFl/5aI=
 github.com/flyteorg/flytestdlib v0.4.14/go.mod h1:fv1ar34LJLMTaf0tbfetisLykUlARi7rP+NQTUn6QQs=