diff --git a/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go new file mode 100755 index 000000000..52fd144b9 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrDeleteConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrDeleteConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrDeleteConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrDeleteConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrDeleteConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultDelConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultDelConfig.AttrFile, "attribute file name to be used for delete attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go new file mode 100755 index 000000000..7a5d60873 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrDeleteConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrDeleteConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrDeleteConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrDeleteConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrDeleteConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrDeleteConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrDeleteConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrDeleteConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrDeleteConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrDeleteConfig(val, result)) +} + +func testDecodeSlice_AttrDeleteConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrDeleteConfig(vStringSlice, result)) +} + +func TestAttrDeleteConfig_GetPFlagSet(t *testing.T) { + val := AttrDeleteConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrDeleteConfig_SetFlags(t *testing.T) { + actual := AttrDeleteConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultDelConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrDeleteConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go new file mode 100755 index 000000000..1cf522b3c --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrFetchConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrFetchConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrFetchConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrFetchConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrFetchConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultFetchConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultFetchConfig.AttrFile, "attribute file name to be used for generating attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go new file mode 100755 index 000000000..e94f54e98 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrFetchConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrFetchConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrFetchConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrFetchConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrFetchConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrFetchConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrFetchConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrFetchConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrFetchConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrFetchConfig(val, result)) +} + +func testDecodeSlice_AttrFetchConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrFetchConfig(vStringSlice, result)) +} + +func TestAttrFetchConfig_GetPFlagSet(t *testing.T) { + val := AttrFetchConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrFetchConfig_SetFlags(t *testing.T) { + actual := AttrFetchConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultFetchConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrFetchConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go new file mode 100755 index 000000000..21c58dbb9 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrUpdateConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrUpdateConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrUpdateConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrUpdateConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrUpdateConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultUpdateConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultUpdateConfig.AttrFile, "attribute file name to be used for updating attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go new file mode 100755 index 000000000..37ee0382c --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrUpdateConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrUpdateConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrUpdateConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrUpdateConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrUpdateConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrUpdateConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrUpdateConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrUpdateConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrUpdateConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrUpdateConfig(val, result)) +} + +func testDecodeSlice_AttrUpdateConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrUpdateConfig(vStringSlice, result)) +} + +func TestAttrUpdateConfig_GetPFlagSet(t *testing.T) { + val := AttrUpdateConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrUpdateConfig_SetFlags(t *testing.T) { + actual := AttrUpdateConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultUpdateConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrUpdateConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/plugin_override/delete_config.go b/cmd/config/subcommand/plugin_override/delete_config.go new file mode 100644 index 000000000..beec54fd9 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/delete_config.go @@ -0,0 +1,10 @@ +package pluginoverride + +//go:generate pflags AttrDeleteConfig --default-var DefaultDelConfig + +// AttrDeleteConfig Matchable resource attributes configuration passed from command line +type AttrDeleteConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for delete attribute for the resource type."` +} + +var DefaultDelConfig = &AttrDeleteConfig{} diff --git a/cmd/config/subcommand/plugin_override/fetch_config.go b/cmd/config/subcommand/plugin_override/fetch_config.go new file mode 100644 index 000000000..9976d8b33 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/fetch_config.go @@ -0,0 +1,9 @@ +package pluginoverride + +//go:generate pflags AttrFetchConfig --default-var DefaultFetchConfig + +type AttrFetchConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for generating attribute for the resource type."` +} + +var DefaultFetchConfig = &AttrFetchConfig{} diff --git a/cmd/config/subcommand/plugin_override/file_config.go b/cmd/config/subcommand/plugin_override/file_config.go new file mode 100644 index 000000000..af40066dd --- /dev/null +++ b/cmd/config/subcommand/plugin_override/file_config.go @@ -0,0 +1,47 @@ +package pluginoverride + +import ( + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +// FileConfig shadow Config for PluginOverrides. +// The shadow Config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. +// As the same structure is being used for both ProjectDomainAttribute/Workflowattribute +type FileConfig struct { + Project string `json:"project"` + Domain string `json:"domain"` + Workflow string `json:"workflow,omitempty"` + *admin.PluginOverrides +} + +// Decorate decorator over PluginOverrides. +func (t FileConfig) Decorate() *admin.MatchingAttributes { + return &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: t.PluginOverrides, + }, + } +} + +// UnDecorate to uncover PluginOverrides. +func (t *FileConfig) UnDecorate(matchingAttribute *admin.MatchingAttributes) { + if matchingAttribute == nil { + return + } + t.PluginOverrides = matchingAttribute.GetPluginOverrides() +} + +// GetProject from the FileConfig +func (t FileConfig) GetProject() string { + return t.Project +} + +// GetDomain from the FileConfig +func (t FileConfig) GetDomain() string { + return t.Domain +} + +// GetWorkflow from the FileConfig +func (t FileConfig) GetWorkflow() string { + return t.Workflow +} diff --git a/cmd/config/subcommand/plugin_override/file_config_test.go b/cmd/config/subcommand/plugin_override/file_config_test.go new file mode 100644 index 000000000..e2ecaa5d0 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/file_config_test.go @@ -0,0 +1,58 @@ +package pluginoverride + +import ( + "testing" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" +) + +func TestFileConfig(t *testing.T) { + pluginOverride1 := &admin.PluginOverride{ + TaskType: "python_task", + PluginId: []string{"plugin-override1", "plugin-override2"}, + MissingPluginBehavior: admin.PluginOverride_FAIL, + } + pluginOverride2 := &admin.PluginOverride{ + TaskType: "java_task", + PluginId: []string{"plugin-override3", "plugin-override3"}, + MissingPluginBehavior: admin.PluginOverride_USE_DEFAULT, + } + pluginOverrides := []*admin.PluginOverride{pluginOverride1, pluginOverride2} + + pluginOverrideFileConfig := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + PluginOverrides: &admin.PluginOverrides{ + Overrides: pluginOverrides, + }, + } + matchingAttr := &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: pluginOverrideFileConfig.PluginOverrides, + }, + } + t.Run("decorate", func(t *testing.T) { + assert.Equal(t, matchingAttr, pluginOverrideFileConfig.Decorate()) + }) + + t.Run("decorate", func(t *testing.T) { + taskAttrFileConfigNew := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + } + taskAttrFileConfigNew.UnDecorate(matchingAttr) + assert.Equal(t, pluginOverrideFileConfig, taskAttrFileConfigNew) + }) + t.Run("get project domain workflow", func(t *testing.T) { + taskAttrFileConfigNew := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + Workflow: "workflow", + } + assert.Equal(t, "dummyProject", taskAttrFileConfigNew.GetProject()) + assert.Equal(t, "dummyDomain", taskAttrFileConfigNew.GetDomain()) + assert.Equal(t, "workflow", taskAttrFileConfigNew.GetWorkflow()) + }) +} diff --git a/cmd/config/subcommand/plugin_override/update_config.go b/cmd/config/subcommand/plugin_override/update_config.go new file mode 100644 index 000000000..e2daefa8f --- /dev/null +++ b/cmd/config/subcommand/plugin_override/update_config.go @@ -0,0 +1,10 @@ +package pluginoverride + +//go:generate pflags AttrUpdateConfig --default-var DefaultUpdateConfig + +// AttrUpdateConfig Matchable resource attributes configuration passed from command line +type AttrUpdateConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for updating attribute for the resource type."` +} + +var DefaultUpdateConfig = &AttrUpdateConfig{} diff --git a/cmd/delete/delete.go b/cmd/delete/delete.go index 8bee8c58d..94ce8e801 100644 --- a/cmd/delete/delete.go +++ b/cmd/delete/delete.go @@ -4,6 +4,7 @@ import ( "github.com/flyteorg/flytectl/cmd/config/subcommand/clusterresourceattribute" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionqueueattribute" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" "github.com/flyteorg/flytectl/cmd/config/subcommand/taskresourceattribute" cmdcore "github.com/flyteorg/flytectl/cmd/core" @@ -43,6 +44,9 @@ func RemoteDeleteCommand() *cobra.Command { "execution-queue-attribute": {CmdFunc: deleteExecutionQueueAttributes, Aliases: []string{"execution-queue-attributes"}, Short: executionQueueAttributesShort, Long: executionQueueAttributesLong, PFlagProvider: executionqueueattribute.DefaultDelConfig, ProjectDomainNotRequired: true}, + "plugin-override": {CmdFunc: deletePluginOverride, Aliases: []string{"plugin-overrides"}, + Short: pluginOverrideShort, + Long: pluginOverrideLong, PFlagProvider: pluginoverride.DefaultDelConfig, ProjectDomainNotRequired: true}, } cmdcore.AddCommands(deleteCmd, terminateResourcesFuncs) return deleteCmd diff --git a/cmd/delete/delete_test.go b/cmd/delete/delete_test.go index c4b9197bb..e26a1d7a9 100644 --- a/cmd/delete/delete_test.go +++ b/cmd/delete/delete_test.go @@ -32,16 +32,16 @@ func TestDeleteCommand(t *testing.T) { assert.Equal(t, deleteCommand.Use, "delete") assert.Equal(t, deleteCommand.Short, deleteCmdShort) assert.Equal(t, deleteCommand.Long, deleteCmdLong) - assert.Equal(t, len(deleteCommand.Commands()), 5) + assert.Equal(t, len(deleteCommand.Commands()), 6) cmdNouns := deleteCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - useArray := []string{"cluster-resource-attribute", "execution", "execution-cluster-label", "execution-queue-attribute", "task-resource-attribute"} - aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-cluster-labels"}, {"execution-queue-attributes"}, {"task-resource-attributes"}} - shortArray := []string{clusterResourceAttributesShort, execCmdShort, executionClusterLabelShort, executionQueueAttributesShort, taskResourceAttributesShort} - longArray := []string{clusterResourceAttributesLong, execCmdLong, executionClusterLabelLong, executionQueueAttributesLong, taskResourceAttributesLong} + useArray := []string{"cluster-resource-attribute", "execution", "execution-cluster-label", "execution-queue-attribute", "plugin-override", "task-resource-attribute"} + aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-cluster-labels"}, {"execution-queue-attributes"}, {"plugin-overrides"}, {"task-resource-attributes"}} + shortArray := []string{clusterResourceAttributesShort, execCmdShort, executionClusterLabelShort, executionQueueAttributesShort, pluginOverrideShort, taskResourceAttributesShort} + longArray := []string{clusterResourceAttributesLong, execCmdLong, executionClusterLabelLong, executionQueueAttributesLong, pluginOverrideLong, taskResourceAttributesLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/cmd/delete/matchable_plugin_override.go b/cmd/delete/matchable_plugin_override.go new file mode 100644 index 000000000..23b044c9b --- /dev/null +++ b/cmd/delete/matchable_plugin_override.go @@ -0,0 +1,83 @@ +package delete + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + pluginOverrideShort = "Deletes matchable resources of plugin overrides" + pluginOverrideLong = ` +Deletes plugin override for given project and domain combination or additionally with workflow name. + +Deletes plugin override for project and domain +Here the command deletes plugin override for project flytectldemo and development domain. +:: + + flytectl delete plugin-override -p flytectldemo -d development + + +Deletes plugin override using config file which was used for creating it. +Here the command deletes plugin overrides from the config file po.yaml +Overrides are optional in the file as they are unread during the delete command but can be kept as the same file can be used for get, update or delete +eg: content of po.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +Deletes plugin override for a workflow +Here the command deletes the plugin override for a workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl delete plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage +` +) + +func deletePluginOverride(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var pwdGetter sconfig.ProjectDomainWorkflowGetter + pwdGetter = sconfig.PDWGetterCommandLine{Config: config.GetConfig(), Args: args} + delConfig := pluginoverride.DefaultDelConfig + + // Get the project domain workflowName from the config file or commandline params + if len(delConfig.AttrFile) > 0 { + // Initialize AttrFileConfig which will be used if delConfig.AttrFile is non empty + // And Reads from the attribute file + pwdGetter = &pluginoverride.FileConfig{} + if err := sconfig.ReadConfigFromFile(pwdGetter, delConfig.AttrFile); err != nil { + return err + } + } + // Use the pwdGetter to initialize the project domain and workflow + project := pwdGetter.GetProject() + domain := pwdGetter.GetDomain() + workflowName := pwdGetter.GetWorkflow() + + // Deletes the matchable attributes using the AttrFileConfig + if err := deleteMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminDeleterExt(), + admin.MatchableResource_PLUGIN_OVERRIDE); err != nil { + return err + } + + return nil +} diff --git a/cmd/delete/matchable_plugin_override_test.go b/cmd/delete/matchable_plugin_override_test.go new file mode 100644 index 000000000..d8ed34818 --- /dev/null +++ b/cmd/delete/matchable_plugin_override_test.go @@ -0,0 +1,134 @@ +package delete + +import ( + "fmt" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func deletePluginOverrideSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + pluginoverride.DefaultDelConfig = &pluginoverride.AttrDeleteConfig{} + args = []string{} +} + +func TestPluginOverride(t *testing.T) { + t.Run("successful project domain attribute deletion commandline", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("failed project domain attribute deletion", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to delte project domain attributes")) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delte project domain attributes"), err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("successful project domain attribute deletion file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "testdata/valid_project_domain_plugin_override.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("successful workflow attribute deletion", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("failed workflow attribute deletion", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to delete workflow attribute")) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delete workflow attribute"), err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("successful workflow attribute deletion file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "testdata/valid_workflow_plugin_override.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("workflow attribute deletion non existent file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = testDataNonExistentFile + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + u.DeleterExt.AssertNotCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("attribute deletion invalid file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = testDataInvalidAttrFile + // No args implying project domain attribute deletion + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, + fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), + err) + u.DeleterExt.AssertNotCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_PLUGIN_OVERRIDE) + }) +} diff --git a/cmd/delete/testdata/valid_project_domain_plugin_override.yaml b/cmd/delete/testdata/valid_project_domain_plugin_override.yaml new file mode 100644 index 000000000..a8ffc0fef --- /dev/null +++ b/cmd/delete/testdata/valid_project_domain_plugin_override.yaml @@ -0,0 +1,8 @@ +domain: development +project: flytectldemo +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT diff --git a/cmd/delete/testdata/valid_workflow_plugin_override.yaml b/cmd/delete/testdata/valid_workflow_plugin_override.yaml new file mode 100644 index 000000000..6fbb58eae --- /dev/null +++ b/cmd/delete/testdata/valid_workflow_plugin_override.yaml @@ -0,0 +1,9 @@ +domain: development +project: flytectldemo +workflow: core.control_flow.run_merge_sort.merge_sort +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT \ No newline at end of file diff --git a/cmd/get/get.go b/cmd/get/get.go index f0f8bbe10..b78bc4f67 100644 --- a/cmd/get/get.go +++ b/cmd/get/get.go @@ -4,6 +4,7 @@ import ( "github.com/flyteorg/flytectl/cmd/config/subcommand/clusterresourceattribute" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionqueueattribute" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" "github.com/flyteorg/flytectl/cmd/config/subcommand/taskresourceattribute" "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" cmdcore "github.com/flyteorg/flytectl/cmd/core" @@ -54,6 +55,9 @@ func CreateGetCommand() *cobra.Command { "execution-cluster-label": {CmdFunc: getExecutionClusterLabel, Aliases: []string{"execution-cluster-labels"}, Short: executionClusterLabelShort, Long: executionClusterLabelLong, PFlagProvider: executionclusterlabel.DefaultFetchConfig}, + "plugin-override": {CmdFunc: getPluginOverridesFunc, Aliases: []string{"plugin-overrides"}, + Short: pluginOverrideShort, + Long: pluginOverrideLong, PFlagProvider: pluginoverride.DefaultFetchConfig}, } cmdcore.AddCommands(getCmd, getResourcesFuncs) diff --git a/cmd/get/get_test.go b/cmd/get/get_test.go index 890138ec0..5b513ccd4 100644 --- a/cmd/get/get_test.go +++ b/cmd/get/get_test.go @@ -42,20 +42,20 @@ func TestCreateGetCommand(t *testing.T) { assert.Equal(t, getCommand.Use, "get") assert.Equal(t, getCommand.Short, "Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project.") fmt.Println(getCommand.Commands()) - assert.Equal(t, len(getCommand.Commands()), 9) + assert.Equal(t, len(getCommand.Commands()), 10) cmdNouns := getCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) useArray := []string{"cluster-resource-attribute", "execution", "execution-cluster-label", - "execution-queue-attribute", "launchplan", "project", "task", "task-resource-attribute", "workflow"} + "execution-queue-attribute", "launchplan", "plugin-override", "project", "task", "task-resource-attribute", "workflow"} aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-cluster-labels"}, - {"execution-queue-attributes"}, {"launchplans"}, {"projects"}, {"tasks"}, {"task-resource-attributes"}, {"workflows"}} - shortArray := []string{clusterResourceAttributesShort, executionShort, executionClusterLabelShort, executionQueueAttributesShort, launchPlanShort, projectShort, - taskShort, taskResourceAttributesShort, workflowShort} - longArray := []string{clusterResourceAttributesLong, executionLong, executionClusterLabelLong, executionQueueAttributesLong, launchPlanLong, projectLong, taskLong, - taskResourceAttributesLong, workflowLong} + {"execution-queue-attributes"}, {"launchplans"}, {"plugin-overrides"}, {"projects"}, {"tasks"}, {"task-resource-attributes"}, {"workflows"}} + shortArray := []string{clusterResourceAttributesShort, executionShort, executionClusterLabelShort, executionQueueAttributesShort, launchPlanShort, + pluginOverrideShort, projectShort, taskShort, taskResourceAttributesShort, workflowShort} + longArray := []string{clusterResourceAttributesLong, executionLong, executionClusterLabelLong, executionQueueAttributesLong, launchPlanLong, + pluginOverrideLong, projectLong, taskLong, taskResourceAttributesLong, workflowLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/cmd/get/matchable_plugin_override.go b/cmd/get/matchable_plugin_override.go new file mode 100644 index 000000000..19d79f3a0 --- /dev/null +++ b/cmd/get/matchable_plugin_override.go @@ -0,0 +1,112 @@ +package get + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + pluginOverrideShort = "Gets matchable resources of plugin override" + pluginOverrideLong = ` +Retrieves plugin overrides for given project and domain combination or additionally with workflow name. + +Retrieves plugin overrides for project and domain +Here the command get plugin override for project flytectldemo and development domain. + +:: + + flytectl get plugin-override -p flytectldemo -d development + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Retrieves plugin override for project and domain and workflow +Here the command get plugin override for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl get plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "workflow": "core.control_flow.run_merge_sort.merge_sort" + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Writing the plugin override to a file. If there are no plugin overrides, command would return an error. +Here the command gets plugin overrides and writes the config file to po.yaml +eg: content of po.yaml + +:: + + flytectl get plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +Usage +` +) + +func getPluginOverridesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var project string + var domain string + var workflowName string + + // Get the project domain workflow name parameters from the command line. Project and domain are mandatory for this command + project = config.GetConfig().Project + domain = config.GetConfig().Domain + if len(args) == 1 { + workflowName = args[0] + } + // Construct a shadow config for PluginOverrides. The shadow config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. + pluginOverrideFileConfig := pluginoverride.FileConfig{Project: project, Domain: domain, Workflow: workflowName} + // Get the plugin overrides from the command line config + fileName := pluginoverride.DefaultFetchConfig.AttrFile + + // Updates the pluginOverrideFileConfig with the fetched matchable attribute + if err := FetchAndUnDecorateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminFetcherExt(), + &pluginOverrideFileConfig, admin.MatchableResource_PLUGIN_OVERRIDE); err != nil { + return err + } + + // Write the config to the file which can be used for update + if err := sconfig.DumpTaskResourceAttr(pluginOverrideFileConfig, fileName); err != nil { + return err + } + return nil +} diff --git a/cmd/get/matchable_plugin_override_test.go b/cmd/get/matchable_plugin_override_test.go new file mode 100644 index 000000000..b72ab6524 --- /dev/null +++ b/cmd/get/matchable_plugin_override_test.go @@ -0,0 +1,148 @@ +package get + +import ( + "fmt" + "os" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getPluginOverrideSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + pluginoverride.DefaultFetchConfig = &pluginoverride.AttrFetchConfig{} + // Clean up the temp directory. + _ = os.Remove(testDataTempFile) +} + +func TestGetPluginOverride(t *testing.T) { + pluginOverride1 := &admin.PluginOverride{ + TaskType: "python_task", + PluginId: []string{"plugin-override1", "plugin-override2"}, + MissingPluginBehavior: admin.PluginOverride_FAIL, + } + pluginOverride2 := &admin.PluginOverride{ + TaskType: "java_task", + PluginId: []string{"plugin-override3", "plugin-override3"}, + MissingPluginBehavior: admin.PluginOverride_USE_DEFAULT, + } + pluginOverrides := []*admin.PluginOverride{pluginOverride1, pluginOverride2} + projectDomainResp := &admin.ProjectDomainAttributesGetResponse{ + Attributes: &admin.ProjectDomainAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: &admin.PluginOverrides{ + Overrides: pluginOverrides, + }, + }, + }, + }, + } + workflowResp := &admin.WorkflowAttributesGetResponse{ + Attributes: &admin.WorkflowAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + Workflow: "workflow", + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: &admin.PluginOverrides{ + Overrides: pluginOverrides, + }, + }, + }, + }, + } + t.Run("successful get project domain attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","overrides":[{"task_type":"python_task","plugin_id":["plugin-override1","plugin-override2"]},{"task_type":"java_task","plugin_id":["plugin-override3","plugin-override3"],"missing_plugin_behavior":1}]}`) + }) + t.Run("successful get project domain attribute and write to file", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + pluginoverride.DefaultFetchConfig.AttrFile = testDataTempFile + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, `wrote the config to file temp-output-file`) + }) + t.Run("successful get project domain attribute and write to file failure", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + pluginoverride.DefaultFetchConfig.AttrFile = testDataNotExistentTempFile + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error dumping in file due to open non-existent-dir/temp-output-file: no such file or directory"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, ``) + }) + t.Run("failed get project domain attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, ``) + }) + t.Run("successful get workflow attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(workflowResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","workflow":"workflow","overrides":[{"task_type":"python_task","plugin_id":["plugin-override1","plugin-override2"]},{"task_type":"java_task","plugin_id":["plugin-override3","plugin-override3"],"missing_plugin_behavior":1}]}`) + }) + t.Run("failed get workflow attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, ``) + }) +} diff --git a/cmd/update/matchable_plugin_override.go b/cmd/update/matchable_plugin_override.go new file mode 100644 index 000000000..e617aac20 --- /dev/null +++ b/cmd/update/matchable_plugin_override.go @@ -0,0 +1,86 @@ +package update + +import ( + "context" + "fmt" + + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + cmdCore "github.com/flyteorg/flytectl/cmd/core" +) + +const ( + pluginOverrideShort = "Updates matchable resources of plugin overrides" + pluginOverrideLong = ` +Updates plugin overrides for given project and domain combination or additionally with workflow name. + +Updating to the plugin override is only available from a generated file. See the get section for generating this file. +Also this will completely overwrite any existing plugins overrides on custom project and domain and workflow combination. +Would be preferable to do get and generate an plugin override file if there is an existing override already set and then update it to have new values +Refer to get plugin-override section on how to generate this file +Here the command updates takes the input for plugin overrides from the config file po.yaml +eg: content of po.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Updating plugin override for project and domain and workflow combination. This will take precedence over any other +plugin overrides defined at project domain level. +Update the plugin overrides for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain + +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Usage + +` +) + +func updatePluginOverridesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + updateConfig := pluginoverride.DefaultUpdateConfig + if len(updateConfig.AttrFile) == 0 { + return fmt.Errorf("attrFile is mandatory while calling update for plugin override") + } + + pluginOverrideFileConfig := pluginoverride.FileConfig{} + if err := sconfig.ReadConfigFromFile(&pluginOverrideFileConfig, updateConfig.AttrFile); err != nil { + return err + } + + // Get project domain workflow name from the read file. + project := pluginOverrideFileConfig.Project + domain := pluginOverrideFileConfig.Domain + workflowName := pluginOverrideFileConfig.Workflow + + // Updates the admin matchable attribute from pluginOverrideFileConfig + if err := DecorateAndUpdateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminUpdaterExt(), + pluginOverrideFileConfig); err != nil { + return err + } + return nil +} diff --git a/cmd/update/matchable_plugin_override_test.go b/cmd/update/matchable_plugin_override_test.go new file mode 100644 index 000000000..8ca9cff07 --- /dev/null +++ b/cmd/update/matchable_plugin_override_test.go @@ -0,0 +1,94 @@ +package update + +import ( + "fmt" + "testing" + + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + u "github.com/flyteorg/flytectl/cmd/testutils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func updatePluginOverrideSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + pluginoverride.DefaultUpdateConfig = &pluginoverride.AttrUpdateConfig{} +} + +func TestPluginOverride(t *testing.T) { + t.Run("no input file for update", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + err = updatePluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("attrFile is mandatory while calling update for plugin override"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update project domain attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_project_domain_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = updatePluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update project domain attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_project_domain_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updatePluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update workflow attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_workflow_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update workflow attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_workflow_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("non existent file", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = testDataNonExistentFile + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to read from testdata/non-existent-file yaml file"), err) + tearDownAndVerify(t, ``) + }) + t.Run("invalid update file", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = testDataInvalidAttrFile + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), err) + tearDownAndVerify(t, ``) + }) +} diff --git a/cmd/update/testdata/valid_project_domain_plugin_override.yaml b/cmd/update/testdata/valid_project_domain_plugin_override.yaml new file mode 100644 index 000000000..a8ffc0fef --- /dev/null +++ b/cmd/update/testdata/valid_project_domain_plugin_override.yaml @@ -0,0 +1,8 @@ +domain: development +project: flytectldemo +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT diff --git a/cmd/update/testdata/valid_workflow_plugin_override.yaml b/cmd/update/testdata/valid_workflow_plugin_override.yaml new file mode 100644 index 000000000..6fbb58eae --- /dev/null +++ b/cmd/update/testdata/valid_workflow_plugin_override.yaml @@ -0,0 +1,9 @@ +domain: development +project: flytectldemo +workflow: core.control_flow.run_merge_sort.merge_sort +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT \ No newline at end of file diff --git a/cmd/update/update.go b/cmd/update/update.go index e58df7db2..34ab70c21 100644 --- a/cmd/update/update.go +++ b/cmd/update/update.go @@ -4,6 +4,7 @@ import ( "github.com/flyteorg/flytectl/cmd/config/subcommand/clusterresourceattribute" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionqueueattribute" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" "github.com/flyteorg/flytectl/cmd/config/subcommand/taskresourceattribute" cmdCore "github.com/flyteorg/flytectl/cmd/core" @@ -48,6 +49,8 @@ func CreateUpdateCommand() *cobra.Command { Short: executionQueueAttributesShort, Long: executionQueueAttributesLong, ProjectDomainNotRequired: true}, "execution-cluster-label": {CmdFunc: updateExecutionClusterLabelFunc, Aliases: []string{}, PFlagProvider: executionclusterlabel.DefaultUpdateConfig, Short: executionClusterLabelShort, Long: executionClusterLabelLong, ProjectDomainNotRequired: true}, + "plugin-override": {CmdFunc: updatePluginOverridesFunc, Aliases: []string{}, PFlagProvider: pluginoverride.DefaultUpdateConfig, + Short: pluginOverrideShort, Long: pluginOverrideLong, ProjectDomainNotRequired: true}, } cmdCore.AddCommands(updateCmd, updateResourcesFuncs) return updateCmd diff --git a/cmd/update/update_test.go b/cmd/update/update_test.go index 289ace5d0..3afcd4ae7 100644 --- a/cmd/update/update_test.go +++ b/cmd/update/update_test.go @@ -32,19 +32,19 @@ func TestUpdateCommand(t *testing.T) { assert.Equal(t, updateCommand.Use, updateUse) assert.Equal(t, updateCommand.Short, updateShort) assert.Equal(t, updateCommand.Long, updatecmdLong) - assert.Equal(t, len(updateCommand.Commands()), 8) + assert.Equal(t, len(updateCommand.Commands()), 9) cmdNouns := updateCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - useArray := []string{"cluster-resource-attribute", "execution-cluster-label", "execution-queue-attribute", "launchplan", "project", "task", - "task-resource-attribute", "workflow"} - aliases := [][]string{{}, {}, {}, {}, {}, {}, {}, {}} - shortArray := []string{clusterResourceAttributesShort, executionClusterLabelShort, executionQueueAttributesShort, updateLPShort, projectShort, updateTaskShort, - taskResourceAttributesShort, updateWorkflowShort} - longArray := []string{clusterResourceAttributesLong, executionClusterLabelLong, executionQueueAttributesLong, updateLPLong, projectLong, updateTaskLong, - taskResourceAttributesLong, updateWorkflowLong} + useArray := []string{"cluster-resource-attribute", "execution-cluster-label", "execution-queue-attribute", "launchplan", + "plugin-override", "project", "task", "task-resource-attribute", "workflow"} + aliases := [][]string{{}, {}, {}, {}, {}, {}, {}, {}, {}} + shortArray := []string{clusterResourceAttributesShort, executionClusterLabelShort, executionQueueAttributesShort, updateLPShort, + pluginOverrideShort, projectShort, updateTaskShort, taskResourceAttributesShort, updateWorkflowShort} + longArray := []string{clusterResourceAttributesLong, executionClusterLabelLong, executionQueueAttributesLong, updateLPLong, + pluginOverrideLong, projectLong, updateTaskLong, taskResourceAttributesLong, updateWorkflowLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/docs/source/gen/flytectl_delete.rst b/docs/source/gen/flytectl_delete.rst index c5699499b..d22416441 100644 --- a/docs/source/gen/flytectl_delete.rst +++ b/docs/source/gen/flytectl_delete.rst @@ -75,5 +75,6 @@ SEE ALSO * :doc:`flytectl_delete_execution` - Terminate/Delete execution resources. * :doc:`flytectl_delete_execution-cluster-label` - Deletes matchable resources of execution cluster label * :doc:`flytectl_delete_execution-queue-attribute` - Deletes matchable resources of execution queue attributes +* :doc:`flytectl_delete_plugin-override` - Deletes matchable resources of plugin overrides * :doc:`flytectl_delete_task-resource-attribute` - Deletes matchable resources of task attributes diff --git a/docs/source/gen/flytectl_delete_plugin-override.rst b/docs/source/gen/flytectl_delete_plugin-override.rst new file mode 100644 index 000000000..3553a3285 --- /dev/null +++ b/docs/source/gen/flytectl_delete_plugin-override.rst @@ -0,0 +1,113 @@ +.. _flytectl_delete_plugin-override: + +flytectl delete plugin-override +------------------------------- + +Deletes matchable resources of plugin overrides + +Synopsis +~~~~~~~~ + + + +Deletes plugin override for given project and domain combination or additionally with workflow name. + +Deletes plugin override for project and domain +Here the command deletes plugin override for project flytectldemo and development domain. +:: + + flytectl delete plugin-override -p flytectldemo -d development + + +Deletes plugin override using config file which was used for creating it. +Here the command deletes plugin overrides from the config file po.yaml +Overrides are optional in the file as they are unread during the delete command but can be kept as the same file can be used for get, update or delete +eg: content of po.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +Deletes plugin override for a workflow +Here the command deletes the plugin override for a workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl delete plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage + + +:: + + flytectl delete plugin-override [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for delete attribute for the resource type. + -h, --help help for plugin-override + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_delete` - Used for terminating/deleting various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/docs/source/gen/flytectl_get.rst b/docs/source/gen/flytectl_get.rst index 988d9ec08..4a0608a26 100644 --- a/docs/source/gen/flytectl_get.rst +++ b/docs/source/gen/flytectl_get.rst @@ -76,6 +76,7 @@ SEE ALSO * :doc:`flytectl_get_execution-cluster-label` - Gets matchable resources of execution cluster label * :doc:`flytectl_get_execution-queue-attribute` - Gets matchable resources of execution queue attributes * :doc:`flytectl_get_launchplan` - Gets launch plan resources +* :doc:`flytectl_get_plugin-override` - Gets matchable resources of plugin override * :doc:`flytectl_get_project` - Gets project resources * :doc:`flytectl_get_task` - Gets task resources * :doc:`flytectl_get_task-resource-attribute` - Gets matchable resources of task attributes diff --git a/docs/source/gen/flytectl_get_plugin-override.rst b/docs/source/gen/flytectl_get_plugin-override.rst new file mode 100644 index 000000000..0fbb195a5 --- /dev/null +++ b/docs/source/gen/flytectl_get_plugin-override.rst @@ -0,0 +1,141 @@ +.. _flytectl_get_plugin-override: + +flytectl get plugin-override +---------------------------- + +Gets matchable resources of plugin override + +Synopsis +~~~~~~~~ + + + +Retrieves plugin overrides for given project and domain combination or additionally with workflow name. + +Retrieves plugin overrides for project and domain +Here the command get plugin override for project flytectldemo and development domain. + +:: + + flytectl get plugin-override -p flytectldemo -d development + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Retrieves plugin override for project and domain and workflow +Here the command get plugin override for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl get plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "workflow": "core.control_flow.run_merge_sort.merge_sort" + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Writing the plugin override to a file. If there are no plugin overrides, command would return an error. +Here the command gets plugin overrides and writes the config file to po.yaml +eg: content of po.yaml + +:: + + flytectl get plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +Usage + + +:: + + flytectl get plugin-override [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for generating attribute for the resource type. + -h, --help help for plugin-override + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_get` - Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/docs/source/gen/flytectl_update.rst b/docs/source/gen/flytectl_update.rst index 2a7fa08d6..1372e0980 100644 --- a/docs/source/gen/flytectl_update.rst +++ b/docs/source/gen/flytectl_update.rst @@ -77,6 +77,7 @@ SEE ALSO * :doc:`flytectl_update_execution-cluster-label` - Updates matchable resources of execution cluster label * :doc:`flytectl_update_execution-queue-attribute` - Updates matchable resources of execution queue attributes * :doc:`flytectl_update_launchplan` - Updates launch plan metadata +* :doc:`flytectl_update_plugin-override` - Updates matchable resources of plugin overrides * :doc:`flytectl_update_project` - Updates project resources * :doc:`flytectl_update_task` - Updates task metadata * :doc:`flytectl_update_task-resource-attribute` - Updates matchable resources of task attributes diff --git a/docs/source/gen/flytectl_update_plugin-override.rst b/docs/source/gen/flytectl_update_plugin-override.rst new file mode 100644 index 000000000..172f950fb --- /dev/null +++ b/docs/source/gen/flytectl_update_plugin-override.rst @@ -0,0 +1,121 @@ +.. _flytectl_update_plugin-override: + +flytectl update plugin-override +------------------------------- + +Updates matchable resources of plugin overrides + +Synopsis +~~~~~~~~ + + + +Updates plugin overrides for given project and domain combination or additionally with workflow name. + +Updating to the plugin override is only available from a generated file. See the get section for generating this file. +Also this will completely overwrite any existing plugins overrides on custom project and domain and workflow combination. +Would be preferable to do get and generate an plugin override file if there is an existing override already set and then update it to have new values +Refer to get plugin-override section on how to generate this file +Here the command updates takes the input for plugin overrides from the config file po.yaml +eg: content of po.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Updating plugin override for project and domain and workflow combination. This will take precedence over any other +plugin overrides defined at project domain level. +Update the plugin overrides for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain + +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Usage + + + +:: + + flytectl update plugin-override [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for updating attribute for the resource type. + -h, --help help for plugin-override + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_update` - Used for updating flyte resources eg: project. + diff --git a/docs/source/nouns.rst b/docs/source/nouns.rst index 58151776b..89d3c74a3 100644 --- a/docs/source/nouns.rst +++ b/docs/source/nouns.rst @@ -16,6 +16,7 @@ Nouns gen/flytectl_get_cluster-resource-attribute gen/flytectl_get_execution-cluster-label gen/flytectl_get_execution-queue-attribute + gen/flytectl_get_plugin-override gen/flytectl_get_launchplan gen/flytectl_update_launchplan gen/flytectl_update_workflow @@ -25,12 +26,14 @@ Nouns gen/flytectl_update_cluster-resource-attribute gen/flytectl_update_execution-cluster-label gen/flytectl_update_execution-queue-attribute + gen/flytectl_update_plugin-override gen/flytectl_register_files gen/flytectl_delete_execution gen/flytectl_delete_task-resource-attribute gen/flytectl_delete_cluster-resource-attribute gen/flytectl_delete_execution-cluster-label gen/flytectl_delete_execution-queue-attribute + gen/flytectl_delete_plugin-override gen/flytectl_version gen/flytectl_config_validate gen/flytectl_config_discover