From 3de1fd3107ad82dbb957385f99380016849fa34c Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss <77798312+pmahindrakar-oss@users.noreply.github.com> Date: Sat, 15 May 2021 06:30:44 +0530 Subject: [PATCH] Adding support for execution cluster label (#68) --- .../attrdeleteconfig_flags.go | 46 ++++++ .../attrdeleteconfig_flags_test.go | 124 +++++++++++++++ .../attrfetchconfig_flags.go | 46 ++++++ .../attrfetchconfig_flags_test.go | 124 +++++++++++++++ .../attrupdateconfig_flags.go | 46 ++++++ .../attrupdateconfig_flags_test.go | 124 +++++++++++++++ .../executionclusterlabel/delete_config.go | 10 ++ .../executionclusterlabel/fetch_config.go | 9 ++ .../executionclusterlabel/file_config.go | 47 ++++++ .../executionclusterlabel/file_config_test.go | 46 ++++++ .../executionclusterlabel/update_config.go | 10 ++ .../plugin_override/attrdeleteconfig_flags.go | 46 ++++++ .../attrdeleteconfig_flags_test.go | 124 +++++++++++++++ .../plugin_override/attrfetchconfig_flags.go | 46 ++++++ .../attrfetchconfig_flags_test.go | 124 +++++++++++++++ .../plugin_override/attrupdateconfig_flags.go | 46 ++++++ .../attrupdateconfig_flags_test.go | 124 +++++++++++++++ .../plugin_override/delete_config.go | 10 ++ .../plugin_override/fetch_config.go | 9 ++ .../subcommand/plugin_override/file_config.go | 47 ++++++ .../plugin_override/file_config_test.go | 58 +++++++ .../plugin_override/update_config.go | 10 ++ cmd/delete/delete.go | 8 + cmd/delete/delete_test.go | 10 +- .../matchable_cluster_resource_attribute.go | 2 +- .../matchable_execution_cluster_label.go | 78 +++++++++ .../matchable_execution_cluster_label_test.go | 134 ++++++++++++++++ .../matchable_execution_queue_attribute.go | 2 +- cmd/delete/matchable_plugin_override.go | 83 ++++++++++ cmd/delete/matchable_plugin_override_test.go | 134 ++++++++++++++++ .../matchable_task_resource_attribute.go | 2 +- ...roject_domain_execution_cluster_label.yaml | 3 + .../valid_project_domain_plugin_override.yaml | 8 + ...alid_workflow_execution_cluster_label.yaml | 4 + .../valid_workflow_plugin_override.yaml | 9 ++ cmd/get/get.go | 10 +- cmd/get/get_test.go | 18 +-- .../matchable_cluster_resource_attribute.go | 2 +- cmd/get/matchable_execution_cluster_label.go | 88 +++++++++++ .../matchable_execution_cluster_label_test.go | 138 ++++++++++++++++ .../matchable_execution_queue_attribute.go | 2 +- cmd/get/matchable_plugin_override.go | 112 +++++++++++++ cmd/get/matchable_plugin_override_test.go | 148 ++++++++++++++++++ cmd/get/matchable_task_resource_attribute.go | 2 +- .../matchable_cluster_resource_attribute.go | 2 +- .../matchable_execution_cluster_label.go | 73 +++++++++ .../matchable_execution_cluster_label_test.go | 94 +++++++++++ .../matchable_execution_queue_attribute.go | 2 +- cmd/update/matchable_plugin_override.go | 86 ++++++++++ cmd/update/matchable_plugin_override_test.go | 94 +++++++++++ .../matchable_task_resource_attribute.go | 2 +- ...roject_domain_execution_cluster_label.yaml | 3 + .../valid_project_domain_plugin_override.yaml | 8 + ...alid_workflow_execution_cluster_label.yaml | 4 + .../valid_workflow_plugin_override.yaml | 9 ++ cmd/update/update.go | 6 + cmd/update/update_test.go | 16 +- docs/source/gen/flytectl_delete.rst | 2 + ...ectl_delete_cluster-resource-attribute.rst | 2 +- ...lytectl_delete_execution-cluster-label.rst | 108 +++++++++++++ ...tectl_delete_execution-queue-attribute.rst | 2 +- .../gen/flytectl_delete_plugin-override.rst | 113 +++++++++++++ ...lytectl_delete_task-resource-attribute.rst | 2 +- docs/source/gen/flytectl_get.rst | 4 +- ...lytectl_get_cluster-resource-attribute.rst | 2 +- .../flytectl_get_execution-cluster-label.rst | 117 ++++++++++++++ ...flytectl_get_execution-queue-attribute.rst | 2 +- .../gen/flytectl_get_plugin-override.rst | 141 +++++++++++++++++ .../flytectl_get_task-resource-attribute.rst | 2 +- docs/source/gen/flytectl_update.rst | 2 + ...ectl_update_cluster-resource-attribute.rst | 2 +- ...lytectl_update_execution-cluster-label.rst | 108 +++++++++++++ ...tectl_update_execution-queue-attribute.rst | 2 +- .../gen/flytectl_update_plugin-override.rst | 121 ++++++++++++++ ...lytectl_update_task-resource-attribute.rst | 2 +- docs/source/nouns.rst | 6 + 76 files changed, 3370 insertions(+), 42 deletions(-) create mode 100755 cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags.go create mode 100755 cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags_test.go create mode 100755 cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags.go create mode 100755 cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags_test.go create mode 100755 cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags.go create mode 100755 cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags_test.go create mode 100644 cmd/config/subcommand/executionclusterlabel/delete_config.go create mode 100644 cmd/config/subcommand/executionclusterlabel/fetch_config.go create mode 100644 cmd/config/subcommand/executionclusterlabel/file_config.go create mode 100644 cmd/config/subcommand/executionclusterlabel/file_config_test.go create mode 100644 cmd/config/subcommand/executionclusterlabel/update_config.go create mode 100755 cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go create mode 100755 cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go create mode 100755 cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go create mode 100755 cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go create mode 100755 cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go create mode 100755 cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go create mode 100644 cmd/config/subcommand/plugin_override/delete_config.go create mode 100644 cmd/config/subcommand/plugin_override/fetch_config.go create mode 100644 cmd/config/subcommand/plugin_override/file_config.go create mode 100644 cmd/config/subcommand/plugin_override/file_config_test.go create mode 100644 cmd/config/subcommand/plugin_override/update_config.go create mode 100644 cmd/delete/matchable_execution_cluster_label.go create mode 100644 cmd/delete/matchable_execution_cluster_label_test.go create mode 100644 cmd/delete/matchable_plugin_override.go create mode 100644 cmd/delete/matchable_plugin_override_test.go create mode 100644 cmd/delete/testdata/valid_project_domain_execution_cluster_label.yaml create mode 100644 cmd/delete/testdata/valid_project_domain_plugin_override.yaml create mode 100644 cmd/delete/testdata/valid_workflow_execution_cluster_label.yaml create mode 100644 cmd/delete/testdata/valid_workflow_plugin_override.yaml create mode 100644 cmd/get/matchable_execution_cluster_label.go create mode 100644 cmd/get/matchable_execution_cluster_label_test.go create mode 100644 cmd/get/matchable_plugin_override.go create mode 100644 cmd/get/matchable_plugin_override_test.go create mode 100644 cmd/update/matchable_execution_cluster_label.go create mode 100644 cmd/update/matchable_execution_cluster_label_test.go create mode 100644 cmd/update/matchable_plugin_override.go create mode 100644 cmd/update/matchable_plugin_override_test.go create mode 100644 cmd/update/testdata/valid_project_domain_execution_cluster_label.yaml create mode 100644 cmd/update/testdata/valid_project_domain_plugin_override.yaml create mode 100644 cmd/update/testdata/valid_workflow_execution_cluster_label.yaml create mode 100644 cmd/update/testdata/valid_workflow_plugin_override.yaml create mode 100644 docs/source/gen/flytectl_delete_execution-cluster-label.rst create mode 100644 docs/source/gen/flytectl_delete_plugin-override.rst create mode 100644 docs/source/gen/flytectl_get_execution-cluster-label.rst create mode 100644 docs/source/gen/flytectl_get_plugin-override.rst create mode 100644 docs/source/gen/flytectl_update_execution-cluster-label.rst create mode 100644 docs/source/gen/flytectl_update_plugin-override.rst diff --git a/cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags.go b/cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags.go new file mode 100755 index 0000000000..3dc56c9837 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package executionclusterlabel + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrDeleteConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrDeleteConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrDeleteConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrDeleteConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrDeleteConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultDelConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultDelConfig.AttrFile, "attribute file name to be used for delete attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags_test.go b/cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags_test.go new file mode 100755 index 0000000000..02ec1217d1 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/attrdeleteconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package executionclusterlabel + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrDeleteConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrDeleteConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrDeleteConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrDeleteConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrDeleteConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrDeleteConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrDeleteConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrDeleteConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrDeleteConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrDeleteConfig(val, result)) +} + +func testDecodeSlice_AttrDeleteConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrDeleteConfig(vStringSlice, result)) +} + +func TestAttrDeleteConfig_GetPFlagSet(t *testing.T) { + val := AttrDeleteConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrDeleteConfig_SetFlags(t *testing.T) { + actual := AttrDeleteConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultDelConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrDeleteConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags.go b/cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags.go new file mode 100755 index 0000000000..0e1d0d060f --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package executionclusterlabel + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrFetchConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrFetchConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrFetchConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrFetchConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrFetchConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultFetchConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultFetchConfig.AttrFile, "attribute file name to be used for generating attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags_test.go b/cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags_test.go new file mode 100755 index 0000000000..a99fd91933 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/attrfetchconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package executionclusterlabel + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrFetchConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrFetchConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrFetchConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrFetchConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrFetchConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrFetchConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrFetchConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrFetchConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrFetchConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrFetchConfig(val, result)) +} + +func testDecodeSlice_AttrFetchConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrFetchConfig(vStringSlice, result)) +} + +func TestAttrFetchConfig_GetPFlagSet(t *testing.T) { + val := AttrFetchConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrFetchConfig_SetFlags(t *testing.T) { + actual := AttrFetchConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultFetchConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrFetchConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags.go b/cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags.go new file mode 100755 index 0000000000..f2f0d84852 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package executionclusterlabel + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrUpdateConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrUpdateConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrUpdateConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrUpdateConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrUpdateConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultUpdateConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultUpdateConfig.AttrFile, "attribute file name to be used for updating attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags_test.go b/cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags_test.go new file mode 100755 index 0000000000..ca7e09eb4d --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/attrupdateconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package executionclusterlabel + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrUpdateConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrUpdateConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrUpdateConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrUpdateConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrUpdateConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrUpdateConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrUpdateConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrUpdateConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrUpdateConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrUpdateConfig(val, result)) +} + +func testDecodeSlice_AttrUpdateConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrUpdateConfig(vStringSlice, result)) +} + +func TestAttrUpdateConfig_GetPFlagSet(t *testing.T) { + val := AttrUpdateConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrUpdateConfig_SetFlags(t *testing.T) { + actual := AttrUpdateConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultUpdateConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrUpdateConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/executionclusterlabel/delete_config.go b/cmd/config/subcommand/executionclusterlabel/delete_config.go new file mode 100644 index 0000000000..5a96d6fdfa --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/delete_config.go @@ -0,0 +1,10 @@ +package executionclusterlabel + +//go:generate pflags AttrDeleteConfig --default-var DefaultDelConfig + +// AttrDeleteConfig Matchable resource attributes configuration passed from command line +type AttrDeleteConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for delete attribute for the resource type."` +} + +var DefaultDelConfig = &AttrDeleteConfig{} diff --git a/cmd/config/subcommand/executionclusterlabel/fetch_config.go b/cmd/config/subcommand/executionclusterlabel/fetch_config.go new file mode 100644 index 0000000000..8e7f878ede --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/fetch_config.go @@ -0,0 +1,9 @@ +package executionclusterlabel + +//go:generate pflags AttrFetchConfig --default-var DefaultFetchConfig + +type AttrFetchConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for generating attribute for the resource type."` +} + +var DefaultFetchConfig = &AttrFetchConfig{} diff --git a/cmd/config/subcommand/executionclusterlabel/file_config.go b/cmd/config/subcommand/executionclusterlabel/file_config.go new file mode 100644 index 0000000000..c05c6ca167 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/file_config.go @@ -0,0 +1,47 @@ +package executionclusterlabel + +import ( + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +// FileConfig shadow Config for ExecutionClusterLabel. +// The shadow Config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. +// As the same structure is being used for both ProjectDomainAttribute/Workflowattribute +type FileConfig struct { + Project string `json:"project"` + Domain string `json:"domain"` + Workflow string `json:"workflow,omitempty"` + *admin.ExecutionClusterLabel +} + +// Decorate decorator over ExecutionClusterLabel. +func (t FileConfig) Decorate() *admin.MatchingAttributes { + return &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_ExecutionClusterLabel{ + ExecutionClusterLabel: t.ExecutionClusterLabel, + }, + } +} + +// UnDecorate to uncover ExecutionClusterLabel. +func (t *FileConfig) UnDecorate(matchingAttribute *admin.MatchingAttributes) { + if matchingAttribute == nil { + return + } + t.ExecutionClusterLabel = matchingAttribute.GetExecutionClusterLabel() +} + +// GetProject from the FileConfig +func (t FileConfig) GetProject() string { + return t.Project +} + +// GetDomain from the FileConfig +func (t FileConfig) GetDomain() string { + return t.Domain +} + +// GetWorkflow from the FileConfig +func (t FileConfig) GetWorkflow() string { + return t.Workflow +} diff --git a/cmd/config/subcommand/executionclusterlabel/file_config_test.go b/cmd/config/subcommand/executionclusterlabel/file_config_test.go new file mode 100644 index 0000000000..662f6658a4 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/file_config_test.go @@ -0,0 +1,46 @@ +package executionclusterlabel + +import ( + "testing" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" +) + +func TestFileConfig(t *testing.T) { + execClusterLabelFileConfig := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + ExecutionClusterLabel: &admin.ExecutionClusterLabel{ + Value: "foo", + }, + } + matchingAttr := &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_ExecutionClusterLabel{ + ExecutionClusterLabel: execClusterLabelFileConfig.ExecutionClusterLabel, + }, + } + t.Run("decorate", func(t *testing.T) { + assert.Equal(t, matchingAttr, execClusterLabelFileConfig.Decorate()) + }) + + t.Run("decorate", func(t *testing.T) { + taskAttrFileConfigNew := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + } + taskAttrFileConfigNew.UnDecorate(matchingAttr) + assert.Equal(t, execClusterLabelFileConfig, taskAttrFileConfigNew) + }) + t.Run("get project domain workflow", func(t *testing.T) { + taskAttrFileConfigNew := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + Workflow: "workflow", + } + assert.Equal(t, "dummyProject", taskAttrFileConfigNew.GetProject()) + assert.Equal(t, "dummyDomain", taskAttrFileConfigNew.GetDomain()) + assert.Equal(t, "workflow", taskAttrFileConfigNew.GetWorkflow()) + }) +} diff --git a/cmd/config/subcommand/executionclusterlabel/update_config.go b/cmd/config/subcommand/executionclusterlabel/update_config.go new file mode 100644 index 0000000000..abd5600553 --- /dev/null +++ b/cmd/config/subcommand/executionclusterlabel/update_config.go @@ -0,0 +1,10 @@ +package executionclusterlabel + +//go:generate pflags AttrUpdateConfig --default-var DefaultUpdateConfig + +// AttrUpdateConfig Matchable resource attributes configuration passed from command line +type AttrUpdateConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for updating attribute for the resource type."` +} + +var DefaultUpdateConfig = &AttrUpdateConfig{} diff --git a/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go new file mode 100755 index 0000000000..52fd144b99 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrDeleteConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrDeleteConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrDeleteConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrDeleteConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrDeleteConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultDelConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultDelConfig.AttrFile, "attribute file name to be used for delete attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go new file mode 100755 index 0000000000..7a5d60873c --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrdeleteconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrDeleteConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrDeleteConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrDeleteConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrDeleteConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrDeleteConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrDeleteConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrDeleteConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrDeleteConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrDeleteConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrDeleteConfig(val, result)) +} + +func testDecodeSlice_AttrDeleteConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrDeleteConfig(vStringSlice, result)) +} + +func TestAttrDeleteConfig_GetPFlagSet(t *testing.T) { + val := AttrDeleteConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrDeleteConfig_SetFlags(t *testing.T) { + actual := AttrDeleteConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultDelConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrDeleteConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go new file mode 100755 index 0000000000..1cf522b3ca --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrFetchConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrFetchConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrFetchConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrFetchConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrFetchConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultFetchConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultFetchConfig.AttrFile, "attribute file name to be used for generating attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go new file mode 100755 index 0000000000..e94f54e982 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrfetchconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrFetchConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrFetchConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrFetchConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrFetchConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrFetchConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrFetchConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrFetchConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrFetchConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrFetchConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrFetchConfig(val, result)) +} + +func testDecodeSlice_AttrFetchConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrFetchConfig(vStringSlice, result)) +} + +func TestAttrFetchConfig_GetPFlagSet(t *testing.T) { + val := AttrFetchConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrFetchConfig_SetFlags(t *testing.T) { + actual := AttrFetchConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultFetchConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrFetchConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go new file mode 100755 index 0000000000..21c58dbb9d --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (AttrUpdateConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (AttrUpdateConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in AttrUpdateConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg AttrUpdateConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("AttrUpdateConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultUpdateConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultUpdateConfig.AttrFile, "attribute file name to be used for updating attribute for the resource type.") + return cmdFlags +} diff --git a/cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go new file mode 100755 index 0000000000..37ee0382c1 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/attrupdateconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package pluginoverride + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsAttrUpdateConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementAttrUpdateConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsAttrUpdateConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookAttrUpdateConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementAttrUpdateConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_AttrUpdateConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookAttrUpdateConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_AttrUpdateConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_AttrUpdateConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_AttrUpdateConfig(val, result)) +} + +func testDecodeSlice_AttrUpdateConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_AttrUpdateConfig(vStringSlice, result)) +} + +func TestAttrUpdateConfig_GetPFlagSet(t *testing.T) { + val := AttrUpdateConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestAttrUpdateConfig_SetFlags(t *testing.T) { + actual := AttrUpdateConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultUpdateConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_AttrUpdateConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/cmd/config/subcommand/plugin_override/delete_config.go b/cmd/config/subcommand/plugin_override/delete_config.go new file mode 100644 index 0000000000..beec54fd95 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/delete_config.go @@ -0,0 +1,10 @@ +package pluginoverride + +//go:generate pflags AttrDeleteConfig --default-var DefaultDelConfig + +// AttrDeleteConfig Matchable resource attributes configuration passed from command line +type AttrDeleteConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for delete attribute for the resource type."` +} + +var DefaultDelConfig = &AttrDeleteConfig{} diff --git a/cmd/config/subcommand/plugin_override/fetch_config.go b/cmd/config/subcommand/plugin_override/fetch_config.go new file mode 100644 index 0000000000..9976d8b33f --- /dev/null +++ b/cmd/config/subcommand/plugin_override/fetch_config.go @@ -0,0 +1,9 @@ +package pluginoverride + +//go:generate pflags AttrFetchConfig --default-var DefaultFetchConfig + +type AttrFetchConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for generating attribute for the resource type."` +} + +var DefaultFetchConfig = &AttrFetchConfig{} diff --git a/cmd/config/subcommand/plugin_override/file_config.go b/cmd/config/subcommand/plugin_override/file_config.go new file mode 100644 index 0000000000..af40066dd4 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/file_config.go @@ -0,0 +1,47 @@ +package pluginoverride + +import ( + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +// FileConfig shadow Config for PluginOverrides. +// The shadow Config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. +// As the same structure is being used for both ProjectDomainAttribute/Workflowattribute +type FileConfig struct { + Project string `json:"project"` + Domain string `json:"domain"` + Workflow string `json:"workflow,omitempty"` + *admin.PluginOverrides +} + +// Decorate decorator over PluginOverrides. +func (t FileConfig) Decorate() *admin.MatchingAttributes { + return &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: t.PluginOverrides, + }, + } +} + +// UnDecorate to uncover PluginOverrides. +func (t *FileConfig) UnDecorate(matchingAttribute *admin.MatchingAttributes) { + if matchingAttribute == nil { + return + } + t.PluginOverrides = matchingAttribute.GetPluginOverrides() +} + +// GetProject from the FileConfig +func (t FileConfig) GetProject() string { + return t.Project +} + +// GetDomain from the FileConfig +func (t FileConfig) GetDomain() string { + return t.Domain +} + +// GetWorkflow from the FileConfig +func (t FileConfig) GetWorkflow() string { + return t.Workflow +} diff --git a/cmd/config/subcommand/plugin_override/file_config_test.go b/cmd/config/subcommand/plugin_override/file_config_test.go new file mode 100644 index 0000000000..e2ecaa5d06 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/file_config_test.go @@ -0,0 +1,58 @@ +package pluginoverride + +import ( + "testing" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" +) + +func TestFileConfig(t *testing.T) { + pluginOverride1 := &admin.PluginOverride{ + TaskType: "python_task", + PluginId: []string{"plugin-override1", "plugin-override2"}, + MissingPluginBehavior: admin.PluginOverride_FAIL, + } + pluginOverride2 := &admin.PluginOverride{ + TaskType: "java_task", + PluginId: []string{"plugin-override3", "plugin-override3"}, + MissingPluginBehavior: admin.PluginOverride_USE_DEFAULT, + } + pluginOverrides := []*admin.PluginOverride{pluginOverride1, pluginOverride2} + + pluginOverrideFileConfig := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + PluginOverrides: &admin.PluginOverrides{ + Overrides: pluginOverrides, + }, + } + matchingAttr := &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: pluginOverrideFileConfig.PluginOverrides, + }, + } + t.Run("decorate", func(t *testing.T) { + assert.Equal(t, matchingAttr, pluginOverrideFileConfig.Decorate()) + }) + + t.Run("decorate", func(t *testing.T) { + taskAttrFileConfigNew := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + } + taskAttrFileConfigNew.UnDecorate(matchingAttr) + assert.Equal(t, pluginOverrideFileConfig, taskAttrFileConfigNew) + }) + t.Run("get project domain workflow", func(t *testing.T) { + taskAttrFileConfigNew := FileConfig{ + Project: "dummyProject", + Domain: "dummyDomain", + Workflow: "workflow", + } + assert.Equal(t, "dummyProject", taskAttrFileConfigNew.GetProject()) + assert.Equal(t, "dummyDomain", taskAttrFileConfigNew.GetDomain()) + assert.Equal(t, "workflow", taskAttrFileConfigNew.GetWorkflow()) + }) +} diff --git a/cmd/config/subcommand/plugin_override/update_config.go b/cmd/config/subcommand/plugin_override/update_config.go new file mode 100644 index 0000000000..e2daefa8f3 --- /dev/null +++ b/cmd/config/subcommand/plugin_override/update_config.go @@ -0,0 +1,10 @@ +package pluginoverride + +//go:generate pflags AttrUpdateConfig --default-var DefaultUpdateConfig + +// AttrUpdateConfig Matchable resource attributes configuration passed from command line +type AttrUpdateConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for updating attribute for the resource type."` +} + +var DefaultUpdateConfig = &AttrUpdateConfig{} diff --git a/cmd/delete/delete.go b/cmd/delete/delete.go index 5a64a58ce3..94ce8e801f 100644 --- a/cmd/delete/delete.go +++ b/cmd/delete/delete.go @@ -2,7 +2,9 @@ package delete import ( "github.com/flyteorg/flytectl/cmd/config/subcommand/clusterresourceattribute" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionqueueattribute" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" "github.com/flyteorg/flytectl/cmd/config/subcommand/taskresourceattribute" cmdcore "github.com/flyteorg/flytectl/cmd/core" @@ -36,9 +38,15 @@ func RemoteDeleteCommand() *cobra.Command { "cluster-resource-attribute": {CmdFunc: deleteClusterResourceAttributes, Aliases: []string{"cluster-resource-attributes"}, Short: clusterResourceAttributesShort, Long: clusterResourceAttributesLong, PFlagProvider: clusterresourceattribute.DefaultDelConfig, ProjectDomainNotRequired: true}, + "execution-cluster-label": {CmdFunc: deleteExecutionClusterLabel, Aliases: []string{"execution-cluster-labels"}, + Short: executionClusterLabelShort, + Long: executionClusterLabelLong, PFlagProvider: executionclusterlabel.DefaultDelConfig, ProjectDomainNotRequired: true}, "execution-queue-attribute": {CmdFunc: deleteExecutionQueueAttributes, Aliases: []string{"execution-queue-attributes"}, Short: executionQueueAttributesShort, Long: executionQueueAttributesLong, PFlagProvider: executionqueueattribute.DefaultDelConfig, ProjectDomainNotRequired: true}, + "plugin-override": {CmdFunc: deletePluginOverride, Aliases: []string{"plugin-overrides"}, + Short: pluginOverrideShort, + Long: pluginOverrideLong, PFlagProvider: pluginoverride.DefaultDelConfig, ProjectDomainNotRequired: true}, } cmdcore.AddCommands(deleteCmd, terminateResourcesFuncs) return deleteCmd diff --git a/cmd/delete/delete_test.go b/cmd/delete/delete_test.go index 3f5739c9cf..e26a1d7a9d 100644 --- a/cmd/delete/delete_test.go +++ b/cmd/delete/delete_test.go @@ -32,16 +32,16 @@ func TestDeleteCommand(t *testing.T) { assert.Equal(t, deleteCommand.Use, "delete") assert.Equal(t, deleteCommand.Short, deleteCmdShort) assert.Equal(t, deleteCommand.Long, deleteCmdLong) - assert.Equal(t, len(deleteCommand.Commands()), 4) + assert.Equal(t, len(deleteCommand.Commands()), 6) cmdNouns := deleteCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - useArray := []string{"cluster-resource-attribute", "execution", "execution-queue-attribute", "task-resource-attribute"} - aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-queue-attributes"}, {"task-resource-attributes"}} - shortArray := []string{clusterResourceAttributesShort, execCmdShort, executionQueueAttributesShort, taskResourceAttributesShort} - longArray := []string{clusterResourceAttributesLong, execCmdLong, executionQueueAttributesLong, taskResourceAttributesLong} + useArray := []string{"cluster-resource-attribute", "execution", "execution-cluster-label", "execution-queue-attribute", "plugin-override", "task-resource-attribute"} + aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-cluster-labels"}, {"execution-queue-attributes"}, {"plugin-overrides"}, {"task-resource-attributes"}} + shortArray := []string{clusterResourceAttributesShort, execCmdShort, executionClusterLabelShort, executionQueueAttributesShort, pluginOverrideShort, taskResourceAttributesShort} + longArray := []string{clusterResourceAttributesLong, execCmdLong, executionClusterLabelLong, executionQueueAttributesLong, pluginOverrideLong, taskResourceAttributesLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/cmd/delete/matchable_cluster_resource_attribute.go b/cmd/delete/matchable_cluster_resource_attribute.go index 9dd6bc526b..e6b41941c0 100644 --- a/cmd/delete/matchable_cluster_resource_attribute.go +++ b/cmd/delete/matchable_cluster_resource_attribute.go @@ -41,7 +41,7 @@ eg: content of cra.yaml which will use the project domain and workflow name for buzz: "lightyear" Deletes cluster resource attribute for a workflow -Here the command deletes cluster resource attributes for a workflow +Here the command deletes cluster resource attributes for a workflow core.control_flow.run_merge_sort.merge_sort :: diff --git a/cmd/delete/matchable_execution_cluster_label.go b/cmd/delete/matchable_execution_cluster_label.go new file mode 100644 index 0000000000..65ab8f7405 --- /dev/null +++ b/cmd/delete/matchable_execution_cluster_label.go @@ -0,0 +1,78 @@ +package delete + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + executionClusterLabelShort = "Deletes matchable resources of execution cluster label" + executionClusterLabelLong = ` +Deletes execution cluster label for given project and domain combination or additionally with workflow name. + +Deletes execution cluster label for project and domain +Here the command delete execution cluster label for project flytectldemo and development domain. +:: + + flytectl delete execution-cluster-label -p flytectldemo -d development + + +Deletes execution cluster label using config file which was used for creating it. +Here the command deletes execution cluster label from the config file ecl.yaml +Value is optional in the file as its unread during the delete command but can be kept as the same file can be used for get, update or delete +eg: content of ecl.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete execution-cluster-label --attrFile ecl.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + value: foo + +Deletes execution cluster label for a workflow +Here the command deletes execution cluster label for a workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl delete execution-cluster-label -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage +` +) + +func deleteExecutionClusterLabel(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var pwdGetter sconfig.ProjectDomainWorkflowGetter + pwdGetter = sconfig.PDWGetterCommandLine{Config: config.GetConfig(), Args: args} + delConfig := executionclusterlabel.DefaultDelConfig + + // Get the project domain workflowName from the config file or commandline params + if len(delConfig.AttrFile) > 0 { + // Initialize FileConfig which will be used if delConfig.AttrFile is non empty + // And Reads from the cluster label file + pwdGetter = &executionclusterlabel.FileConfig{} + if err := sconfig.ReadConfigFromFile(pwdGetter, delConfig.AttrFile); err != nil { + return err + } + } + // Use the pwdGetter to initialize the project domain and workflow + project := pwdGetter.GetProject() + domain := pwdGetter.GetDomain() + workflowName := pwdGetter.GetWorkflow() + + // Deletes the matchable attributes using the ExecClusterLabelFileConfig + if err := deleteMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminDeleterExt(), + admin.MatchableResource_EXECUTION_CLUSTER_LABEL); err != nil { + return err + } + + return nil +} diff --git a/cmd/delete/matchable_execution_cluster_label_test.go b/cmd/delete/matchable_execution_cluster_label_test.go new file mode 100644 index 0000000000..fe90153dbf --- /dev/null +++ b/cmd/delete/matchable_execution_cluster_label_test.go @@ -0,0 +1,134 @@ +package delete + +import ( + "fmt" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func deleteExecutionClusterLabelSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + executionclusterlabel.DefaultDelConfig = &executionclusterlabel.AttrDeleteConfig{} + args = []string{} +} + +func TestDeleteExecutionClusterLabels(t *testing.T) { + t.Run("successful project domain attribute deletion commandline", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = "" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("failed project domain attribute deletion", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to delte project domain attributes")) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delte project domain attributes"), err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("successful project domain attribute deletion file", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = "testdata/valid_project_domain_execution_cluster_label.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("successful workflow attribute deletion", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("failed workflow attribute deletion", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to delete workflow attribute")) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delete workflow attribute"), err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("successful workflow attribute deletion file", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = "testdata/valid_workflow_execution_cluster_label.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("workflow attribute deletion non existent file", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = testDataNonExistentFile + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + u.DeleterExt.AssertNotCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) + t.Run("attribute deletion invalid file", func(t *testing.T) { + setup() + deleteExecutionClusterLabelSetup() + // Empty attribute file + executionclusterlabel.DefaultDelConfig.AttrFile = testDataInvalidAttrFile + // No args implying project domain attribute deletion + err = deleteExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, + fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), + err) + u.DeleterExt.AssertNotCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + }) +} diff --git a/cmd/delete/matchable_execution_queue_attribute.go b/cmd/delete/matchable_execution_queue_attribute.go index 2f11ce5c5b..392157a03f 100644 --- a/cmd/delete/matchable_execution_queue_attribute.go +++ b/cmd/delete/matchable_execution_queue_attribute.go @@ -43,7 +43,7 @@ eg: content of era.yaml which will use the project domain and workflow name for - lightyear Deletes execution queue attribute for a workflow -Here the command deletes the execution queue attributes for a workflow +Here the command deletes the execution queue attributes for a workflow core.control_flow.run_merge_sort.merge_sort :: diff --git a/cmd/delete/matchable_plugin_override.go b/cmd/delete/matchable_plugin_override.go new file mode 100644 index 0000000000..09353a83f8 --- /dev/null +++ b/cmd/delete/matchable_plugin_override.go @@ -0,0 +1,83 @@ +package delete + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + pluginOverrideShort = "Deletes matchable resources of plugin overrides" + pluginOverrideLong = ` +Deletes plugin override for given project and domain combination or additionally with workflow name. + +Deletes plugin override for project and domain +Here the command deletes plugin override for project flytectldemo and development domain. +:: + + flytectl delete plugin-override -p flytectldemo -d development + + +Deletes plugin override using config file which was used for creating it. +Here the command deletes plugin overrides from the config file po.yaml +Overrides are optional in the file as they are unread during the delete command but can be kept as the same file can be used for get, update or delete +eg: content of po.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +Deletes plugin override for a workflow +Here the command deletes the plugin override for a workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl delete plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage +` +) + +func deletePluginOverride(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var pwdGetter sconfig.ProjectDomainWorkflowGetter + pwdGetter = sconfig.PDWGetterCommandLine{Config: config.GetConfig(), Args: args} + delConfig := pluginoverride.DefaultDelConfig + + // Get the project domain workflowName from the config file or commandline params + if len(delConfig.AttrFile) > 0 { + // Initialize AttrFileConfig which will be used if delConfig.AttrFile is non empty + // And Reads from the attribute file + pwdGetter = &pluginoverride.FileConfig{} + if err := sconfig.ReadConfigFromFile(pwdGetter, delConfig.AttrFile); err != nil { + return err + } + } + // Use the pwdGetter to initialize the project domain and workflow + project := pwdGetter.GetProject() + domain := pwdGetter.GetDomain() + workflowName := pwdGetter.GetWorkflow() + + // Deletes the matchable attributes using the AttrFileConfig + if err := deleteMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminDeleterExt(), + admin.MatchableResource_PLUGIN_OVERRIDE); err != nil { + return err + } + + return nil +} diff --git a/cmd/delete/matchable_plugin_override_test.go b/cmd/delete/matchable_plugin_override_test.go new file mode 100644 index 0000000000..d8ed34818f --- /dev/null +++ b/cmd/delete/matchable_plugin_override_test.go @@ -0,0 +1,134 @@ +package delete + +import ( + "fmt" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func deletePluginOverrideSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + pluginoverride.DefaultDelConfig = &pluginoverride.AttrDeleteConfig{} + args = []string{} +} + +func TestPluginOverride(t *testing.T) { + t.Run("successful project domain attribute deletion commandline", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("failed project domain attribute deletion", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to delte project domain attributes")) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delte project domain attributes"), err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("successful project domain attribute deletion file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "testdata/valid_project_domain_plugin_override.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("successful workflow attribute deletion", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("failed workflow attribute deletion", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to delete workflow attribute")) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delete workflow attribute"), err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("successful workflow attribute deletion file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = "testdata/valid_workflow_plugin_override.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("workflow attribute deletion non existent file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = testDataNonExistentFile + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + u.DeleterExt.AssertNotCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_PLUGIN_OVERRIDE) + }) + t.Run("attribute deletion invalid file", func(t *testing.T) { + setup() + deletePluginOverrideSetup() + // Empty attribute file + pluginoverride.DefaultDelConfig.AttrFile = testDataInvalidAttrFile + // No args implying project domain attribute deletion + err = deletePluginOverride(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, + fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), + err) + u.DeleterExt.AssertNotCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_PLUGIN_OVERRIDE) + }) +} diff --git a/cmd/delete/matchable_task_resource_attribute.go b/cmd/delete/matchable_task_resource_attribute.go index cafc354714..3f1a719ca7 100644 --- a/cmd/delete/matchable_task_resource_attribute.go +++ b/cmd/delete/matchable_task_resource_attribute.go @@ -44,7 +44,7 @@ eg: content of tra.yaml which will use the project domain and workflow name for memory: "450Mi" Deletes task resource attribute for a workflow -Here the command deletes task resource attributes for a workflow +Here the command deletes task resource attributes for a workflow core.control_flow.run_merge_sort.merge_sort :: diff --git a/cmd/delete/testdata/valid_project_domain_execution_cluster_label.yaml b/cmd/delete/testdata/valid_project_domain_execution_cluster_label.yaml new file mode 100644 index 0000000000..37f8a630c2 --- /dev/null +++ b/cmd/delete/testdata/valid_project_domain_execution_cluster_label.yaml @@ -0,0 +1,3 @@ +domain: development +project: flytectldemo +value: foo \ No newline at end of file diff --git a/cmd/delete/testdata/valid_project_domain_plugin_override.yaml b/cmd/delete/testdata/valid_project_domain_plugin_override.yaml new file mode 100644 index 0000000000..a8ffc0fef8 --- /dev/null +++ b/cmd/delete/testdata/valid_project_domain_plugin_override.yaml @@ -0,0 +1,8 @@ +domain: development +project: flytectldemo +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT diff --git a/cmd/delete/testdata/valid_workflow_execution_cluster_label.yaml b/cmd/delete/testdata/valid_workflow_execution_cluster_label.yaml new file mode 100644 index 0000000000..ccd978fa76 --- /dev/null +++ b/cmd/delete/testdata/valid_workflow_execution_cluster_label.yaml @@ -0,0 +1,4 @@ +domain: development +project: flytectldemo +workflow: core.control_flow.run_merge_sort.merge_sort +value: foo \ No newline at end of file diff --git a/cmd/delete/testdata/valid_workflow_plugin_override.yaml b/cmd/delete/testdata/valid_workflow_plugin_override.yaml new file mode 100644 index 0000000000..6fbb58eae0 --- /dev/null +++ b/cmd/delete/testdata/valid_workflow_plugin_override.yaml @@ -0,0 +1,9 @@ +domain: development +project: flytectldemo +workflow: core.control_flow.run_merge_sort.merge_sort +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT \ No newline at end of file diff --git a/cmd/get/get.go b/cmd/get/get.go index 10f1e6bdb1..b78bc4f67d 100644 --- a/cmd/get/get.go +++ b/cmd/get/get.go @@ -2,7 +2,9 @@ package get import ( "github.com/flyteorg/flytectl/cmd/config/subcommand/clusterresourceattribute" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionqueueattribute" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" "github.com/flyteorg/flytectl/cmd/config/subcommand/taskresourceattribute" "github.com/flyteorg/flytectl/cmd/config/subcommand/workflow" cmdcore "github.com/flyteorg/flytectl/cmd/core" @@ -17,7 +19,7 @@ const ( Example get projects. :: - bin/flytectl get project + flytectl get project ` ) @@ -50,6 +52,12 @@ func CreateGetCommand() *cobra.Command { "execution-queue-attribute": {CmdFunc: getExecutionQueueAttributes, Aliases: []string{"execution-queue-attributes"}, Short: executionQueueAttributesShort, Long: executionQueueAttributesLong, PFlagProvider: executionqueueattribute.DefaultFetchConfig}, + "execution-cluster-label": {CmdFunc: getExecutionClusterLabel, Aliases: []string{"execution-cluster-labels"}, + Short: executionClusterLabelShort, + Long: executionClusterLabelLong, PFlagProvider: executionclusterlabel.DefaultFetchConfig}, + "plugin-override": {CmdFunc: getPluginOverridesFunc, Aliases: []string{"plugin-overrides"}, + Short: pluginOverrideShort, + Long: pluginOverrideLong, PFlagProvider: pluginoverride.DefaultFetchConfig}, } cmdcore.AddCommands(getCmd, getResourcesFuncs) diff --git a/cmd/get/get_test.go b/cmd/get/get_test.go index 1f567dacc3..5b513ccd47 100644 --- a/cmd/get/get_test.go +++ b/cmd/get/get_test.go @@ -42,20 +42,20 @@ func TestCreateGetCommand(t *testing.T) { assert.Equal(t, getCommand.Use, "get") assert.Equal(t, getCommand.Short, "Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project.") fmt.Println(getCommand.Commands()) - assert.Equal(t, len(getCommand.Commands()), 8) + assert.Equal(t, len(getCommand.Commands()), 10) cmdNouns := getCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - useArray := []string{"cluster-resource-attribute", "execution", "execution-queue-attribute", "launchplan", "project", - "task", "task-resource-attribute", "workflow"} - aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-queue-attributes"}, {"launchplans"}, {"projects"}, {"tasks"}, - {"task-resource-attributes"}, {"workflows"}} - shortArray := []string{clusterResourceAttributesShort, executionShort, executionQueueAttributesShort, launchPlanShort, projectShort, - taskShort, taskResourceAttributesShort, workflowShort} - longArray := []string{clusterResourceAttributesLong, executionLong, executionQueueAttributesLong, launchPlanLong, projectLong, taskLong, - taskResourceAttributesLong, workflowLong} + useArray := []string{"cluster-resource-attribute", "execution", "execution-cluster-label", + "execution-queue-attribute", "launchplan", "plugin-override", "project", "task", "task-resource-attribute", "workflow"} + aliases := [][]string{{"cluster-resource-attributes"}, {"executions"}, {"execution-cluster-labels"}, + {"execution-queue-attributes"}, {"launchplans"}, {"plugin-overrides"}, {"projects"}, {"tasks"}, {"task-resource-attributes"}, {"workflows"}} + shortArray := []string{clusterResourceAttributesShort, executionShort, executionClusterLabelShort, executionQueueAttributesShort, launchPlanShort, + pluginOverrideShort, projectShort, taskShort, taskResourceAttributesShort, workflowShort} + longArray := []string{clusterResourceAttributesLong, executionLong, executionClusterLabelLong, executionQueueAttributesLong, launchPlanLong, + pluginOverrideLong, projectLong, taskLong, taskResourceAttributesLong, workflowLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/cmd/get/matchable_cluster_resource_attribute.go b/cmd/get/matchable_cluster_resource_attribute.go index 7df8190d09..e29f61d5a1 100644 --- a/cmd/get/matchable_cluster_resource_attribute.go +++ b/cmd/get/matchable_cluster_resource_attribute.go @@ -28,7 +28,7 @@ eg : output from the command {"project":"flytectldemo","domain":"development","attributes":{"buzz":"lightyear","foo":"bar"}} Retrieves cluster resource attribute for project and domain and workflow -Here the command get cluster resource attributes for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort +Here the command get cluster resource attributes for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort :: flytectl get cluster-resource-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort diff --git a/cmd/get/matchable_execution_cluster_label.go b/cmd/get/matchable_execution_cluster_label.go new file mode 100644 index 0000000000..56e9e7b94c --- /dev/null +++ b/cmd/get/matchable_execution_cluster_label.go @@ -0,0 +1,88 @@ +package get + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + executionClusterLabelShort = "Gets matchable resources of execution cluster label" + executionClusterLabelLong = ` +Retrieves execution cluster label for given project and domain combination or additionally with workflow name. + +Retrieves execution cluster label for project and domain +Here the command get execution cluster label for project flytectldemo and development domain. +:: + + flytectl get execution-cluster-label -p flytectldemo -d development + +eg : output from the command + +.. code-block:: json + + {"project":"flytectldemo","domain":"development","value":"foo"} + +Retrieves execution cluster label for project and domain and workflow +Here the command get execution cluster label for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort +:: + + flytectl get execution-cluster-label -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +eg : output from the command + +.. code-block:: json + + {"project":"flytectldemo","domain":"development","workflow":"core.control_flow.run_merge_sort.merge_sort","value":"foo"} + +Writing the execution cluster label to a file. If there are no execution cluster label, command would return an error. +Here the command gets execution cluster label and writes the config file to ecl.yaml +eg: content of ecl.yaml + +:: + + flytectl get execution-cluster-label --attrFile ecl.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + value: foo + +Usage +` +) + +func getExecutionClusterLabel(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var project string + var domain string + var workflowName string + + // Get the project domain workflow name parameters from the command line. Project and domain are mandatory for this command + project = config.GetConfig().Project + domain = config.GetConfig().Domain + if len(args) == 1 { + workflowName = args[0] + } + // Construct a shadow config for ExecutionClusterLabel. The shadow config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. + executionClusterLabelFileConfig := executionclusterlabel.FileConfig{Project: project, Domain: domain, Workflow: workflowName} + // Get the attribute file name from the command line config + fileName := executionclusterlabel.DefaultFetchConfig.AttrFile + + // Updates the taskResourceAttrFileConfig with the fetched matchable attribute + if err := FetchAndUnDecorateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminFetcherExt(), + &executionClusterLabelFileConfig, admin.MatchableResource_EXECUTION_CLUSTER_LABEL); err != nil { + return err + } + + // Write the config to the file which can be used for update + if err := sconfig.DumpTaskResourceAttr(executionClusterLabelFileConfig, fileName); err != nil { + return err + } + return nil +} diff --git a/cmd/get/matchable_execution_cluster_label_test.go b/cmd/get/matchable_execution_cluster_label_test.go new file mode 100644 index 0000000000..04231373a3 --- /dev/null +++ b/cmd/get/matchable_execution_cluster_label_test.go @@ -0,0 +1,138 @@ +package get + +import ( + "fmt" + "os" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getExecutionClusterLabelSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + executionclusterlabel.DefaultFetchConfig = &executionclusterlabel.AttrFetchConfig{} + // Clean up the temp directory. + _ = os.Remove(testDataTempFile) +} + +func TestGetExecutionClusterLabel(t *testing.T) { + executionClusterLabel := &admin.ExecutionClusterLabel{ + Value: "foo", + } + projectDomainResp := &admin.ProjectDomainAttributesGetResponse{ + Attributes: &admin.ProjectDomainAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_ExecutionClusterLabel{ + ExecutionClusterLabel: executionClusterLabel, + }, + }, + }, + } + workflowResp := &admin.WorkflowAttributesGetResponse{ + Attributes: &admin.WorkflowAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + Workflow: "workflow", + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_ExecutionClusterLabel{ + ExecutionClusterLabel: executionClusterLabel, + }, + }, + }, + } + t.Run("successful get project domain attribute", func(t *testing.T) { + var args []string + setup() + getExecutionClusterLabelSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","value":"foo"}`) + }) + t.Run("successful get project domain attribute and write to file", func(t *testing.T) { + var args []string + setup() + getExecutionClusterLabelSetup() + executionclusterlabel.DefaultFetchConfig.AttrFile = testDataTempFile + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + tearDownAndVerify(t, `wrote the config to file temp-output-file`) + }) + t.Run("successful get project domain attribute and write to file failure", func(t *testing.T) { + var args []string + setup() + getExecutionClusterLabelSetup() + executionclusterlabel.DefaultFetchConfig.AttrFile = testDataNotExistentTempFile + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error dumping in file due to open non-existent-dir/temp-output-file: no such file or directory"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + tearDownAndVerify(t, ``) + }) + t.Run("failed get project domain attribute", func(t *testing.T) { + var args []string + setup() + getExecutionClusterLabelSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + tearDownAndVerify(t, ``) + }) + t.Run("successful get workflow attribute", func(t *testing.T) { + var args []string + setup() + getExecutionClusterLabelSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(workflowResp, nil) + err = getExecutionClusterLabel(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", + admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","workflow":"workflow","value":"foo"}`) + }) + t.Run("failed get workflow attribute", func(t *testing.T) { + var args []string + setup() + getExecutionClusterLabelSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getExecutionClusterLabel(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", + admin.MatchableResource_EXECUTION_CLUSTER_LABEL) + tearDownAndVerify(t, ``) + }) +} diff --git a/cmd/get/matchable_execution_queue_attribute.go b/cmd/get/matchable_execution_queue_attribute.go index 07e51e3d91..b51517e595 100644 --- a/cmd/get/matchable_execution_queue_attribute.go +++ b/cmd/get/matchable_execution_queue_attribute.go @@ -28,7 +28,7 @@ eg : output from the command {"project":"flytectldemo","domain":"development","tags":["foo", "bar"]} Retrieves execution queue attribute for project and domain and workflow -Here the command get execution queue attributes for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort +Here the command get execution queue attributes for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort :: flytectl get execution-queue-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort diff --git a/cmd/get/matchable_plugin_override.go b/cmd/get/matchable_plugin_override.go new file mode 100644 index 0000000000..7c42a35219 --- /dev/null +++ b/cmd/get/matchable_plugin_override.go @@ -0,0 +1,112 @@ +package get + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + pluginOverrideShort = "Gets matchable resources of plugin override" + pluginOverrideLong = ` +Retrieves plugin overrides for given project and domain combination or additionally with workflow name. + +Retrieves plugin overrides for project and domain +Here the command get plugin override for project flytectldemo and development domain. + +:: + + flytectl get plugin-override -p flytectldemo -d development + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Retrieves plugin override for project and domain and workflow +Here the command get plugin override for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl get plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "workflow": "core.control_flow.run_merge_sort.merge_sort" + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Writing the plugin override to a file. If there are no plugin overrides, command would return an error. +Here the command gets plugin overrides and writes the config file to po.yaml +eg: content of po.yaml + +:: + + flytectl get plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +Usage +` +) + +func getPluginOverridesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var project string + var domain string + var workflowName string + + // Get the project domain workflow name parameters from the command line. Project and domain are mandatory for this command + project = config.GetConfig().Project + domain = config.GetConfig().Domain + if len(args) == 1 { + workflowName = args[0] + } + // Construct a shadow config for PluginOverrides. The shadow config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. + pluginOverrideFileConfig := pluginoverride.FileConfig{Project: project, Domain: domain, Workflow: workflowName} + // Get the plugin overrides from the command line config + fileName := pluginoverride.DefaultFetchConfig.AttrFile + + // Updates the pluginOverrideFileConfig with the fetched matchable attribute + if err := FetchAndUnDecorateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminFetcherExt(), + &pluginOverrideFileConfig, admin.MatchableResource_PLUGIN_OVERRIDE); err != nil { + return err + } + + // Write the config to the file which can be used for update + if err := sconfig.DumpTaskResourceAttr(pluginOverrideFileConfig, fileName); err != nil { + return err + } + return nil +} diff --git a/cmd/get/matchable_plugin_override_test.go b/cmd/get/matchable_plugin_override_test.go new file mode 100644 index 0000000000..b72ab6524e --- /dev/null +++ b/cmd/get/matchable_plugin_override_test.go @@ -0,0 +1,148 @@ +package get + +import ( + "fmt" + "os" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getPluginOverrideSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + pluginoverride.DefaultFetchConfig = &pluginoverride.AttrFetchConfig{} + // Clean up the temp directory. + _ = os.Remove(testDataTempFile) +} + +func TestGetPluginOverride(t *testing.T) { + pluginOverride1 := &admin.PluginOverride{ + TaskType: "python_task", + PluginId: []string{"plugin-override1", "plugin-override2"}, + MissingPluginBehavior: admin.PluginOverride_FAIL, + } + pluginOverride2 := &admin.PluginOverride{ + TaskType: "java_task", + PluginId: []string{"plugin-override3", "plugin-override3"}, + MissingPluginBehavior: admin.PluginOverride_USE_DEFAULT, + } + pluginOverrides := []*admin.PluginOverride{pluginOverride1, pluginOverride2} + projectDomainResp := &admin.ProjectDomainAttributesGetResponse{ + Attributes: &admin.ProjectDomainAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: &admin.PluginOverrides{ + Overrides: pluginOverrides, + }, + }, + }, + }, + } + workflowResp := &admin.WorkflowAttributesGetResponse{ + Attributes: &admin.WorkflowAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + Workflow: "workflow", + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_PluginOverrides{ + PluginOverrides: &admin.PluginOverrides{ + Overrides: pluginOverrides, + }, + }, + }, + }, + } + t.Run("successful get project domain attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","overrides":[{"task_type":"python_task","plugin_id":["plugin-override1","plugin-override2"]},{"task_type":"java_task","plugin_id":["plugin-override3","plugin-override3"],"missing_plugin_behavior":1}]}`) + }) + t.Run("successful get project domain attribute and write to file", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + pluginoverride.DefaultFetchConfig.AttrFile = testDataTempFile + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, `wrote the config to file temp-output-file`) + }) + t.Run("successful get project domain attribute and write to file failure", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + pluginoverride.DefaultFetchConfig.AttrFile = testDataNotExistentTempFile + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error dumping in file due to open non-existent-dir/temp-output-file: no such file or directory"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, ``) + }) + t.Run("failed get project domain attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, ``) + }) + t.Run("successful get workflow attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(workflowResp, nil) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","workflow":"workflow","overrides":[{"task_type":"python_task","plugin_id":["plugin-override1","plugin-override2"]},{"task_type":"java_task","plugin_id":["plugin-override3","plugin-override3"],"missing_plugin_behavior":1}]}`) + }) + t.Run("failed get workflow attribute", func(t *testing.T) { + var args []string + setup() + getPluginOverrideSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getPluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", admin.MatchableResource_PLUGIN_OVERRIDE) + tearDownAndVerify(t, ``) + }) +} diff --git a/cmd/get/matchable_task_resource_attribute.go b/cmd/get/matchable_task_resource_attribute.go index b3ba8023c4..74c8a9a99d 100644 --- a/cmd/get/matchable_task_resource_attribute.go +++ b/cmd/get/matchable_task_resource_attribute.go @@ -28,7 +28,7 @@ eg : output from the command {"project":"flytectldemo","domain":"development","workflow":"","defaults":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"2","memory":"450Mi"}} Retrieves task resource attribute for project and domain and workflow -Here the command get task resource attributes for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort +Here the command get task resource attributes for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort :: flytectl get task-resource-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort diff --git a/cmd/update/matchable_cluster_resource_attribute.go b/cmd/update/matchable_cluster_resource_attribute.go index 77b0821c7c..c41b74f23c 100644 --- a/cmd/update/matchable_cluster_resource_attribute.go +++ b/cmd/update/matchable_cluster_resource_attribute.go @@ -35,7 +35,7 @@ resource attribute defined at project domain level. Also this will completely overwrite any existing custom project and domain and workflow combination attributes. Would be preferable to do get and generate an attribute file if there is an existing attribute already set and then update it to have new values Refer to get cluster-resource-attribute section on how to generate this file -Update the cluster resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +Update the cluster resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain .. code-block:: yaml diff --git a/cmd/update/matchable_execution_cluster_label.go b/cmd/update/matchable_execution_cluster_label.go new file mode 100644 index 0000000000..0a3eb6965c --- /dev/null +++ b/cmd/update/matchable_execution_cluster_label.go @@ -0,0 +1,73 @@ +package update + +import ( + "context" + "fmt" + + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" + cmdCore "github.com/flyteorg/flytectl/cmd/core" +) + +const ( + executionClusterLabelShort = "Updates matchable resources of execution cluster label" + executionClusterLabelLong = ` +Updates execution cluster label for given project and domain combination or additionally with workflow name. + +Updating to the execution cluster label is only available from a generated file. See the get section for generating this file. +Here the command updates takes the input for execution cluster label from the config file ecl.yaml +eg: content of ecl.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + value: foo + +:: + + flytectl update execution-cluster-label --attrFile ecl.yaml + +Updating execution cluster label for project and domain and workflow combination. This will take precedence over any other +execution cluster label defined at project domain level. +Update the execution cluster label for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain + +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + value: foo + +:: + + flytectl update execution-cluster-label --attrFile ecl.yaml + +Usage + +` +) + +func updateExecutionClusterLabelFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + updateConfig := executionclusterlabel.DefaultUpdateConfig + if len(updateConfig.AttrFile) == 0 { + return fmt.Errorf("attrFile is mandatory while calling update for execution cluster label") + } + + executionClusterLabelFileConfig := executionclusterlabel.FileConfig{} + if err := sconfig.ReadConfigFromFile(&executionClusterLabelFileConfig, updateConfig.AttrFile); err != nil { + return err + } + + // Get project domain workflow name from the read file. + project := executionClusterLabelFileConfig.Project + domain := executionClusterLabelFileConfig.Domain + workflowName := executionClusterLabelFileConfig.Workflow + + // Updates the admin matchable attribute from executionClusterLabelFileConfig + if err := DecorateAndUpdateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminUpdaterExt(), + executionClusterLabelFileConfig); err != nil { + return err + } + return nil +} diff --git a/cmd/update/matchable_execution_cluster_label_test.go b/cmd/update/matchable_execution_cluster_label_test.go new file mode 100644 index 0000000000..b22b444b7a --- /dev/null +++ b/cmd/update/matchable_execution_cluster_label_test.go @@ -0,0 +1,94 @@ +package update + +import ( + "fmt" + "testing" + + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" + u "github.com/flyteorg/flytectl/cmd/testutils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func updateExecutionClusterLabelSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + executionclusterlabel.DefaultUpdateConfig = &executionclusterlabel.AttrUpdateConfig{} +} + +func TestExecutionClusterLabel(t *testing.T) { + t.Run("no input file for update", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + err = updateExecutionClusterLabelFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("attrFile is mandatory while calling update for execution cluster label"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update project domain attribute", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + executionclusterlabel.DefaultUpdateConfig.AttrFile = "testdata/valid_project_domain_execution_cluster_label.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = updateExecutionClusterLabelFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update project domain attribute", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + executionclusterlabel.DefaultUpdateConfig.AttrFile = "testdata/valid_project_domain_execution_cluster_label.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updateExecutionClusterLabelFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update workflow attribute", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + executionclusterlabel.DefaultUpdateConfig.AttrFile = "testdata/valid_workflow_execution_cluster_label.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = updateExecutionClusterLabelFunc(ctx, nil, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update workflow attribute", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + executionclusterlabel.DefaultUpdateConfig.AttrFile = "testdata/valid_workflow_execution_cluster_label.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updateExecutionClusterLabelFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("non existent file", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + executionclusterlabel.DefaultUpdateConfig.AttrFile = testDataNonExistentFile + err = updateExecutionClusterLabelFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to read from testdata/non-existent-file yaml file"), err) + tearDownAndVerify(t, ``) + }) + t.Run("invalid update file", func(t *testing.T) { + setup() + updateExecutionClusterLabelSetup() + executionclusterlabel.DefaultUpdateConfig.AttrFile = testDataInvalidAttrFile + err = updateExecutionClusterLabelFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), err) + tearDownAndVerify(t, ``) + }) +} diff --git a/cmd/update/matchable_execution_queue_attribute.go b/cmd/update/matchable_execution_queue_attribute.go index 676c351396..f48bfa2d3b 100644 --- a/cmd/update/matchable_execution_queue_attribute.go +++ b/cmd/update/matchable_execution_queue_attribute.go @@ -37,7 +37,7 @@ eg: content of era.yaml Updating execution queue attribute for project and domain and workflow combination. This will take precedence over any other execution queue attribute defined at project domain level. -Update the execution queue attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +Update the execution queue attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain .. code-block:: yaml diff --git a/cmd/update/matchable_plugin_override.go b/cmd/update/matchable_plugin_override.go new file mode 100644 index 0000000000..ceeaca0d96 --- /dev/null +++ b/cmd/update/matchable_plugin_override.go @@ -0,0 +1,86 @@ +package update + +import ( + "context" + "fmt" + + sconfig "github.com/flyteorg/flytectl/cmd/config/subcommand" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + cmdCore "github.com/flyteorg/flytectl/cmd/core" +) + +const ( + pluginOverrideShort = "Updates matchable resources of plugin overrides" + pluginOverrideLong = ` +Updates plugin overrides for given project and domain combination or additionally with workflow name. + +Updating to the plugin override is only available from a generated file. See the get section for generating this file. +Also this will completely overwrite any existing plugins overrides on custom project and domain and workflow combination. +Would be preferable to do get and generate an plugin override file if there is an existing override already set and then update it to have new values +Refer to get plugin-override section on how to generate this file +Here the command updates takes the input for plugin overrides from the config file po.yaml +eg: content of po.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Updating plugin override for project and domain and workflow combination. This will take precedence over any other +plugin overrides defined at project domain level. +Update the plugin overrides for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain + +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Usage + +` +) + +func updatePluginOverridesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + updateConfig := pluginoverride.DefaultUpdateConfig + if len(updateConfig.AttrFile) == 0 { + return fmt.Errorf("attrFile is mandatory while calling update for plugin override") + } + + pluginOverrideFileConfig := pluginoverride.FileConfig{} + if err := sconfig.ReadConfigFromFile(&pluginOverrideFileConfig, updateConfig.AttrFile); err != nil { + return err + } + + // Get project domain workflow name from the read file. + project := pluginOverrideFileConfig.Project + domain := pluginOverrideFileConfig.Domain + workflowName := pluginOverrideFileConfig.Workflow + + // Updates the admin matchable attribute from pluginOverrideFileConfig + if err := DecorateAndUpdateMatchableAttr(ctx, project, domain, workflowName, cmdCtx.AdminUpdaterExt(), + pluginOverrideFileConfig); err != nil { + return err + } + return nil +} diff --git a/cmd/update/matchable_plugin_override_test.go b/cmd/update/matchable_plugin_override_test.go new file mode 100644 index 0000000000..8ca9cff072 --- /dev/null +++ b/cmd/update/matchable_plugin_override_test.go @@ -0,0 +1,94 @@ +package update + +import ( + "fmt" + "testing" + + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" + u "github.com/flyteorg/flytectl/cmd/testutils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func updatePluginOverrideSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + pluginoverride.DefaultUpdateConfig = &pluginoverride.AttrUpdateConfig{} +} + +func TestPluginOverride(t *testing.T) { + t.Run("no input file for update", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + err = updatePluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("attrFile is mandatory while calling update for plugin override"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update project domain attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_project_domain_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = updatePluginOverridesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update project domain attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_project_domain_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updatePluginOverridesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update workflow attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_workflow_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update workflow attribute", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = "testdata/valid_workflow_plugin_override.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("non existent file", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = testDataNonExistentFile + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to read from testdata/non-existent-file yaml file"), err) + tearDownAndVerify(t, ``) + }) + t.Run("invalid update file", func(t *testing.T) { + setup() + updatePluginOverrideSetup() + pluginoverride.DefaultUpdateConfig.AttrFile = testDataInvalidAttrFile + err = updatePluginOverridesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), err) + tearDownAndVerify(t, ``) + }) +} diff --git a/cmd/update/matchable_task_resource_attribute.go b/cmd/update/matchable_task_resource_attribute.go index 3e351baa95..1719dc9c40 100644 --- a/cmd/update/matchable_task_resource_attribute.go +++ b/cmd/update/matchable_task_resource_attribute.go @@ -38,7 +38,7 @@ eg: content of tra.yaml Updating task resource attribute for project and domain and workflow combination. This will take precedence over any other resource attribute defined at project domain level. -Update the resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +Update the resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain .. code-block:: yaml diff --git a/cmd/update/testdata/valid_project_domain_execution_cluster_label.yaml b/cmd/update/testdata/valid_project_domain_execution_cluster_label.yaml new file mode 100644 index 0000000000..37f8a630c2 --- /dev/null +++ b/cmd/update/testdata/valid_project_domain_execution_cluster_label.yaml @@ -0,0 +1,3 @@ +domain: development +project: flytectldemo +value: foo \ No newline at end of file diff --git a/cmd/update/testdata/valid_project_domain_plugin_override.yaml b/cmd/update/testdata/valid_project_domain_plugin_override.yaml new file mode 100644 index 0000000000..a8ffc0fef8 --- /dev/null +++ b/cmd/update/testdata/valid_project_domain_plugin_override.yaml @@ -0,0 +1,8 @@ +domain: development +project: flytectldemo +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT diff --git a/cmd/update/testdata/valid_workflow_execution_cluster_label.yaml b/cmd/update/testdata/valid_workflow_execution_cluster_label.yaml new file mode 100644 index 0000000000..ccd978fa76 --- /dev/null +++ b/cmd/update/testdata/valid_workflow_execution_cluster_label.yaml @@ -0,0 +1,4 @@ +domain: development +project: flytectldemo +workflow: core.control_flow.run_merge_sort.merge_sort +value: foo \ No newline at end of file diff --git a/cmd/update/testdata/valid_workflow_plugin_override.yaml b/cmd/update/testdata/valid_workflow_plugin_override.yaml new file mode 100644 index 0000000000..6fbb58eae0 --- /dev/null +++ b/cmd/update/testdata/valid_workflow_plugin_override.yaml @@ -0,0 +1,9 @@ +domain: development +project: flytectldemo +workflow: core.control_flow.run_merge_sort.merge_sort +overrides: + - task_type: python_task + plugin_id: + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # 0 : FAIL , 1: DEFAULT \ No newline at end of file diff --git a/cmd/update/update.go b/cmd/update/update.go index 5ea12d3899..34ab70c21c 100644 --- a/cmd/update/update.go +++ b/cmd/update/update.go @@ -2,7 +2,9 @@ package update import ( "github.com/flyteorg/flytectl/cmd/config/subcommand/clusterresourceattribute" + "github.com/flyteorg/flytectl/cmd/config/subcommand/executionclusterlabel" "github.com/flyteorg/flytectl/cmd/config/subcommand/executionqueueattribute" + pluginoverride "github.com/flyteorg/flytectl/cmd/config/subcommand/plugin_override" "github.com/flyteorg/flytectl/cmd/config/subcommand/taskresourceattribute" cmdCore "github.com/flyteorg/flytectl/cmd/core" @@ -45,6 +47,10 @@ func CreateUpdateCommand() *cobra.Command { Short: clusterResourceAttributesShort, Long: clusterResourceAttributesLong, ProjectDomainNotRequired: true}, "execution-queue-attribute": {CmdFunc: updateExecutionQueueAttributesFunc, Aliases: []string{}, PFlagProvider: executionqueueattribute.DefaultUpdateConfig, Short: executionQueueAttributesShort, Long: executionQueueAttributesLong, ProjectDomainNotRequired: true}, + "execution-cluster-label": {CmdFunc: updateExecutionClusterLabelFunc, Aliases: []string{}, PFlagProvider: executionclusterlabel.DefaultUpdateConfig, + Short: executionClusterLabelShort, Long: executionClusterLabelLong, ProjectDomainNotRequired: true}, + "plugin-override": {CmdFunc: updatePluginOverridesFunc, Aliases: []string{}, PFlagProvider: pluginoverride.DefaultUpdateConfig, + Short: pluginOverrideShort, Long: pluginOverrideLong, ProjectDomainNotRequired: true}, } cmdCore.AddCommands(updateCmd, updateResourcesFuncs) return updateCmd diff --git a/cmd/update/update_test.go b/cmd/update/update_test.go index 953bc712ab..3afcd4ae70 100644 --- a/cmd/update/update_test.go +++ b/cmd/update/update_test.go @@ -32,19 +32,19 @@ func TestUpdateCommand(t *testing.T) { assert.Equal(t, updateCommand.Use, updateUse) assert.Equal(t, updateCommand.Short, updateShort) assert.Equal(t, updateCommand.Long, updatecmdLong) - assert.Equal(t, len(updateCommand.Commands()), 7) + assert.Equal(t, len(updateCommand.Commands()), 9) cmdNouns := updateCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - useArray := []string{"cluster-resource-attribute", "execution-queue-attribute", "launchplan", "project", "task", - "task-resource-attribute", "workflow"} - aliases := [][]string{{}, {}, {}, {}, {}, {}, {}} - shortArray := []string{clusterResourceAttributesShort, executionQueueAttributesShort, updateLPShort, projectShort, updateTaskShort, - taskResourceAttributesShort, updateWorkflowShort} - longArray := []string{clusterResourceAttributesLong, executionQueueAttributesLong, updateLPLong, projectLong, updateTaskLong, - taskResourceAttributesLong, updateWorkflowLong} + useArray := []string{"cluster-resource-attribute", "execution-cluster-label", "execution-queue-attribute", "launchplan", + "plugin-override", "project", "task", "task-resource-attribute", "workflow"} + aliases := [][]string{{}, {}, {}, {}, {}, {}, {}, {}, {}} + shortArray := []string{clusterResourceAttributesShort, executionClusterLabelShort, executionQueueAttributesShort, updateLPShort, + pluginOverrideShort, projectShort, updateTaskShort, taskResourceAttributesShort, updateWorkflowShort} + longArray := []string{clusterResourceAttributesLong, executionClusterLabelLong, executionQueueAttributesLong, updateLPLong, + pluginOverrideLong, projectLong, updateTaskLong, taskResourceAttributesLong, updateWorkflowLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/docs/source/gen/flytectl_delete.rst b/docs/source/gen/flytectl_delete.rst index f766162a09..d22416441d 100644 --- a/docs/source/gen/flytectl_delete.rst +++ b/docs/source/gen/flytectl_delete.rst @@ -73,6 +73,8 @@ SEE ALSO * :doc:`flytectl` - flyetcl CLI tool * :doc:`flytectl_delete_cluster-resource-attribute` - Deletes matchable resources of cluster attributes * :doc:`flytectl_delete_execution` - Terminate/Delete execution resources. +* :doc:`flytectl_delete_execution-cluster-label` - Deletes matchable resources of execution cluster label * :doc:`flytectl_delete_execution-queue-attribute` - Deletes matchable resources of execution queue attributes +* :doc:`flytectl_delete_plugin-override` - Deletes matchable resources of plugin overrides * :doc:`flytectl_delete_task-resource-attribute` - Deletes matchable resources of task attributes diff --git a/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst b/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst index 9971f0bc79..ee2e26a889 100644 --- a/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst +++ b/docs/source/gen/flytectl_delete_cluster-resource-attribute.rst @@ -38,7 +38,7 @@ eg: content of cra.yaml which will use the project domain and workflow name for buzz: "lightyear" Deletes cluster resource attribute for a workflow -Here the command deletes cluster resource attributes for a workflow +Here the command deletes cluster resource attributes for a workflow core.control_flow.run_merge_sort.merge_sort :: diff --git a/docs/source/gen/flytectl_delete_execution-cluster-label.rst b/docs/source/gen/flytectl_delete_execution-cluster-label.rst new file mode 100644 index 0000000000..9290a5444b --- /dev/null +++ b/docs/source/gen/flytectl_delete_execution-cluster-label.rst @@ -0,0 +1,108 @@ +.. _flytectl_delete_execution-cluster-label: + +flytectl delete execution-cluster-label +--------------------------------------- + +Deletes matchable resources of execution cluster label + +Synopsis +~~~~~~~~ + + + +Deletes execution cluster label for given project and domain combination or additionally with workflow name. + +Deletes execution cluster label for project and domain +Here the command delete execution cluster label for project flytectldemo and development domain. +:: + + flytectl delete execution-cluster-label -p flytectldemo -d development + + +Deletes execution cluster label using config file which was used for creating it. +Here the command deletes execution cluster label from the config file ecl.yaml +Value is optional in the file as its unread during the delete command but can be kept as the same file can be used for get, update or delete +eg: content of ecl.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete execution-cluster-label --attrFile ecl.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + value: foo + +Deletes execution cluster label for a workflow +Here the command deletes execution cluster label for a workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl delete execution-cluster-label -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage + + +:: + + flytectl delete execution-cluster-label [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for delete attribute for the resource type. + -h, --help help for execution-cluster-label + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_delete` - Used for terminating/deleting various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/docs/source/gen/flytectl_delete_execution-queue-attribute.rst b/docs/source/gen/flytectl_delete_execution-queue-attribute.rst index d016f47ced..20c4f4506c 100644 --- a/docs/source/gen/flytectl_delete_execution-queue-attribute.rst +++ b/docs/source/gen/flytectl_delete_execution-queue-attribute.rst @@ -40,7 +40,7 @@ eg: content of era.yaml which will use the project domain and workflow name for - lightyear Deletes execution queue attribute for a workflow -Here the command deletes the execution queue attributes for a workflow +Here the command deletes the execution queue attributes for a workflow core.control_flow.run_merge_sort.merge_sort :: diff --git a/docs/source/gen/flytectl_delete_plugin-override.rst b/docs/source/gen/flytectl_delete_plugin-override.rst new file mode 100644 index 0000000000..f76d01d2f3 --- /dev/null +++ b/docs/source/gen/flytectl_delete_plugin-override.rst @@ -0,0 +1,113 @@ +.. _flytectl_delete_plugin-override: + +flytectl delete plugin-override +------------------------------- + +Deletes matchable resources of plugin overrides + +Synopsis +~~~~~~~~ + + + +Deletes plugin override for given project and domain combination or additionally with workflow name. + +Deletes plugin override for project and domain +Here the command deletes plugin override for project flytectldemo and development domain. +:: + + flytectl delete plugin-override -p flytectldemo -d development + + +Deletes plugin override using config file which was used for creating it. +Here the command deletes plugin overrides from the config file po.yaml +Overrides are optional in the file as they are unread during the delete command but can be kept as the same file can be used for get, update or delete +eg: content of po.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +Deletes plugin override for a workflow +Here the command deletes the plugin override for a workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl delete plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage + + +:: + + flytectl delete plugin-override [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for delete attribute for the resource type. + -h, --help help for plugin-override + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_delete` - Used for terminating/deleting various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/docs/source/gen/flytectl_delete_task-resource-attribute.rst b/docs/source/gen/flytectl_delete_task-resource-attribute.rst index c393da6c54..91b3f785b0 100644 --- a/docs/source/gen/flytectl_delete_task-resource-attribute.rst +++ b/docs/source/gen/flytectl_delete_task-resource-attribute.rst @@ -41,7 +41,7 @@ eg: content of tra.yaml which will use the project domain and workflow name for memory: "450Mi" Deletes task resource attribute for a workflow -Here the command deletes task resource attributes for a workflow +Here the command deletes task resource attributes for a workflow core.control_flow.run_merge_sort.merge_sort :: diff --git a/docs/source/gen/flytectl_get.rst b/docs/source/gen/flytectl_get.rst index 2852cda14a..4a0608a262 100644 --- a/docs/source/gen/flytectl_get.rst +++ b/docs/source/gen/flytectl_get.rst @@ -13,7 +13,7 @@ Synopsis Example get projects. :: - bin/flytectl get project + flytectl get project Options @@ -73,8 +73,10 @@ SEE ALSO * :doc:`flytectl` - flyetcl CLI tool * :doc:`flytectl_get_cluster-resource-attribute` - Gets matchable resources of cluster resource attributes * :doc:`flytectl_get_execution` - Gets execution resources +* :doc:`flytectl_get_execution-cluster-label` - Gets matchable resources of execution cluster label * :doc:`flytectl_get_execution-queue-attribute` - Gets matchable resources of execution queue attributes * :doc:`flytectl_get_launchplan` - Gets launch plan resources +* :doc:`flytectl_get_plugin-override` - Gets matchable resources of plugin override * :doc:`flytectl_get_project` - Gets project resources * :doc:`flytectl_get_task` - Gets task resources * :doc:`flytectl_get_task-resource-attribute` - Gets matchable resources of task attributes diff --git a/docs/source/gen/flytectl_get_cluster-resource-attribute.rst b/docs/source/gen/flytectl_get_cluster-resource-attribute.rst index 5dfa50b3e1..0606779729 100644 --- a/docs/source/gen/flytectl_get_cluster-resource-attribute.rst +++ b/docs/source/gen/flytectl_get_cluster-resource-attribute.rst @@ -25,7 +25,7 @@ eg : output from the command {"project":"flytectldemo","domain":"development","attributes":{"buzz":"lightyear","foo":"bar"}} Retrieves cluster resource attribute for project and domain and workflow -Here the command get cluster resource attributes for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort +Here the command get cluster resource attributes for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort :: flytectl get cluster-resource-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort diff --git a/docs/source/gen/flytectl_get_execution-cluster-label.rst b/docs/source/gen/flytectl_get_execution-cluster-label.rst new file mode 100644 index 0000000000..e8752979d7 --- /dev/null +++ b/docs/source/gen/flytectl_get_execution-cluster-label.rst @@ -0,0 +1,117 @@ +.. _flytectl_get_execution-cluster-label: + +flytectl get execution-cluster-label +------------------------------------ + +Gets matchable resources of execution cluster label + +Synopsis +~~~~~~~~ + + + +Retrieves execution cluster label for given project and domain combination or additionally with workflow name. + +Retrieves execution cluster label for project and domain +Here the command get execution cluster label for project flytectldemo and development domain. +:: + + flytectl get execution-cluster-label -p flytectldemo -d development + +eg : output from the command + +.. code-block:: json + + {"project":"flytectldemo","domain":"development","value":"foo"} + +Retrieves execution cluster label for project and domain and workflow +Here the command get execution cluster label for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort +:: + + flytectl get execution-cluster-label -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +eg : output from the command + +.. code-block:: json + + {"project":"flytectldemo","domain":"development","workflow":"core.control_flow.run_merge_sort.merge_sort","value":"foo"} + +Writing the execution cluster label to a file. If there are no execution cluster label, command would return an error. +Here the command gets execution cluster label and writes the config file to ecl.yaml +eg: content of ecl.yaml + +:: + + flytectl get execution-cluster-label --attrFile ecl.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + value: foo + +Usage + + +:: + + flytectl get execution-cluster-label [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for generating attribute for the resource type. + -h, --help help for execution-cluster-label + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_get` - Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/docs/source/gen/flytectl_get_execution-queue-attribute.rst b/docs/source/gen/flytectl_get_execution-queue-attribute.rst index db008340a4..b9eff077be 100644 --- a/docs/source/gen/flytectl_get_execution-queue-attribute.rst +++ b/docs/source/gen/flytectl_get_execution-queue-attribute.rst @@ -25,7 +25,7 @@ eg : output from the command {"project":"flytectldemo","domain":"development","tags":["foo", "bar"]} Retrieves execution queue attribute for project and domain and workflow -Here the command get execution queue attributes for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort +Here the command get execution queue attributes for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort :: flytectl get execution-queue-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort diff --git a/docs/source/gen/flytectl_get_plugin-override.rst b/docs/source/gen/flytectl_get_plugin-override.rst new file mode 100644 index 0000000000..1735945a10 --- /dev/null +++ b/docs/source/gen/flytectl_get_plugin-override.rst @@ -0,0 +1,141 @@ +.. _flytectl_get_plugin-override: + +flytectl get plugin-override +---------------------------- + +Gets matchable resources of plugin override + +Synopsis +~~~~~~~~ + + + +Retrieves plugin overrides for given project and domain combination or additionally with workflow name. + +Retrieves plugin overrides for project and domain +Here the command get plugin override for project flytectldemo and development domain. + +:: + + flytectl get plugin-override -p flytectldemo -d development + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Retrieves plugin override for project and domain and workflow +Here the command get plugin override for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort + +:: + + flytectl get plugin-override -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +eg : output from the command + +.. code-block:: json + + { + "project": "flytectldemo", + "domain": "development", + "workflow": "core.control_flow.run_merge_sort.merge_sort" + "overrides": [{ + "task_type": "python_task", + "plugin_id": ["pluginoverride1", "pluginoverride2"], + "missing_plugin_behavior": 0 + }] + } + +Writing the plugin override to a file. If there are no plugin overrides, command would return an error. +Here the command gets plugin overrides and writes the config file to po.yaml +eg: content of po.yaml + +:: + + flytectl get plugin-override --attrFile po.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +Usage + + +:: + + flytectl get plugin-override [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for generating attribute for the resource type. + -h, --help help for plugin-override + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_get` - Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/docs/source/gen/flytectl_get_task-resource-attribute.rst b/docs/source/gen/flytectl_get_task-resource-attribute.rst index 7b4fa5ba74..a49cb1f96d 100644 --- a/docs/source/gen/flytectl_get_task-resource-attribute.rst +++ b/docs/source/gen/flytectl_get_task-resource-attribute.rst @@ -25,7 +25,7 @@ eg : output from the command {"project":"flytectldemo","domain":"development","workflow":"","defaults":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"2","memory":"450Mi"}} Retrieves task resource attribute for project and domain and workflow -Here the command get task resource attributes for project flytectldemo ,development domain and workflow core.control_flow.run_merge_sort.merge_sort +Here the command get task resource attributes for project flytectldemo, development domain and workflow core.control_flow.run_merge_sort.merge_sort :: flytectl get task-resource-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort diff --git a/docs/source/gen/flytectl_update.rst b/docs/source/gen/flytectl_update.rst index 3fd3d51b06..1372e09809 100644 --- a/docs/source/gen/flytectl_update.rst +++ b/docs/source/gen/flytectl_update.rst @@ -74,8 +74,10 @@ SEE ALSO * :doc:`flytectl` - flyetcl CLI tool * :doc:`flytectl_update_cluster-resource-attribute` - Updates matchable resources of cluster attributes +* :doc:`flytectl_update_execution-cluster-label` - Updates matchable resources of execution cluster label * :doc:`flytectl_update_execution-queue-attribute` - Updates matchable resources of execution queue attributes * :doc:`flytectl_update_launchplan` - Updates launch plan metadata +* :doc:`flytectl_update_plugin-override` - Updates matchable resources of plugin overrides * :doc:`flytectl_update_project` - Updates project resources * :doc:`flytectl_update_task` - Updates task metadata * :doc:`flytectl_update_task-resource-attribute` - Updates matchable resources of task attributes diff --git a/docs/source/gen/flytectl_update_cluster-resource-attribute.rst b/docs/source/gen/flytectl_update_cluster-resource-attribute.rst index f17c81d62b..1646cdfc90 100644 --- a/docs/source/gen/flytectl_update_cluster-resource-attribute.rst +++ b/docs/source/gen/flytectl_update_cluster-resource-attribute.rst @@ -33,7 +33,7 @@ resource attribute defined at project domain level. Also this will completely overwrite any existing custom project and domain and workflow combination attributes. Would be preferable to do get and generate an attribute file if there is an existing attribute already set and then update it to have new values Refer to get cluster-resource-attribute section on how to generate this file -Update the cluster resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +Update the cluster resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain .. code-block:: yaml diff --git a/docs/source/gen/flytectl_update_execution-cluster-label.rst b/docs/source/gen/flytectl_update_execution-cluster-label.rst new file mode 100644 index 0000000000..325dda91a6 --- /dev/null +++ b/docs/source/gen/flytectl_update_execution-cluster-label.rst @@ -0,0 +1,108 @@ +.. _flytectl_update_execution-cluster-label: + +flytectl update execution-cluster-label +--------------------------------------- + +Updates matchable resources of execution cluster label + +Synopsis +~~~~~~~~ + + + +Updates execution cluster label for given project and domain combination or additionally with workflow name. + +Updating to the execution cluster label is only available from a generated file. See the get section for generating this file. +Here the command updates takes the input for execution cluster label from the config file ecl.yaml +eg: content of ecl.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + value: foo + +:: + + flytectl update execution-cluster-label --attrFile ecl.yaml + +Updating execution cluster label for project and domain and workflow combination. This will take precedence over any other +execution cluster label defined at project domain level. +Update the execution cluster label for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain + +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + value: foo + +:: + + flytectl update execution-cluster-label --attrFile ecl.yaml + +Usage + + + +:: + + flytectl update execution-cluster-label [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for updating attribute for the resource type. + -h, --help help for execution-cluster-label + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_update` - Used for updating flyte resources eg: project. + diff --git a/docs/source/gen/flytectl_update_execution-queue-attribute.rst b/docs/source/gen/flytectl_update_execution-queue-attribute.rst index e7aa206301..85f28eec5b 100644 --- a/docs/source/gen/flytectl_update_execution-queue-attribute.rst +++ b/docs/source/gen/flytectl_update_execution-queue-attribute.rst @@ -35,7 +35,7 @@ eg: content of era.yaml Updating execution queue attribute for project and domain and workflow combination. This will take precedence over any other execution queue attribute defined at project domain level. -Update the execution queue attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +Update the execution queue attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain .. code-block:: yaml diff --git a/docs/source/gen/flytectl_update_plugin-override.rst b/docs/source/gen/flytectl_update_plugin-override.rst new file mode 100644 index 0000000000..3cff796b1d --- /dev/null +++ b/docs/source/gen/flytectl_update_plugin-override.rst @@ -0,0 +1,121 @@ +.. _flytectl_update_plugin-override: + +flytectl update plugin-override +------------------------------- + +Updates matchable resources of plugin overrides + +Synopsis +~~~~~~~~ + + + +Updates plugin overrides for given project and domain combination or additionally with workflow name. + +Updating to the plugin override is only available from a generated file. See the get section for generating this file. +Also this will completely overwrite any existing plugins overrides on custom project and domain and workflow combination. +Would be preferable to do get and generate an plugin override file if there is an existing override already set and then update it to have new values +Refer to get plugin-override section on how to generate this file +Here the command updates takes the input for plugin overrides from the config file po.yaml +eg: content of po.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Updating plugin override for project and domain and workflow combination. This will take precedence over any other +plugin overrides defined at project domain level. +Update the plugin overrides for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain + +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + overrides: + - task_type: python_task # Task type for which to apply plugin implementation overrides + plugin_id: # Plugin id(s) to be used in place of the default for the task type. + - plugin_override1 + - plugin_override2 + missing_plugin_behavior: 1 # Behavior when no specified plugin_id has an associated handler. 0 : FAIL , 1: DEFAULT + +:: + + flytectl update plugin-override --attrFile po.yaml + +Usage + + + +:: + + flytectl update plugin-override [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for updating attribute for the resource type. + -h, --help help for plugin-override + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_update` - Used for updating flyte resources eg: project. + diff --git a/docs/source/gen/flytectl_update_task-resource-attribute.rst b/docs/source/gen/flytectl_update_task-resource-attribute.rst index 8e13080509..37c3439eff 100644 --- a/docs/source/gen/flytectl_update_task-resource-attribute.rst +++ b/docs/source/gen/flytectl_update_task-resource-attribute.rst @@ -36,7 +36,7 @@ eg: content of tra.yaml Updating task resource attribute for project and domain and workflow combination. This will take precedence over any other resource attribute defined at project domain level. -Update the resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +Update the resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo, development domain .. code-block:: yaml diff --git a/docs/source/nouns.rst b/docs/source/nouns.rst index 3c546cbfab..89d3c74a3d 100644 --- a/docs/source/nouns.rst +++ b/docs/source/nouns.rst @@ -14,7 +14,9 @@ Nouns gen/flytectl_get_task gen/flytectl_get_task-resource-attribute gen/flytectl_get_cluster-resource-attribute + gen/flytectl_get_execution-cluster-label gen/flytectl_get_execution-queue-attribute + gen/flytectl_get_plugin-override gen/flytectl_get_launchplan gen/flytectl_update_launchplan gen/flytectl_update_workflow @@ -22,12 +24,16 @@ Nouns gen/flytectl_update_task gen/flytectl_update_task-resource-attribute gen/flytectl_update_cluster-resource-attribute + gen/flytectl_update_execution-cluster-label gen/flytectl_update_execution-queue-attribute + gen/flytectl_update_plugin-override gen/flytectl_register_files gen/flytectl_delete_execution gen/flytectl_delete_task-resource-attribute gen/flytectl_delete_cluster-resource-attribute + gen/flytectl_delete_execution-cluster-label gen/flytectl_delete_execution-queue-attribute + gen/flytectl_delete_plugin-override gen/flytectl_version gen/flytectl_config_validate gen/flytectl_config_discover