From 661de8de42b555c582dd770a8c75f0a64e07bb1b Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss <77798312+pmahindrakar-oss@users.noreply.github.com> Date: Tue, 11 May 2021 22:25:22 +0530 Subject: [PATCH] Matchable resource support in flytectl for task resource attributes (#63) * Adding commands for task resource attributes support Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> --- flytectl/.gitignore | 1 + .../task_resource_attribute_delete_config.go | 10 ++ .../task_resource_attribute_fetch_config.go | 9 ++ .../task_resource_attribute_file_config.go | 86 ++++++++++ .../task_resource_attribute_update_config.go | 10 ++ .../taskresourceattrdeleteconfig_flags.go | 46 ++++++ ...taskresourceattrdeleteconfig_flags_test.go | 124 ++++++++++++++ .../taskresourceattrfetchconfig_flags.go | 46 ++++++ .../taskresourceattrfetchconfig_flags_test.go | 124 ++++++++++++++ .../taskresourceattrupdateconfig_flags.go | 46 ++++++ ...taskresourceattrupdateconfig_flags_test.go | 124 ++++++++++++++ flytectl/cmd/core/cmd.go | 11 +- flytectl/cmd/core/cmd_ctx.go | 27 +++- flytectl/cmd/create/execution_test.go | 4 +- flytectl/cmd/delete/delete.go | 4 + flytectl/cmd/delete/delete_test.go | 20 ++- flytectl/cmd/delete/execution_test.go | 66 ++++---- .../matchable_task_resource_attribute.go | 97 +++++++++++ .../matchable_task_resource_attribute_test.go | 133 +++++++++++++++ .../testdata/invalid_task_attribute.yaml | 5 + .../valid_project_domain_task_attribute.yaml | 9 ++ .../valid_workflow_task_attribute.yaml | 9 ++ flytectl/cmd/get/get.go | 4 + flytectl/cmd/get/get_test.go | 12 +- flytectl/cmd/get/launch_plan_test.go | 10 +- .../get/matchable_task_resource_attribute.go | 98 ++++++++++++ .../matchable_task_resource_attribute_test.go | 144 +++++++++++++++++ flytectl/cmd/get/task_test.go | 10 +- flytectl/cmd/get/taskconfig_flags.go | 2 +- flytectl/cmd/testutils/test_utils.go | 19 ++- .../matchable_task_resource_attribute.go | 95 +++++++++++ .../matchable_task_resource_attribute_test.go | 94 +++++++++++ flytectl/cmd/update/project_test.go | 81 ++++------ .../testdata/invalid_task_attribute.yaml | 5 + .../valid_project_domain_task_attribute.yaml | 9 ++ .../valid_workflow_task_attribute.yaml | 9 ++ flytectl/cmd/update/update.go | 3 + flytectl/cmd/update/update_test.go | 24 ++- flytectl/docs/coverage.out | 1 + flytectl/docs/source/gen/flytectl.rst | 12 +- flytectl/docs/source/gen/flytectl_config.rst | 12 +- .../source/gen/flytectl_config_discover.rst | 12 +- .../source/gen/flytectl_config_validate.rst | 12 +- flytectl/docs/source/gen/flytectl_create.rst | 12 +- .../source/gen/flytectl_create_execution.rst | 12 +- .../source/gen/flytectl_create_project.rst | 12 +- flytectl/docs/source/gen/flytectl_delete.rst | 13 +- .../source/gen/flytectl_delete_execution.rst | 12 +- ...lytectl_delete_task-resource-attribute.rst | 112 +++++++++++++ flytectl/docs/source/gen/flytectl_get.rst | 13 +- .../source/gen/flytectl_get_execution.rst | 12 +- .../source/gen/flytectl_get_launchplan.rst | 12 +- .../docs/source/gen/flytectl_get_project.rst | 12 +- .../flytectl_get_task-resource-attribute.rst | 110 +++++++++++++ .../docs/source/gen/flytectl_get_task.rst | 12 +- .../docs/source/gen/flytectl_get_workflow.rst | 12 +- .../docs/source/gen/flytectl_register.rst | 12 +- .../source/gen/flytectl_register_files.rst | 12 +- flytectl/docs/source/gen/flytectl_update.rst | 13 +- .../source/gen/flytectl_update_launchplan.rst | 12 +- .../source/gen/flytectl_update_project.rst | 12 +- ...lytectl_update_task-resource-attribute.rst | 117 ++++++++++++++ .../docs/source/gen/flytectl_update_task.rst | 12 +- .../source/gen/flytectl_update_workflow.rst | 12 +- flytectl/docs/source/gen/flytectl_version.rst | 12 +- .../pkg/ext/attribute_match_deleter_test.go | 49 ++++++ flytectl/pkg/ext/attribute_match_fetcher.go | 29 ++++ .../pkg/ext/attribute_match_fetcher_test.go | 49 ++++++ flytectl/pkg/ext/attribute_match_updater.go | 31 ++++ .../pkg/ext/attribute_match_updater_test.go | 55 +++++++ flytectl/pkg/ext/attribute_matcher_deleter.go | 26 +++ flytectl/pkg/ext/deleter.go | 33 ++++ flytectl/pkg/ext/doc.go | 2 + ...on_fetcher_ext.go => execution_fetcher.go} | 0 flytectl/pkg/ext/execution_fetcher_test.go | 68 ++++++++ .../ext/{fetcher_ext_client.go => fetcher.go} | 6 + flytectl/pkg/ext/launch_plan_fetcher_test.go | 151 ++++++++++++++++++ .../ext/mocks/admin_deleter_ext_interface.go | 116 ++++++++++++++ .../ext/mocks/admin_fetcher_ext_interface.go | 82 ++++++++++ .../ext/mocks/admin_updater_ext_interface.go | 116 ++++++++++++++ flytectl/pkg/ext/task_fetcher_test.go | 125 +++++++++++++++ flytectl/pkg/ext/updater.go | 33 ++++ 82 files changed, 2961 insertions(+), 259 deletions(-) create mode 100644 flytectl/cmd/config/subcommand/task_resource_attribute_delete_config.go create mode 100644 flytectl/cmd/config/subcommand/task_resource_attribute_fetch_config.go create mode 100644 flytectl/cmd/config/subcommand/task_resource_attribute_file_config.go create mode 100644 flytectl/cmd/config/subcommand/task_resource_attribute_update_config.go create mode 100755 flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags.go create mode 100755 flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags_test.go create mode 100755 flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags.go create mode 100755 flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags_test.go create mode 100755 flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags.go create mode 100755 flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags_test.go create mode 100644 flytectl/cmd/delete/matchable_task_resource_attribute.go create mode 100644 flytectl/cmd/delete/matchable_task_resource_attribute_test.go create mode 100644 flytectl/cmd/delete/testdata/invalid_task_attribute.yaml create mode 100644 flytectl/cmd/delete/testdata/valid_project_domain_task_attribute.yaml create mode 100644 flytectl/cmd/delete/testdata/valid_workflow_task_attribute.yaml create mode 100644 flytectl/cmd/get/matchable_task_resource_attribute.go create mode 100644 flytectl/cmd/get/matchable_task_resource_attribute_test.go create mode 100644 flytectl/cmd/update/matchable_task_resource_attribute.go create mode 100644 flytectl/cmd/update/matchable_task_resource_attribute_test.go create mode 100644 flytectl/cmd/update/testdata/invalid_task_attribute.yaml create mode 100644 flytectl/cmd/update/testdata/valid_project_domain_task_attribute.yaml create mode 100644 flytectl/cmd/update/testdata/valid_workflow_task_attribute.yaml create mode 100644 flytectl/docs/coverage.out create mode 100644 flytectl/docs/source/gen/flytectl_delete_task-resource-attribute.rst create mode 100644 flytectl/docs/source/gen/flytectl_get_task-resource-attribute.rst create mode 100644 flytectl/docs/source/gen/flytectl_update_task-resource-attribute.rst create mode 100644 flytectl/pkg/ext/attribute_match_deleter_test.go create mode 100644 flytectl/pkg/ext/attribute_match_fetcher.go create mode 100644 flytectl/pkg/ext/attribute_match_fetcher_test.go create mode 100644 flytectl/pkg/ext/attribute_match_updater.go create mode 100644 flytectl/pkg/ext/attribute_match_updater_test.go create mode 100644 flytectl/pkg/ext/attribute_matcher_deleter.go create mode 100644 flytectl/pkg/ext/deleter.go create mode 100644 flytectl/pkg/ext/doc.go rename flytectl/pkg/ext/{execution_fetcher_ext.go => execution_fetcher.go} (100%) create mode 100644 flytectl/pkg/ext/execution_fetcher_test.go rename flytectl/pkg/ext/{fetcher_ext_client.go => fetcher.go} (79%) create mode 100644 flytectl/pkg/ext/launch_plan_fetcher_test.go create mode 100644 flytectl/pkg/ext/mocks/admin_deleter_ext_interface.go create mode 100644 flytectl/pkg/ext/mocks/admin_updater_ext_interface.go create mode 100644 flytectl/pkg/ext/task_fetcher_test.go create mode 100644 flytectl/pkg/ext/updater.go diff --git a/flytectl/.gitignore b/flytectl/.gitignore index 45283c9a91..5f7a3898d1 100644 --- a/flytectl/.gitignore +++ b/flytectl/.gitignore @@ -6,3 +6,4 @@ bin _test ./config.yaml docs/build/* +cmd/get/temp-output-file diff --git a/flytectl/cmd/config/subcommand/task_resource_attribute_delete_config.go b/flytectl/cmd/config/subcommand/task_resource_attribute_delete_config.go new file mode 100644 index 0000000000..34a6a4e466 --- /dev/null +++ b/flytectl/cmd/config/subcommand/task_resource_attribute_delete_config.go @@ -0,0 +1,10 @@ +package subcommand + +//go:generate pflags TaskResourceAttrDeleteConfig --default-var DefaultTaskResourceDelConfig + +// TaskResourceAttrDeleteConfig Matchable resource attributes configuration passed from command line +type TaskResourceAttrDeleteConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for delete attribute for the resource type."` +} + +var DefaultTaskResourceDelConfig = &TaskResourceAttrDeleteConfig{} diff --git a/flytectl/cmd/config/subcommand/task_resource_attribute_fetch_config.go b/flytectl/cmd/config/subcommand/task_resource_attribute_fetch_config.go new file mode 100644 index 0000000000..34b1d435f4 --- /dev/null +++ b/flytectl/cmd/config/subcommand/task_resource_attribute_fetch_config.go @@ -0,0 +1,9 @@ +package subcommand + +//go:generate pflags TaskResourceAttrFetchConfig --default-var DefaultTaskResourceFetchConfig + +type TaskResourceAttrFetchConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for generating attribute for the resource type."` +} + +var DefaultTaskResourceFetchConfig = &TaskResourceAttrFetchConfig{} diff --git a/flytectl/cmd/config/subcommand/task_resource_attribute_file_config.go b/flytectl/cmd/config/subcommand/task_resource_attribute_file_config.go new file mode 100644 index 0000000000..d0c0a0656c --- /dev/null +++ b/flytectl/cmd/config/subcommand/task_resource_attribute_file_config.go @@ -0,0 +1,86 @@ +package subcommand + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + + cmdUtil "github.com/flyteorg/flytectl/pkg/commandutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flytestdlib/logger" + + "sigs.k8s.io/yaml" +) + +// TaskResourceAttrFileConfig shadow config for TaskResourceAttribute. +// The shadow config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. +// As the same structure is being used for both ProjectDomainAttribute/Workflowattribute +type TaskResourceAttrFileConfig struct { + Project string `json:"project"` + Domain string `json:"domain"` + Workflow string `json:"workflow,omitempty"` + *admin.TaskResourceAttributes +} + +// WriteConfigToFile used for marshaling the config to a file which can then be used for update/delete +func (t TaskResourceAttrFileConfig) WriteConfigToFile(fileName string) error { + d, err := yaml.Marshal(t) + if err != nil { + return fmt.Errorf("error: %v", err) + } + if _, err = os.Stat(fileName); err == nil { + if !cmdUtil.AskForConfirmation(fmt.Sprintf("warning file %v will be overwritten", fileName)) { + return fmt.Errorf("backup the file before continuing") + } + } + return ioutil.WriteFile(fileName, d, 0600) +} + +// Dumps the json representation of the TaskResourceAttrFileConfig +func (t TaskResourceAttrFileConfig) String() string { + tj, err := json.Marshal(t) + if err != nil { + fmt.Println(err) + return "marshaling error" + } + return fmt.Sprintf("%s\n", tj) +} + +// ReadConfigFromFile used for unmarshaling the config from a file which is used for update/delete +func (t *TaskResourceAttrFileConfig) ReadConfigFromFile(fileName string) error { + data, err := ioutil.ReadFile(fileName) + if err != nil { + return fmt.Errorf("unable to read from %v yaml file", fileName) + } + if err = yaml.UnmarshalStrict(data, t); err != nil { + return err + } + return nil +} + +// MatchableAttributeDecorator decorator over TaskResourceAttributes. Similar decorator would exist for other MatchingAttributes +func (t *TaskResourceAttrFileConfig) MatchableAttributeDecorator() *admin.MatchingAttributes { + return &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_TaskResourceAttributes{ + TaskResourceAttributes: t.TaskResourceAttributes, + }, + } +} + +func (t TaskResourceAttrFileConfig) DumpTaskResourceAttr(ctx context.Context, fileName string) { + // Dump empty task resource attr for editing + // Write config to file if filename provided in the command + if len(fileName) > 0 { + // Read the config from the file and update the TaskResourceAttrFileConfig with the TaskResourceAttrConfig + if err := t.WriteConfigToFile(fileName); err != nil { + fmt.Printf("error dumping in file due to %v", err) + logger.Warnf(ctx, "error dumping in file due to %v", err) + return + } + fmt.Printf("wrote the config to file %v", fileName) + } else { + fmt.Printf("%v", t) + } +} diff --git a/flytectl/cmd/config/subcommand/task_resource_attribute_update_config.go b/flytectl/cmd/config/subcommand/task_resource_attribute_update_config.go new file mode 100644 index 0000000000..d85944777d --- /dev/null +++ b/flytectl/cmd/config/subcommand/task_resource_attribute_update_config.go @@ -0,0 +1,10 @@ +package subcommand + +//go:generate pflags TaskResourceAttrUpdateConfig --default-var DefaultTaskResourceUpdateConfig + +// TaskResourceAttrUpdateConfig Matchable resource attributes configuration passed from command line +type TaskResourceAttrUpdateConfig struct { + AttrFile string `json:"attrFile" pflag:",attribute file name to be used for updating attribute for the resource type."` +} + +var DefaultTaskResourceUpdateConfig = &TaskResourceAttrUpdateConfig{} diff --git a/flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags.go b/flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags.go new file mode 100755 index 0000000000..fadea1f465 --- /dev/null +++ b/flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package subcommand + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (TaskResourceAttrDeleteConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (TaskResourceAttrDeleteConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in TaskResourceAttrDeleteConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg TaskResourceAttrDeleteConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("TaskResourceAttrDeleteConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultTaskResourceDelConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultTaskResourceDelConfig.AttrFile, "attribute file name to be used for delete attribute for the resource type.") + return cmdFlags +} diff --git a/flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags_test.go b/flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags_test.go new file mode 100755 index 0000000000..222fae2c50 --- /dev/null +++ b/flytectl/cmd/config/subcommand/taskresourceattrdeleteconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package subcommand + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsTaskResourceAttrDeleteConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementTaskResourceAttrDeleteConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsTaskResourceAttrDeleteConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookTaskResourceAttrDeleteConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementTaskResourceAttrDeleteConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_TaskResourceAttrDeleteConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookTaskResourceAttrDeleteConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_TaskResourceAttrDeleteConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_TaskResourceAttrDeleteConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_TaskResourceAttrDeleteConfig(val, result)) +} + +func testDecodeSlice_TaskResourceAttrDeleteConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_TaskResourceAttrDeleteConfig(vStringSlice, result)) +} + +func TestTaskResourceAttrDeleteConfig_GetPFlagSet(t *testing.T) { + val := TaskResourceAttrDeleteConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestTaskResourceAttrDeleteConfig_SetFlags(t *testing.T) { + actual := TaskResourceAttrDeleteConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultTaskResourceDelConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_TaskResourceAttrDeleteConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags.go b/flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags.go new file mode 100755 index 0000000000..ebf6ec6220 --- /dev/null +++ b/flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package subcommand + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (TaskResourceAttrFetchConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (TaskResourceAttrFetchConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in TaskResourceAttrFetchConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg TaskResourceAttrFetchConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("TaskResourceAttrFetchConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultTaskResourceFetchConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultTaskResourceFetchConfig.AttrFile, "attribute file name to be used for generating attribute for the resource type.") + return cmdFlags +} diff --git a/flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags_test.go b/flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags_test.go new file mode 100755 index 0000000000..c2c77e82c2 --- /dev/null +++ b/flytectl/cmd/config/subcommand/taskresourceattrfetchconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package subcommand + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsTaskResourceAttrFetchConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementTaskResourceAttrFetchConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsTaskResourceAttrFetchConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookTaskResourceAttrFetchConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementTaskResourceAttrFetchConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_TaskResourceAttrFetchConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookTaskResourceAttrFetchConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_TaskResourceAttrFetchConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_TaskResourceAttrFetchConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_TaskResourceAttrFetchConfig(val, result)) +} + +func testDecodeSlice_TaskResourceAttrFetchConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_TaskResourceAttrFetchConfig(vStringSlice, result)) +} + +func TestTaskResourceAttrFetchConfig_GetPFlagSet(t *testing.T) { + val := TaskResourceAttrFetchConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestTaskResourceAttrFetchConfig_SetFlags(t *testing.T) { + actual := TaskResourceAttrFetchConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultTaskResourceFetchConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_TaskResourceAttrFetchConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags.go b/flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags.go new file mode 100755 index 0000000000..d94413106f --- /dev/null +++ b/flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags.go @@ -0,0 +1,46 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package subcommand + +import ( + "encoding/json" + "reflect" + + "fmt" + + "github.com/spf13/pflag" +) + +// If v is a pointer, it will get its element value or the zero value of the element type. +// If v is not a pointer, it will return it as is. +func (TaskResourceAttrUpdateConfig) elemValueOrNil(v interface{}) interface{} { + if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { + if reflect.ValueOf(v).IsNil() { + return reflect.Zero(t.Elem()).Interface() + } else { + return reflect.ValueOf(v).Interface() + } + } else if v == nil { + return reflect.Zero(t).Interface() + } + + return v +} + +func (TaskResourceAttrUpdateConfig) mustMarshalJSON(v json.Marshaler) string { + raw, err := v.MarshalJSON() + if err != nil { + panic(err) + } + + return string(raw) +} + +// GetPFlagSet will return strongly types pflags for all fields in TaskResourceAttrUpdateConfig and its nested types. The format of the +// flags is json-name.json-sub-name... etc. +func (cfg TaskResourceAttrUpdateConfig) GetPFlagSet(prefix string) *pflag.FlagSet { + cmdFlags := pflag.NewFlagSet("TaskResourceAttrUpdateConfig", pflag.ExitOnError) + cmdFlags.StringVar(&(DefaultTaskResourceUpdateConfig.AttrFile), fmt.Sprintf("%v%v", prefix, "attrFile"), DefaultTaskResourceUpdateConfig.AttrFile, "attribute file name to be used for updating attribute for the resource type.") + return cmdFlags +} diff --git a/flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags_test.go b/flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags_test.go new file mode 100755 index 0000000000..73f272b45e --- /dev/null +++ b/flytectl/cmd/config/subcommand/taskresourceattrupdateconfig_flags_test.go @@ -0,0 +1,124 @@ +// Code generated by go generate; DO NOT EDIT. +// This file was generated by robots. + +package subcommand + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +var dereferencableKindsTaskResourceAttrUpdateConfig = map[reflect.Kind]struct{}{ + reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, +} + +// Checks if t is a kind that can be dereferenced to get its underlying type. +func canGetElementTaskResourceAttrUpdateConfig(t reflect.Kind) bool { + _, exists := dereferencableKindsTaskResourceAttrUpdateConfig[t] + return exists +} + +// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the +// object. Otherwise, it'll just pass on the original data. +func jsonUnmarshalerHookTaskResourceAttrUpdateConfig(_, to reflect.Type, data interface{}) (interface{}, error) { + unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() + if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || + (canGetElementTaskResourceAttrUpdateConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) { + + raw, err := json.Marshal(data) + if err != nil { + fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + res := reflect.New(to).Interface() + err = json.Unmarshal(raw, &res) + if err != nil { + fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) + return data, nil + } + + return res, nil + } + + return data, nil +} + +func decode_TaskResourceAttrUpdateConfig(input, result interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "json", + WeaklyTypedInput: true, + Result: result, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + jsonUnmarshalerHookTaskResourceAttrUpdateConfig, + ), + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +func join_TaskResourceAttrUpdateConfig(arr interface{}, sep string) string { + listValue := reflect.ValueOf(arr) + strs := make([]string, 0, listValue.Len()) + for i := 0; i < listValue.Len(); i++ { + strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) + } + + return strings.Join(strs, sep) +} + +func testDecodeJson_TaskResourceAttrUpdateConfig(t *testing.T, val, result interface{}) { + assert.NoError(t, decode_TaskResourceAttrUpdateConfig(val, result)) +} + +func testDecodeSlice_TaskResourceAttrUpdateConfig(t *testing.T, vStringSlice, result interface{}) { + assert.NoError(t, decode_TaskResourceAttrUpdateConfig(vStringSlice, result)) +} + +func TestTaskResourceAttrUpdateConfig_GetPFlagSet(t *testing.T) { + val := TaskResourceAttrUpdateConfig{} + cmdFlags := val.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) +} + +func TestTaskResourceAttrUpdateConfig_SetFlags(t *testing.T) { + actual := TaskResourceAttrUpdateConfig{} + cmdFlags := actual.GetPFlagSet("") + assert.True(t, cmdFlags.HasFlags()) + + t.Run("Test_attrFile", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + assert.Equal(t, string(DefaultTaskResourceUpdateConfig.AttrFile), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("attrFile", testValue) + if vString, err := cmdFlags.GetString("attrFile"); err == nil { + testDecodeJson_TaskResourceAttrUpdateConfig(t, fmt.Sprintf("%v", vString), &actual.AttrFile) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) +} diff --git a/flytectl/cmd/core/cmd.go b/flytectl/cmd/core/cmd.go index 2a13e36c05..54483c5c8f 100644 --- a/flytectl/cmd/core/cmd.go +++ b/flytectl/cmd/core/cmd.go @@ -28,11 +28,12 @@ type CommandEntry struct { func AddCommands(rootCmd *cobra.Command, cmdFuncs map[string]CommandEntry) { for resource, cmdEntry := range cmdFuncs { cmd := &cobra.Command{ - Use: resource, - Short: cmdEntry.Short, - Long: cmdEntry.Long, - Aliases: cmdEntry.Aliases, - RunE: generateCommandFunc(cmdEntry), + Use: resource, + Short: cmdEntry.Short, + Long: cmdEntry.Long, + Aliases: cmdEntry.Aliases, + RunE: generateCommandFunc(cmdEntry), + SilenceUsage: true, } if cmdEntry.PFlagProvider != nil { cmd.Flags().AddFlagSet(cmdEntry.PFlagProvider.GetPFlagSet("")) diff --git a/flytectl/cmd/core/cmd_ctx.go b/flytectl/cmd/core/cmd_ctx.go index 58f3ff457b..ab1c6c6afd 100644 --- a/flytectl/cmd/core/cmd_ctx.go +++ b/flytectl/cmd/core/cmd_ctx.go @@ -10,13 +10,30 @@ import ( type CommandContext struct { adminClient service.AdminServiceClient adminClientFetcherExt ext.AdminFetcherExtInterface + adminClientUpdateExt ext.AdminUpdaterExtInterface + adminClientDeleteExt ext.AdminDeleterExtInterface in io.Reader out io.Writer } func NewCommandContext(adminClient service.AdminServiceClient, out io.Writer) CommandContext { return CommandContext{adminClient: adminClient, out: out, - adminClientFetcherExt: &ext.AdminFetcherExtClient{AdminClient: adminClient}} + adminClientFetcherExt: &ext.AdminFetcherExtClient{AdminClient: adminClient}, + adminClientUpdateExt: &ext.AdminUpdaterExtClient{AdminClient: adminClient}, + adminClientDeleteExt: &ext.AdminDeleterExtClient{AdminClient: adminClient}} +} + +// NewCommandContextWithExt construct command context with injected extensions. Helps in injecting mocked ones for testing. +func NewCommandContextWithExt( + adminClient service.AdminServiceClient, + fetcher ext.AdminFetcherExtInterface, + updater ext.AdminUpdaterExtInterface, + deleter ext.AdminDeleterExtInterface, + out io.Writer) CommandContext { + return CommandContext{adminClient: adminClient, out: out, + adminClientFetcherExt: fetcher, + adminClientUpdateExt: updater, + adminClientDeleteExt: deleter} } func (c CommandContext) AdminClient() service.AdminServiceClient { @@ -34,3 +51,11 @@ func (c CommandContext) InputPipe() io.Reader { func (c CommandContext) AdminFetcherExt() ext.AdminFetcherExtInterface { return c.adminClientFetcherExt } + +func (c CommandContext) AdminUpdaterExt() ext.AdminUpdaterExtInterface { + return c.adminClientUpdateExt +} + +func (c CommandContext) AdminDeleterExt() ext.AdminDeleterExtInterface { + return c.adminClientDeleteExt +} diff --git a/flytectl/cmd/create/execution_test.go b/flytectl/cmd/create/execution_test.go index c833da4889..d83d99ae96 100644 --- a/flytectl/cmd/create/execution_test.go +++ b/flytectl/cmd/create/execution_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/flyteorg/flytectl/cmd/config" + cmdCore "github.com/flyteorg/flytectl/cmd/core" "github.com/flyteorg/flytectl/cmd/testutils" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -17,8 +18,9 @@ import ( // This function needs to be called after testutils.Steup() func createExecutionSetup() { ctx = testutils.Ctx - cmdCtx = testutils.CmdCtx mockClient = testutils.MockClient + // TODO: migrate to new command context from testutils + cmdCtx = cmdCore.NewCommandContext(mockClient, testutils.MockOutStream) sortedListLiteralType := core.Variable{ Type: &core.LiteralType{ Type: &core.LiteralType_CollectionType{ diff --git a/flytectl/cmd/delete/delete.go b/flytectl/cmd/delete/delete.go index 2b6cf36310..2c694db819 100644 --- a/flytectl/cmd/delete/delete.go +++ b/flytectl/cmd/delete/delete.go @@ -1,6 +1,7 @@ package delete import ( + "github.com/flyteorg/flytectl/cmd/config/subcommand" cmdcore "github.com/flyteorg/flytectl/cmd/core" "github.com/spf13/cobra" @@ -27,6 +28,9 @@ func RemoteDeleteCommand() *cobra.Command { terminateResourcesFuncs := map[string]cmdcore.CommandEntry{ "execution": {CmdFunc: terminateExecutionFunc, Aliases: []string{"executions"}, Short: execCmdShort, Long: execCmdLong}, + "task-resource-attribute": {CmdFunc: deleteTaskResourceAttributes, Aliases: []string{"task-resource-attributes"}, + Short: taskResourceAttributesShort, + Long: taskResourceAttributesLong, PFlagProvider: subcommand.DefaultTaskResourceDelConfig, ProjectDomainNotRequired: true}, } cmdcore.AddCommands(deleteCmd, terminateResourcesFuncs) return deleteCmd diff --git a/flytectl/cmd/delete/delete_test.go b/flytectl/cmd/delete/delete_test.go index 5543c88d28..dd3075edac 100644 --- a/flytectl/cmd/delete/delete_test.go +++ b/flytectl/cmd/delete/delete_test.go @@ -1,18 +1,32 @@ package delete import ( + "context" "sort" "testing" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/stretchr/testify/assert" ) +var ( + err error + ctx context.Context + mockClient *mocks.AdminServiceClient + cmdCtx cmdCore.CommandContext +) +var setup = testutils.Setup +var tearDownAndVerify = testutils.TearDownAndVerify + func TestDeleteCommand(t *testing.T) { deleteCommand := RemoteDeleteCommand() assert.Equal(t, deleteCommand.Use, "delete") assert.Equal(t, deleteCommand.Short, deleteCmdShort) assert.Equal(t, deleteCommand.Long, deleteCmdLong) - assert.Equal(t, len(deleteCommand.Commands()), 1) + assert.Equal(t, len(deleteCommand.Commands()), 2) cmdNouns := deleteCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { @@ -22,4 +36,8 @@ func TestDeleteCommand(t *testing.T) { assert.Equal(t, cmdNouns[0].Aliases, []string{"executions"}) assert.Equal(t, cmdNouns[0].Short, execCmdShort) assert.Equal(t, cmdNouns[0].Long, execCmdLong) + assert.Equal(t, cmdNouns[1].Use, "task-resource-attribute") + assert.Equal(t, cmdNouns[1].Aliases, []string{"task-resource-attributes"}) + assert.Equal(t, cmdNouns[1].Short, taskResourceAttributesShort) + assert.Equal(t, cmdNouns[1].Long, taskResourceAttributesLong) } diff --git a/flytectl/cmd/delete/execution_test.go b/flytectl/cmd/delete/execution_test.go index f2f3ec11bb..c9a5b11c62 100644 --- a/flytectl/cmd/delete/execution_test.go +++ b/flytectl/cmd/delete/execution_test.go @@ -1,13 +1,11 @@ package delete import ( - "context" "errors" - "io" "testing" - cmdCore "github.com/flyteorg/flytectl/cmd/core" - "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/testutils" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -15,68 +13,64 @@ import ( ) var ( - ctx context.Context - args []string + args []string + terminateExecRequests []*admin.ExecutionTerminateRequest ) -func setup() { - ctx = context.Background() - args = []string{} +func terminateExecutionSetup() { + ctx = testutils.Ctx + cmdCtx = testutils.CmdCtx + mockClient = testutils.MockClient + args = append(args, "exec1", "exec2") + terminateExecRequests = []*admin.ExecutionTerminateRequest{ + {Id: &core.WorkflowExecutionIdentifier{ + Name: "exec1", + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + }}, + {Id: &core.WorkflowExecutionIdentifier{ + Name: "exec2", + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + }}, + } } func TestTerminateExecutionFunc(t *testing.T) { setup() - args = append(args, "exec1", "exec2") - mockClient := new(mocks.AdminServiceClient) - mockOutStream := new(io.Writer) - cmdCtx := cmdCore.NewCommandContext(mockClient, *mockOutStream) - terminateExecRequests := []*admin.ExecutionTerminateRequest{ - {Id: &core.WorkflowExecutionIdentifier{Name: "exec1"}}, - {Id: &core.WorkflowExecutionIdentifier{Name: "exec2"}}, - } + terminateExecutionSetup() terminateExecResponse := &admin.ExecutionTerminateResponse{} mockClient.OnTerminateExecutionMatch(ctx, terminateExecRequests[0]).Return(terminateExecResponse, nil) mockClient.OnTerminateExecutionMatch(ctx, terminateExecRequests[1]).Return(terminateExecResponse, nil) - err := terminateExecutionFunc(ctx, args, cmdCtx) + err = terminateExecutionFunc(ctx, args, cmdCtx) assert.Nil(t, err) mockClient.AssertCalled(t, "TerminateExecution", ctx, terminateExecRequests[0]) mockClient.AssertCalled(t, "TerminateExecution", ctx, terminateExecRequests[1]) + tearDownAndVerify(t, "") } func TestTerminateExecutionFuncWithError(t *testing.T) { setup() - args = append(args, "exec1", "exec2") - mockClient := new(mocks.AdminServiceClient) - mockOutStream := new(io.Writer) - cmdCtx := cmdCore.NewCommandContext(mockClient, *mockOutStream) - terminateExecRequests := []*admin.ExecutionTerminateRequest{ - {Id: &core.WorkflowExecutionIdentifier{Name: "exec1"}}, - {Id: &core.WorkflowExecutionIdentifier{Name: "exec2"}}, - } + terminateExecutionSetup() terminateExecResponse := &admin.ExecutionTerminateResponse{} mockClient.OnTerminateExecutionMatch(ctx, terminateExecRequests[0]).Return(nil, errors.New("failed to terminate")) mockClient.OnTerminateExecutionMatch(ctx, terminateExecRequests[1]).Return(terminateExecResponse, nil) - err := terminateExecutionFunc(ctx, args, cmdCtx) + err = terminateExecutionFunc(ctx, args, cmdCtx) assert.Equal(t, errors.New("failed to terminate"), err) mockClient.AssertCalled(t, "TerminateExecution", ctx, terminateExecRequests[0]) mockClient.AssertNotCalled(t, "TerminateExecution", ctx, terminateExecRequests[1]) + tearDownAndVerify(t, "") } func TestTerminateExecutionFuncWithPartialSuccess(t *testing.T) { setup() - args = append(args, "exec1", "exec2") - mockClient := new(mocks.AdminServiceClient) - mockOutStream := new(io.Writer) - cmdCtx := cmdCore.NewCommandContext(mockClient, *mockOutStream) - terminateExecRequests := []*admin.ExecutionTerminateRequest{ - {Id: &core.WorkflowExecutionIdentifier{Name: "exec1"}}, - {Id: &core.WorkflowExecutionIdentifier{Name: "exec2"}}, - } + terminateExecutionSetup() terminateExecResponse := &admin.ExecutionTerminateResponse{} mockClient.OnTerminateExecutionMatch(ctx, terminateExecRequests[0]).Return(terminateExecResponse, nil) mockClient.OnTerminateExecutionMatch(ctx, terminateExecRequests[1]).Return(nil, errors.New("failed to terminate")) - err := terminateExecutionFunc(ctx, args, cmdCtx) + err = terminateExecutionFunc(ctx, args, cmdCtx) assert.Equal(t, errors.New("failed to terminate"), err) mockClient.AssertCalled(t, "TerminateExecution", ctx, terminateExecRequests[0]) mockClient.AssertCalled(t, "TerminateExecution", ctx, terminateExecRequests[1]) + tearDownAndVerify(t, "") } diff --git a/flytectl/cmd/delete/matchable_task_resource_attribute.go b/flytectl/cmd/delete/matchable_task_resource_attribute.go new file mode 100644 index 0000000000..57791d033d --- /dev/null +++ b/flytectl/cmd/delete/matchable_task_resource_attribute.go @@ -0,0 +1,97 @@ +package delete + +import ( + "context" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/config/subcommand" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flytestdlib/logger" +) + +const ( + taskResourceAttributesShort = "Deletes matchable resources of task attributes" + taskResourceAttributesLong = ` +Deletes task resource attributes for given project,domain combination or additionally with workflow name. + +Deletes task resource attribute for project and domain +Here the command delete task resource attributes for project flytectldemo and development domain. +:: + + flytectl delete task-resource-attribute -p flytectldemo -d development + + +Deleting task resource attribute using config file which was used for creating it. +Here the command deletes task resource attributes from the config file tra.yaml +eg: content of tra.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete task-resource-attribute --attrFile tra.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +Deleting task resource attribute for a workflow +Here the command deletes task resource attributes for a workflow + +:: + + flytectl delete task-resource-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage +` +) + +func deleteTaskResourceAttributes(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + delConfig := subcommand.DefaultTaskResourceDelConfig + var project string + var domain string + var workflowName string + + if len(delConfig.AttrFile) > 0 { + // Read the config from the file + taskResourceAttrFileConfig := subcommand.TaskResourceAttrFileConfig{} + if err := taskResourceAttrFileConfig.ReadConfigFromFile(delConfig.AttrFile); err != nil { + return err + } + // Get project domain workflow name from the read file. + project = taskResourceAttrFileConfig.Project + domain = taskResourceAttrFileConfig.Domain + workflowName = taskResourceAttrFileConfig.Workflow + } else { + // Get all the parameters for deletion from the command line + project = config.GetConfig().Project + domain = config.GetConfig().Domain + if len(args) == 1 { + workflowName = args[0] + } + } + + if len(workflowName) > 0 { + // Delete the workflow attribute from the admin. If the attribute doesn't exist , admin deesn't return an error and same behavior is followed here + err := cmdCtx.AdminDeleterExt().DeleteWorkflowAttributes(ctx, project, domain, workflowName, admin.MatchableResource_TASK_RESOURCE) + if err != nil { + return err + } + logger.Debugf(ctx, "Deleted task resource attributes from %v project and domain %v and workflow %v", project, domain, workflowName) + } else { + // Delete the project domain attribute from the admin. If the attribute doesn't exist , admin deesn't return an error and same behavior is followed here + err := cmdCtx.AdminDeleterExt().DeleteProjectDomainAttributes(ctx, project, domain, admin.MatchableResource_TASK_RESOURCE) + if err != nil { + return err + } + logger.Debugf(ctx, "Deleted task resource attributes from %v project and domain %v", project, domain) + } + return nil +} diff --git a/flytectl/cmd/delete/matchable_task_resource_attribute_test.go b/flytectl/cmd/delete/matchable_task_resource_attribute_test.go new file mode 100644 index 0000000000..211aeb694b --- /dev/null +++ b/flytectl/cmd/delete/matchable_task_resource_attribute_test.go @@ -0,0 +1,133 @@ +package delete + +import ( + "fmt" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/config/subcommand" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func deleteTaskResourceAttributeSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + subcommand.DefaultTaskResourceDelConfig = &subcommand.TaskResourceAttrDeleteConfig{} +} + +func TestDeleteTaskResourceAttributes(t *testing.T) { + t.Run("successful project domain attribute deletion commandline", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("failed project domain attribute deletion", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to delte project domain attributes")) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delte project domain attributes"), err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("successful project domain attribute deletion file", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "testdata/valid_project_domain_task_attribute.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("successful workflow attribute deletion", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("failed workflow attribute deletion", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "" + args := []string{"workflow1"} + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to delete workflow attribute")) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to delete workflow attribute"), err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow1", + admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("successful workflow attribute deletion file", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "testdata/valid_workflow_task_attribute.yaml" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.DeleterExt.AssertCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("workflow attribute deletion non existent file", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "testdata/non-existent" + // No args implying project domain attribute deletion + u.DeleterExt.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.NotNil(t, err) + u.DeleterExt.AssertNotCalled(t, "DeleteWorkflowAttributes", + ctx, "flytectldemo", "development", "core.control_flow.run_merge_sort.merge_sort", + admin.MatchableResource_TASK_RESOURCE) + }) + t.Run("attribute deletion invalid file", func(t *testing.T) { + setup() + deleteTaskResourceAttributeSetup() + // Empty attribute file + subcommand.DefaultTaskResourceDelConfig.AttrFile = "testdata/invalid_task_attribute.yaml" + // No args implying project domain attribute deletion + err = deleteTaskResourceAttributes(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, + fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), + err) + u.DeleterExt.AssertNotCalled(t, "DeleteProjectDomainAttributes", + ctx, "flytectldemo", "development", admin.MatchableResource_TASK_RESOURCE) + }) +} diff --git a/flytectl/cmd/delete/testdata/invalid_task_attribute.yaml b/flytectl/cmd/delete/testdata/invalid_task_attribute.yaml new file mode 100644 index 0000000000..3804d837a3 --- /dev/null +++ b/flytectl/cmd/delete/testdata/invalid_task_attribute.yaml @@ -0,0 +1,5 @@ +InvalidDomain: development +InvalidProject: flytectldemo +InvalidWorkflow: "" +cpu: "1" +memory: 150Mi \ No newline at end of file diff --git a/flytectl/cmd/delete/testdata/valid_project_domain_task_attribute.yaml b/flytectl/cmd/delete/testdata/valid_project_domain_task_attribute.yaml new file mode 100644 index 0000000000..0051a4c2cc --- /dev/null +++ b/flytectl/cmd/delete/testdata/valid_project_domain_task_attribute.yaml @@ -0,0 +1,9 @@ +Domain: development +Project: flytectldemo +Workflow: "" +defaults: + cpu: "1" + memory: 150Mi +limits: + cpu: "2" + memory: 450Mi \ No newline at end of file diff --git a/flytectl/cmd/delete/testdata/valid_workflow_task_attribute.yaml b/flytectl/cmd/delete/testdata/valid_workflow_task_attribute.yaml new file mode 100644 index 0000000000..2593cad309 --- /dev/null +++ b/flytectl/cmd/delete/testdata/valid_workflow_task_attribute.yaml @@ -0,0 +1,9 @@ +Domain: development +Project: flytectldemo +Workflow: core.control_flow.run_merge_sort.merge_sort +defaults: + cpu: "2" + memory: 250Mi +limits: + cpu: "3" + memory: 350Mi \ No newline at end of file diff --git a/flytectl/cmd/get/get.go b/flytectl/cmd/get/get.go index e875a0ca01..0b7bd5da93 100644 --- a/flytectl/cmd/get/get.go +++ b/flytectl/cmd/get/get.go @@ -1,6 +1,7 @@ package get import ( + "github.com/flyteorg/flytectl/cmd/config/subcommand" cmdcore "github.com/flyteorg/flytectl/cmd/core" "github.com/spf13/cobra" @@ -37,6 +38,9 @@ func CreateGetCommand() *cobra.Command { Long: launchPlanLong, PFlagProvider: launchPlanConfig}, "execution": {CmdFunc: getExecutionFunc, Aliases: []string{"executions"}, Short: executionShort, Long: executionLong}, + "task-resource-attribute": {CmdFunc: getTaskResourceAttributes, Aliases: []string{"task-resource-attributes"}, + Short: taskResourceAttributesShort, + Long: taskResourceAttributesLong, PFlagProvider: subcommand.DefaultTaskResourceFetchConfig}, } cmdcore.AddCommands(getCmd, getResourcesFuncs) diff --git a/flytectl/cmd/get/get_test.go b/flytectl/cmd/get/get_test.go index 852994db00..e277ba1624 100644 --- a/flytectl/cmd/get/get_test.go +++ b/flytectl/cmd/get/get_test.go @@ -37,7 +37,7 @@ func TestCreateGetCommand(t *testing.T) { assert.Equal(t, getCommand.Use, "get") assert.Equal(t, getCommand.Short, "Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project.") fmt.Println(getCommand.Commands()) - assert.Equal(t, len(getCommand.Commands()), 5) + assert.Equal(t, len(getCommand.Commands()), 6) cmdNouns := getCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { @@ -55,7 +55,11 @@ func TestCreateGetCommand(t *testing.T) { assert.Equal(t, cmdNouns[3].Use, "task") assert.Equal(t, cmdNouns[3].Aliases, []string{"tasks"}) assert.Equal(t, cmdNouns[3].Short, "Gets task resources") - assert.Equal(t, cmdNouns[4].Use, "workflow") - assert.Equal(t, cmdNouns[4].Aliases, []string{"workflows"}) - assert.Equal(t, cmdNouns[4].Short, "Gets workflow resources") + assert.Equal(t, cmdNouns[4].Use, "task-resource-attribute") + assert.Equal(t, cmdNouns[4].Aliases, []string{"task-resource-attributes"}) + assert.Equal(t, cmdNouns[4].Short, taskResourceAttributesShort) + assert.Equal(t, cmdNouns[4].Long, taskResourceAttributesLong) + assert.Equal(t, cmdNouns[5].Use, "workflow") + assert.Equal(t, cmdNouns[5].Aliases, []string{"workflows"}) + assert.Equal(t, cmdNouns[5].Short, "Gets workflow resources") } diff --git a/flytectl/cmd/get/launch_plan_test.go b/flytectl/cmd/get/launch_plan_test.go index 058e745cd0..5933746acb 100644 --- a/flytectl/cmd/get/launch_plan_test.go +++ b/flytectl/cmd/get/launch_plan_test.go @@ -5,7 +5,8 @@ import ( "os" "testing" - "github.com/flyteorg/flytectl/cmd/testutils" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + u "github.com/flyteorg/flytectl/cmd/testutils" "github.com/flyteorg/flytectl/pkg/ext/mocks" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -26,9 +27,10 @@ var ( ) func getLaunchPlanSetup() { - ctx = testutils.Ctx - cmdCtx = testutils.CmdCtx - mockClient = testutils.MockClient + ctx = u.Ctx + mockClient = u.MockClient + // TODO: migrate to new command context from testutils + cmdCtx = cmdCore.NewCommandContext(mockClient, u.MockOutStream) argsLp = []string{"launchplan1"} parameterMap := map[string]*core.Parameter{ "numbers": { diff --git a/flytectl/cmd/get/matchable_task_resource_attribute.go b/flytectl/cmd/get/matchable_task_resource_attribute.go new file mode 100644 index 0000000000..09e8b5df99 --- /dev/null +++ b/flytectl/cmd/get/matchable_task_resource_attribute.go @@ -0,0 +1,98 @@ +package get + +import ( + "context" + "fmt" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/config/subcommand" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +const ( + taskResourceAttributesShort = "Gets matchable resources of task attributes" + taskResourceAttributesLong = ` +Retrieves task resource attributes for given project,domain combination or additionally with workflow name. + +Retrieves task resource attribute for project and domain +Here the command get task resource attributes for project flytectldemo and development domain. +:: + + flytectl get task-resource-attribute -p flytectldemo -d development + +eg : O/P + +.. code-block:: json + + {"Project":"flytectldemo","Domain":"development","Workflow":"","defaults":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"2","memory":"450Mi"}} + +Writing the task resource attribute to a file. If there are no task resource attributes a file would be written with basic data populated. +Here the command gets task resource attributes and writes the config file to tra.yaml +eg: content of tra.yaml + +:: + + flytectl get task-resource-attribute --attrFile tra.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +Usage +` +) + +func getTaskResourceAttributes(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + var project string + var domain string + var workflowName string + + // Get the project domain workflow name parameters from the command line. Project and domain are mandatory for this command + project = config.GetConfig().Project + domain = config.GetConfig().Domain + if len(args) == 1 { + workflowName = args[0] + } + // Construct a shadow config for TaskResourceAttribute. The shadow config is not using ProjectDomainAttribute/Workflowattribute directly inorder to simplify the inputs. + taskResourceAttrFileConfig := subcommand.TaskResourceAttrFileConfig{Project: project, Domain: domain, Workflow: workflowName} + // Get the attribute file name from the command line config + fileName := subcommand.DefaultTaskResourceFetchConfig.AttrFile + + if len(workflowName) > 0 { + // Fetch the workflow attribute from the admin + workflowAttr, err := cmdCtx.AdminFetcherExt().FetchWorkflowAttributes(ctx, + project, domain, workflowName, admin.MatchableResource_TASK_RESOURCE) + if err != nil { + return err + } + if workflowAttr.GetAttributes() == nil || workflowAttr.GetAttributes().GetMatchingAttributes() == nil { + return fmt.Errorf("attribute doesn't exist") + } + // Update the shadow config with the fetched taskResourceAttribute which can then be written to a file which can then be called for an update. + taskResourceAttrFileConfig.TaskResourceAttributes = workflowAttr.GetAttributes().GetMatchingAttributes().GetTaskResourceAttributes() + } else { + // Fetch the project domain attribute from the admin + projectDomainAttr, err := cmdCtx.AdminFetcherExt().FetchProjectDomainAttributes(ctx, + project, domain, admin.MatchableResource_TASK_RESOURCE) + if err != nil { + return err + } + if projectDomainAttr.GetAttributes() == nil || projectDomainAttr.GetAttributes().GetMatchingAttributes() == nil { + return fmt.Errorf("attribute doesn't exist") + } + // Update the shadow config with the fetched taskResourceAttribute which can then be written to a file which can then be called for an update. + taskResourceAttrFileConfig.TaskResourceAttributes = projectDomainAttr.GetAttributes().GetMatchingAttributes().GetTaskResourceAttributes() + } + // Write the config to the file which can be used for update + taskResourceAttrFileConfig.DumpTaskResourceAttr(ctx, fileName) + return nil +} diff --git a/flytectl/cmd/get/matchable_task_resource_attribute_test.go b/flytectl/cmd/get/matchable_task_resource_attribute_test.go new file mode 100644 index 0000000000..00f3efad26 --- /dev/null +++ b/flytectl/cmd/get/matchable_task_resource_attribute_test.go @@ -0,0 +1,144 @@ +package get + +import ( + "fmt" + "os" + "testing" + + "github.com/flyteorg/flytectl/cmd/config" + "github.com/flyteorg/flytectl/cmd/config/subcommand" + u "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getTaskResourceAttributeSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + subcommand.DefaultTaskResourceFetchConfig = &subcommand.TaskResourceAttrFetchConfig{} + // Clean up the temp directory. + _ = os.Remove("temp-output-file") +} + +func TestGetTaskResourceAttributes(t *testing.T) { + taskResourceAttr := &admin.TaskResourceAttributes{ + Defaults: &admin.TaskResourceSpec{ + Cpu: "1", + Memory: "150Mi", + }, + Limits: &admin.TaskResourceSpec{ + Cpu: "2", + Memory: "350Mi", + }, + } + projectDomainResp := &admin.ProjectDomainAttributesGetResponse{ + Attributes: &admin.ProjectDomainAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_TaskResourceAttributes{ + TaskResourceAttributes: taskResourceAttr, + }, + }, + }, + } + workflowResp := &admin.WorkflowAttributesGetResponse{ + Attributes: &admin.WorkflowAttributes{ + Project: config.GetConfig().Project, + Domain: config.GetConfig().Domain, + Workflow: "workflow", + MatchingAttributes: &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_TaskResourceAttributes{ + TaskResourceAttributes: taskResourceAttr, + }, + }, + }, + } + t.Run("successful get project domain attribute", func(t *testing.T) { + var args []string + setup() + getTaskResourceAttributeSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_TASK_RESOURCE) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","defaults":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"2","memory":"350Mi"}}`) + }) + t.Run("successful get project domain attribute and write to file", func(t *testing.T) { + var args []string + setup() + getTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceFetchConfig.AttrFile = "temp-output-file" + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_TASK_RESOURCE) + tearDownAndVerify(t, `wrote the config to file temp-output-file`) + }) + t.Run("successful get project domain attribute and write to file failure", func(t *testing.T) { + var args []string + setup() + getTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceFetchConfig.AttrFile = "non-existent-dir/temp-output-file" + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(projectDomainResp, nil) + err = getTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_TASK_RESOURCE) + tearDownAndVerify(t, `error dumping in file due to open non-existent-dir/temp-output-file: no such file or directory`) + }) + t.Run("failed get project domain attribute", func(t *testing.T) { + var args []string + setup() + getTaskResourceAttributeSetup() + // No args implying project domain attribute deletion + u.FetcherExt.OnFetchProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getTaskResourceAttributes(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchProjectDomainAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, admin.MatchableResource_TASK_RESOURCE) + tearDownAndVerify(t, ``) + }) + t.Run("successful get workflow attribute", func(t *testing.T) { + var args []string + setup() + getTaskResourceAttributeSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(workflowResp, nil) + err = getTaskResourceAttributes(ctx, args, cmdCtx) + assert.Nil(t, err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", + admin.MatchableResource_TASK_RESOURCE) + tearDownAndVerify(t, `{"project":"dummyProject","domain":"dummyDomain","workflow":"workflow","defaults":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"2","memory":"350Mi"}}`) + }) + t.Run("failed get workflow attribute", func(t *testing.T) { + var args []string + setup() + getTaskResourceAttributeSetup() + args = []string{"workflow"} + u.FetcherExt.OnFetchWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed to fetch response")) + err = getTaskResourceAttributes(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to fetch response"), err) + u.FetcherExt.AssertCalled(t, "FetchWorkflowAttributes", + ctx, config.GetConfig().Project, config.GetConfig().Domain, "workflow", + admin.MatchableResource_TASK_RESOURCE) + tearDownAndVerify(t, ``) + }) +} diff --git a/flytectl/cmd/get/task_test.go b/flytectl/cmd/get/task_test.go index e5f2c62246..7164840518 100644 --- a/flytectl/cmd/get/task_test.go +++ b/flytectl/cmd/get/task_test.go @@ -5,7 +5,8 @@ import ( "os" "testing" - "github.com/flyteorg/flytectl/cmd/testutils" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + u "github.com/flyteorg/flytectl/cmd/testutils" "github.com/flyteorg/flytectl/pkg/ext/mocks" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -26,9 +27,10 @@ var ( ) func getTaskSetup() { - ctx = testutils.Ctx - cmdCtx = testutils.CmdCtx - mockClient = testutils.MockClient + ctx = u.Ctx + mockClient = u.MockClient + // TODO: migrate to new command context from testutils + cmdCtx = cmdCore.NewCommandContext(mockClient, u.MockOutStream) argsTask = []string{"task1"} sortedListLiteralType := core.Variable{ Type: &core.LiteralType{ diff --git a/flytectl/cmd/get/taskconfig_flags.go b/flytectl/cmd/get/taskconfig_flags.go index 71a3fadabc..525884466b 100755 --- a/flytectl/cmd/get/taskconfig_flags.go +++ b/flytectl/cmd/get/taskconfig_flags.go @@ -40,7 +40,7 @@ func (TaskConfig) mustMarshalJSON(v json.Marshaler) string { // GetPFlagSet will return strongly types pflags for all fields in TaskConfig and its nested types. The format of the // flags is json-name.json-sub-name... etc. func (cfg TaskConfig) GetPFlagSet(prefix string) *pflag.FlagSet { - cmdFlags := pflag.NewFlagSet("TaskConfig", pflag.ExitOnError) + cmdFlags := pflag.NewFlagSet("TaskResourceAttrConfig", pflag.ExitOnError) cmdFlags.StringVar(&(taskConfig.ExecFile), fmt.Sprintf("%v%v", prefix, "execFile"), taskConfig.ExecFile, "execution file name to be used for generating execution spec of a single task.") cmdFlags.StringVar(&(taskConfig.Version), fmt.Sprintf("%v%v", prefix, "version"), taskConfig.Version, "version of the task to be fetched.") cmdFlags.BoolVar(&(taskConfig.Latest), fmt.Sprintf("%v%v", prefix, "latest"), taskConfig.Latest, "flag to indicate to fetch the latest version, version flag will be ignored in this case") diff --git a/flytectl/cmd/testutils/test_utils.go b/flytectl/cmd/testutils/test_utils.go index 3285ed1523..f386341f32 100644 --- a/flytectl/cmd/testutils/test_utils.go +++ b/flytectl/cmd/testutils/test_utils.go @@ -11,7 +11,7 @@ import ( "github.com/flyteorg/flytectl/cmd/config" cmdCore "github.com/flyteorg/flytectl/cmd/core" - "github.com/flyteorg/flytectl/pkg/ext" + extMocks "github.com/flyteorg/flytectl/pkg/ext/mocks" "github.com/flyteorg/flyteidl/clients/go/admin/mocks" "github.com/stretchr/testify/assert" @@ -27,8 +27,10 @@ var ( Err error Ctx context.Context MockClient *mocks.AdminServiceClient - FetcherExt ext.AdminFetcherExtInterface - mockOutStream io.Writer + FetcherExt *extMocks.AdminFetcherExtInterface + UpdaterExt *extMocks.AdminUpdaterExtInterface + DeleterExt *extMocks.AdminDeleterExtInterface + MockOutStream io.Writer CmdCtx cmdCore.CommandContext stdOut *os.File stderr *os.File @@ -46,13 +48,20 @@ func Setup() { os.Stderr = writer log.SetOutput(writer) MockClient = new(mocks.AdminServiceClient) - mockOutStream = writer - CmdCtx = cmdCore.NewCommandContext(MockClient, mockOutStream) + FetcherExt = new(extMocks.AdminFetcherExtInterface) + UpdaterExt = new(extMocks.AdminUpdaterExtInterface) + DeleterExt = new(extMocks.AdminDeleterExtInterface) + FetcherExt.OnAdminServiceClient().Return(MockClient) + UpdaterExt.OnAdminServiceClient().Return(MockClient) + DeleterExt.OnAdminServiceClient().Return(MockClient) + MockOutStream = writer + CmdCtx = cmdCore.NewCommandContextWithExt(MockClient, FetcherExt, UpdaterExt, DeleterExt, MockOutStream) config.GetConfig().Project = projectValue config.GetConfig().Domain = domainValue config.GetConfig().Output = output } +// TearDownAndVerify TODO: Change this to verify log lines from context func TearDownAndVerify(t *testing.T, expectedLog string) { writer.Close() os.Stdout = stdOut diff --git a/flytectl/cmd/update/matchable_task_resource_attribute.go b/flytectl/cmd/update/matchable_task_resource_attribute.go new file mode 100644 index 0000000000..239a81d736 --- /dev/null +++ b/flytectl/cmd/update/matchable_task_resource_attribute.go @@ -0,0 +1,95 @@ +package update + +import ( + "context" + "fmt" + + "github.com/flyteorg/flytectl/cmd/config/subcommand" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flytestdlib/logger" +) + +const ( + taskResourceAttributesShort = "Updates matchable resources of task attributes" + taskResourceAttributesLong = ` +Updates task resource attributes for given project and domain combination or additionally with workflow name. + +Updating the task resource attribute is only available from a generated file. See the get section for generating this file. +Here the command updates takes the input for task resource attributes from the config file tra.yaml +eg: content of tra.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +:: + + flytectl update task-resource-attribute -attrFile tra.yaml + +Updating task resource attribute for project and domain and workflow combination. This will take precedence over any other +resource attribute defined at project domain level. +Update the resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +:: + + flytectl update task-resource-attribute -attrFile tra.yaml + +Usage + +` +) + +func updateTaskResourceAttributesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { + updateConfig := subcommand.DefaultTaskResourceUpdateConfig + if len(updateConfig.AttrFile) == 0 { + return fmt.Errorf("attrFile is mandatory while calling update for task resource attribute") + } + + taskResourceAttrFileConfig := subcommand.TaskResourceAttrFileConfig{} + if err := taskResourceAttrFileConfig.ReadConfigFromFile(updateConfig.AttrFile); err != nil { + return err + } + + // Get project domain workflow name from the read file. + project := taskResourceAttrFileConfig.Project + domain := taskResourceAttrFileConfig.Domain + workflowName := taskResourceAttrFileConfig.Workflow + + // decorate the taskresource Attributes with MatchingAttributes + matchingAttr := taskResourceAttrFileConfig.MatchableAttributeDecorator() + + if len(workflowName) > 0 { + // Update the workflow attribute using the admin. + err := cmdCtx.AdminUpdaterExt().UpdateWorkflowAttributes(ctx, project, domain, workflowName, matchingAttr) + if err != nil { + return err + } + logger.Debugf(ctx, "Updated task resource attributes from %v project and domain %v and workflow %v", project, domain, workflowName) + } else { + // Update the project domain attribute using the admin. + err := cmdCtx.AdminUpdaterExt().UpdateProjectDomainAttributes(ctx, project, domain, matchingAttr) + if err != nil { + return err + } + logger.Debugf(ctx, "Updated task resource attributes from %v project and domain %v", project, domain) + } + return nil +} diff --git a/flytectl/cmd/update/matchable_task_resource_attribute_test.go b/flytectl/cmd/update/matchable_task_resource_attribute_test.go new file mode 100644 index 0000000000..4ab7b84cab --- /dev/null +++ b/flytectl/cmd/update/matchable_task_resource_attribute_test.go @@ -0,0 +1,94 @@ +package update + +import ( + "fmt" + "testing" + + "github.com/flyteorg/flytectl/cmd/config/subcommand" + u "github.com/flyteorg/flytectl/cmd/testutils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func updateTaskResourceAttributeSetup() { + ctx = u.Ctx + cmdCtx = u.CmdCtx + mockClient = u.MockClient + subcommand.DefaultTaskResourceUpdateConfig = &subcommand.TaskResourceAttrUpdateConfig{} +} + +func TestUpdateTaskResourceAttributes(t *testing.T) { + t.Run("no input file for update", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + err = updateTaskResourceAttributesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("attrFile is mandatory while calling update for task resource attribute"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update project domain attribute", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceUpdateConfig.AttrFile = "testdata/valid_project_domain_task_attribute.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(nil) + err = updateTaskResourceAttributesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update project domain attribute", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceUpdateConfig.AttrFile = "testdata/valid_project_domain_task_attribute.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updateTaskResourceAttributesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("successful update workflow attribute", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceUpdateConfig.AttrFile = "testdata/valid_workflow_task_attribute.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(nil) + err = updateTaskResourceAttributesFunc(ctx, nil, cmdCtx) + assert.Nil(t, err) + tearDownAndVerify(t, ``) + }) + t.Run("failed update workflow attribute", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceUpdateConfig.AttrFile = "testdata/valid_workflow_task_attribute.yaml" + // No args implying project domain attribute deletion + u.UpdaterExt.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything).Return(fmt.Errorf("failed to update attributes")) + err = updateTaskResourceAttributesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("failed to update attributes"), err) + tearDownAndVerify(t, ``) + }) + t.Run("non existent file", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceUpdateConfig.AttrFile = "testdata/non-existent-filel" + err = updateTaskResourceAttributesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("unable to read from testdata/non-existent-filel yaml file"), err) + tearDownAndVerify(t, ``) + }) + t.Run("invalid update file", func(t *testing.T) { + setup() + updateTaskResourceAttributeSetup() + subcommand.DefaultTaskResourceUpdateConfig.AttrFile = "testdata/invalid_task_attribute.yaml" + err = updateTaskResourceAttributesFunc(ctx, nil, cmdCtx) + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("error unmarshaling JSON: while decoding JSON: json: unknown field \"InvalidDomain\""), err) + tearDownAndVerify(t, ``) + }) +} diff --git a/flytectl/cmd/update/project_test.go b/flytectl/cmd/update/project_test.go index a3e7d9e2e5..e3eac5ff84 100644 --- a/flytectl/cmd/update/project_test.go +++ b/flytectl/cmd/update/project_test.go @@ -1,17 +1,13 @@ package update import ( - "bytes" - "context" "errors" - "io" - "log" - "os" + "fmt" "testing" + "github.com/flyteorg/flytectl/clierrors" "github.com/flyteorg/flytectl/cmd/config" - cmdCore "github.com/flyteorg/flytectl/cmd/core" - "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + u "github.com/flyteorg/flytectl/cmd/testutils" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" "github.com/stretchr/testify/assert" @@ -20,49 +16,19 @@ import ( const projectValue = "dummyProject" var ( - reader *os.File - writer *os.File - err error - ctx context.Context - mockClient *mocks.AdminServiceClient - mockOutStream io.Writer args []string - cmdCtx cmdCore.CommandContext projectUpdateRequest *admin.Project - stdOut *os.File - stderr *os.File ) -func setup() { - reader, writer, err = os.Pipe() - if err != nil { - panic(err) - } - stdOut = os.Stdout - stderr = os.Stderr - os.Stdout = writer - os.Stderr = writer - log.SetOutput(writer) - config.GetConfig().Project = projectValue - mockClient = new(mocks.AdminServiceClient) - mockOutStream = writer - cmdCtx = cmdCore.NewCommandContext(mockClient, mockOutStream) +func updateProjectSetup() { + mockClient = u.MockClient + cmdCtx = u.CmdCtx projectUpdateRequest = &admin.Project{ Id: projectValue, State: admin.Project_ACTIVE, } } -func teardownAndVerify(t *testing.T, expectedLog string) { - writer.Close() - os.Stdout = stdOut - os.Stderr = stderr - var buf bytes.Buffer - if _, err := io.Copy(&buf, reader); err != nil { - assert.Equal(t, expectedLog, buf.String()) - } -} - func modifyProjectFlags(archiveProject *bool, newArchiveVal bool, activateProject *bool, newActivateVal bool) { *archiveProject = newArchiveVal *activateProject = newActivateVal @@ -70,69 +36,76 @@ func modifyProjectFlags(archiveProject *bool, newArchiveVal bool, activateProjec func TestActivateProjectFunc(t *testing.T) { setup() - defer teardownAndVerify(t, "Project dummyProject updated to ACTIVE state\n") + updateProjectSetup() modifyProjectFlags(&(projectConfig.ArchiveProject), false, &(projectConfig.ActivateProject), true) mockClient.OnUpdateProjectMatch(ctx, projectUpdateRequest).Return(nil, nil) - err := updateProjectsFunc(ctx, args, cmdCtx) + err = updateProjectsFunc(ctx, args, cmdCtx) assert.Nil(t, err) mockClient.AssertCalled(t, "UpdateProject", ctx, projectUpdateRequest) + tearDownAndVerify(t, "Project dummyProject updated to ACTIVE state\n") } func TestActivateProjectFuncWithError(t *testing.T) { setup() - defer teardownAndVerify(t, "Project dummyProject failed to get updated to ACTIVE state due to Error Updating Project\n") + updateProjectSetup() modifyProjectFlags(&(projectConfig.ArchiveProject), false, &(projectConfig.ActivateProject), true) mockClient.OnUpdateProjectMatch(ctx, projectUpdateRequest).Return(nil, errors.New("Error Updating Project")) - err := updateProjectsFunc(ctx, args, cmdCtx) + err = updateProjectsFunc(ctx, args, cmdCtx) assert.NotNil(t, err) mockClient.AssertCalled(t, "UpdateProject", ctx, projectUpdateRequest) + tearDownAndVerify(t, "Project dummyProject failed to get updated to ACTIVE state due to Error Updating Project\n") } func TestArchiveProjectFunc(t *testing.T) { setup() - defer teardownAndVerify(t, "Project dummyProject updated to ARCHIVED state\n") + updateProjectSetup() modifyProjectFlags(&(projectConfig.ArchiveProject), true, &(projectConfig.ActivateProject), false) - projectUpdateRequest := &admin.Project{ + projectUpdateRequest = &admin.Project{ Id: projectValue, State: admin.Project_ARCHIVED, } mockClient.OnUpdateProjectMatch(ctx, projectUpdateRequest).Return(nil, nil) - err := updateProjectsFunc(ctx, args, cmdCtx) + err = updateProjectsFunc(ctx, args, cmdCtx) assert.Nil(t, err) mockClient.AssertCalled(t, "UpdateProject", ctx, projectUpdateRequest) + tearDownAndVerify(t, "Project dummyProject updated to ARCHIVED state\n") } func TestArchiveProjectFuncWithError(t *testing.T) { setup() - defer teardownAndVerify(t, "Project dummyProject failed to get updated to ARCHIVED state due to Error Updating Project\n") + updateProjectSetup() modifyProjectFlags(&(projectConfig.ArchiveProject), true, &(projectConfig.ActivateProject), false) - projectUpdateRequest := &admin.Project{ + projectUpdateRequest = &admin.Project{ Id: projectValue, State: admin.Project_ARCHIVED, } mockClient.OnUpdateProjectMatch(ctx, projectUpdateRequest).Return(nil, errors.New("Error Updating Project")) - err := updateProjectsFunc(ctx, args, cmdCtx) + err = updateProjectsFunc(ctx, args, cmdCtx) assert.NotNil(t, err) mockClient.AssertCalled(t, "UpdateProject", ctx, projectUpdateRequest) + tearDownAndVerify(t, "Project dummyProject failed to get updated to ARCHIVED state due to Error Updating Project\n") } func TestEmptyProjectInput(t *testing.T) { setup() - defer teardownAndVerify(t, "Project not passed\n") + updateProjectSetup() config.GetConfig().Project = "" modifyProjectFlags(&(projectConfig.ArchiveProject), false, &(projectConfig.ActivateProject), true) mockClient.OnUpdateProjectMatch(ctx, projectUpdateRequest).Return(nil, nil) - err := updateProjectsFunc(ctx, args, cmdCtx) + err = updateProjectsFunc(ctx, args, cmdCtx) assert.Nil(t, err) mockClient.AssertNotCalled(t, "UpdateProject", ctx, projectUpdateRequest) + tearDownAndVerify(t, "Project not passed") } func TestInvalidInput(t *testing.T) { setup() - defer teardownAndVerify(t, "Invalid state passed. Specify either activate or archive\n") + updateProjectSetup() modifyProjectFlags(&(projectConfig.ArchiveProject), false, &(projectConfig.ActivateProject), false) mockClient.OnUpdateProjectMatch(ctx, projectUpdateRequest).Return(nil, nil) - err := updateProjectsFunc(ctx, args, cmdCtx) + err = updateProjectsFunc(ctx, args, cmdCtx) assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf(clierrors.ErrInvalidStateUpdate), err) mockClient.AssertNotCalled(t, "UpdateProject", ctx, projectUpdateRequest) + tearDownAndVerify(t, "") } diff --git a/flytectl/cmd/update/testdata/invalid_task_attribute.yaml b/flytectl/cmd/update/testdata/invalid_task_attribute.yaml new file mode 100644 index 0000000000..3804d837a3 --- /dev/null +++ b/flytectl/cmd/update/testdata/invalid_task_attribute.yaml @@ -0,0 +1,5 @@ +InvalidDomain: development +InvalidProject: flytectldemo +InvalidWorkflow: "" +cpu: "1" +memory: 150Mi \ No newline at end of file diff --git a/flytectl/cmd/update/testdata/valid_project_domain_task_attribute.yaml b/flytectl/cmd/update/testdata/valid_project_domain_task_attribute.yaml new file mode 100644 index 0000000000..0051a4c2cc --- /dev/null +++ b/flytectl/cmd/update/testdata/valid_project_domain_task_attribute.yaml @@ -0,0 +1,9 @@ +Domain: development +Project: flytectldemo +Workflow: "" +defaults: + cpu: "1" + memory: 150Mi +limits: + cpu: "2" + memory: 450Mi \ No newline at end of file diff --git a/flytectl/cmd/update/testdata/valid_workflow_task_attribute.yaml b/flytectl/cmd/update/testdata/valid_workflow_task_attribute.yaml new file mode 100644 index 0000000000..2593cad309 --- /dev/null +++ b/flytectl/cmd/update/testdata/valid_workflow_task_attribute.yaml @@ -0,0 +1,9 @@ +Domain: development +Project: flytectldemo +Workflow: core.control_flow.run_merge_sort.merge_sort +defaults: + cpu: "2" + memory: 250Mi +limits: + cpu: "3" + memory: 350Mi \ No newline at end of file diff --git a/flytectl/cmd/update/update.go b/flytectl/cmd/update/update.go index 3f801ce08f..c842c0855b 100644 --- a/flytectl/cmd/update/update.go +++ b/flytectl/cmd/update/update.go @@ -1,6 +1,7 @@ package update import ( + "github.com/flyteorg/flytectl/cmd/config/subcommand" cmdCore "github.com/flyteorg/flytectl/cmd/core" "github.com/spf13/cobra" ) @@ -35,6 +36,8 @@ func CreateUpdateCommand() *cobra.Command { Short: updateTaskShort, Long: updateTaskLong}, "workflow": {CmdFunc: updateWorkflowFunc, Aliases: []string{}, ProjectDomainNotRequired: false, PFlagProvider: namedEntityConfig, Short: updateWorkflowShort, Long: updateWorkflowLong}, + "task-resource-attribute": {CmdFunc: updateTaskResourceAttributesFunc, Aliases: []string{}, PFlagProvider: subcommand.DefaultTaskResourceUpdateConfig, + Short: taskResourceAttributesShort, Long: taskResourceAttributesLong, ProjectDomainNotRequired: true}, } cmdCore.AddCommands(updateCmd, updateResourcesFuncs) return updateCmd diff --git a/flytectl/cmd/update/update_test.go b/flytectl/cmd/update/update_test.go index 9a610f9f5d..33e9b9eb5e 100644 --- a/flytectl/cmd/update/update_test.go +++ b/flytectl/cmd/update/update_test.go @@ -1,27 +1,41 @@ package update import ( + "context" "sort" "testing" + cmdCore "github.com/flyteorg/flytectl/cmd/core" + "github.com/flyteorg/flytectl/cmd/testutils" + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/stretchr/testify/assert" ) +var ( + err error + ctx context.Context + mockClient *mocks.AdminServiceClient + cmdCtx cmdCore.CommandContext +) +var setup = testutils.Setup +var tearDownAndVerify = testutils.TearDownAndVerify + func TestUpdateCommand(t *testing.T) { updateCommand := CreateUpdateCommand() assert.Equal(t, updateCommand.Use, updateUse) assert.Equal(t, updateCommand.Short, updateShort) assert.Equal(t, updateCommand.Long, updatecmdLong) - assert.Equal(t, len(updateCommand.Commands()), 4) + assert.Equal(t, len(updateCommand.Commands()), 5) cmdNouns := updateCommand.Commands() // Sort by Use value. sort.Slice(cmdNouns, func(i, j int) bool { return cmdNouns[i].Use < cmdNouns[j].Use }) - useArray := []string{"launchplan", "project", "task", "workflow"} - aliases := [][]string{{}, {}, {}, {}} - shortArray := []string{updateLPShort, projectShort, updateTaskShort, updateWorkflowShort} - longArray := []string{updateLPLong, projectLong, updateTaskLong, updateWorkflowLong} + useArray := []string{"launchplan", "project", "task", "task-resource-attribute", "workflow"} + aliases := [][]string{{}, {}, {}, {}, {}} + shortArray := []string{updateLPShort, projectShort, updateTaskShort, taskResourceAttributesShort, updateWorkflowShort} + longArray := []string{updateLPLong, projectLong, updateTaskLong, taskResourceAttributesLong, updateWorkflowLong} for i := range cmdNouns { assert.Equal(t, cmdNouns[i].Use, useArray[i]) assert.Equal(t, cmdNouns[i].Aliases, aliases[i]) diff --git a/flytectl/docs/coverage.out b/flytectl/docs/coverage.out new file mode 100644 index 0000000000..5f02b11199 --- /dev/null +++ b/flytectl/docs/coverage.out @@ -0,0 +1 @@ +mode: set diff --git a/flytectl/docs/source/gen/flytectl.rst b/flytectl/docs/source/gen/flytectl.rst index 89e4a4db9c..c878c96298 100644 --- a/flytectl/docs/source/gen/flytectl.rst +++ b/flytectl/docs/source/gen/flytectl.rst @@ -17,20 +17,20 @@ Options :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. -h, --help help for flytectl --logger.formatter.type string Sets logging format type. (default "json") diff --git a/flytectl/docs/source/gen/flytectl_config.rst b/flytectl/docs/source/gen/flytectl_config.rst index d3bf35e5bf..a2f96c1e7a 100644 --- a/flytectl/docs/source/gen/flytectl_config.rst +++ b/flytectl/docs/source/gen/flytectl_config.rst @@ -26,20 +26,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_config_discover.rst b/flytectl/docs/source/gen/flytectl_config_discover.rst index 2ce677324c..c5e86530a1 100644 --- a/flytectl/docs/source/gen/flytectl_config_discover.rst +++ b/flytectl/docs/source/gen/flytectl_config_discover.rst @@ -28,20 +28,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --file stringArray Passes the config file to load. If empty, it'll first search for the config file path then, if found, will load config from there. diff --git a/flytectl/docs/source/gen/flytectl_config_validate.rst b/flytectl/docs/source/gen/flytectl_config_validate.rst index ac0f848d42..7223e1a240 100644 --- a/flytectl/docs/source/gen/flytectl_config_validate.rst +++ b/flytectl/docs/source/gen/flytectl_config_validate.rst @@ -30,20 +30,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --file stringArray Passes the config file to load. If empty, it'll first search for the config file path then, if found, will load config from there. diff --git a/flytectl/docs/source/gen/flytectl_create.rst b/flytectl/docs/source/gen/flytectl_create.rst index a7528e911e..7233aa611a 100644 --- a/flytectl/docs/source/gen/flytectl_create.rst +++ b/flytectl/docs/source/gen/flytectl_create.rst @@ -29,20 +29,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_create_execution.rst b/flytectl/docs/source/gen/flytectl_create_execution.rst index 5487019357..ade3a51e11 100644 --- a/flytectl/docs/source/gen/flytectl_create_execution.rst +++ b/flytectl/docs/source/gen/flytectl_create_execution.rst @@ -102,20 +102,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_create_project.rst b/flytectl/docs/source/gen/flytectl_create_project.rst index 3b1aa15100..b99b60268f 100644 --- a/flytectl/docs/source/gen/flytectl_create_project.rst +++ b/flytectl/docs/source/gen/flytectl_create_project.rst @@ -53,20 +53,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_delete.rst b/flytectl/docs/source/gen/flytectl_delete.rst index f9c3e5b589..49006715a4 100644 --- a/flytectl/docs/source/gen/flytectl_delete.rst +++ b/flytectl/docs/source/gen/flytectl_delete.rst @@ -29,20 +29,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) @@ -72,4 +72,5 @@ SEE ALSO * :doc:`flytectl` - flyetcl CLI tool * :doc:`flytectl_delete_execution` - Terminate/Delete execution resources. +* :doc:`flytectl_delete_task-resource-attribute` - Deletes matchable resources of task attributes diff --git a/flytectl/docs/source/gen/flytectl_delete_execution.rst b/flytectl/docs/source/gen/flytectl_delete_execution.rst index d606658174..60dda389ad 100644 --- a/flytectl/docs/source/gen/flytectl_delete_execution.rst +++ b/flytectl/docs/source/gen/flytectl_delete_execution.rst @@ -72,20 +72,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_delete_task-resource-attribute.rst b/flytectl/docs/source/gen/flytectl_delete_task-resource-attribute.rst new file mode 100644 index 0000000000..4f19d5b32c --- /dev/null +++ b/flytectl/docs/source/gen/flytectl_delete_task-resource-attribute.rst @@ -0,0 +1,112 @@ +.. _flytectl_delete_task-resource-attribute: + +flytectl delete task-resource-attribute +--------------------------------------- + +Deletes matchable resources of task attributes + +Synopsis +~~~~~~~~ + + + +Deletes task resource attributes for given project,domain combination or additionally with workflow name. + +Deletes task resource attribute for project and domain +Here the command delete task resource attributes for project flytectldemo and development domain. +:: + + flytectl delete task-resource-attribute -p flytectldemo -d development + + +Deleting task resource attribute using config file which was used for creating it. +Here the command deletes task resource attributes from the config file tra.yaml +eg: content of tra.yaml which will use the project domain and workflow name for deleting the resource + +:: + + flytectl delete task-resource-attribute --attrFile tra.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +Deleting task resource attribute for a workflow +Here the command deletes task resource attributes for a workflow + +:: + + flytectl delete task-resource-attribute -p flytectldemo -d development core.control_flow.run_merge_sort.merge_sort + +Usage + + +:: + + flytectl delete task-resource-attribute [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for delete attribute for the resource type. + -h, --help help for task-resource-attribute + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_delete` - Used for terminating/deleting various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/flytectl/docs/source/gen/flytectl_get.rst b/flytectl/docs/source/gen/flytectl_get.rst index ae7a6ae65d..add9e81c9b 100644 --- a/flytectl/docs/source/gen/flytectl_get.rst +++ b/flytectl/docs/source/gen/flytectl_get.rst @@ -29,20 +29,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) @@ -75,5 +75,6 @@ SEE ALSO * :doc:`flytectl_get_launchplan` - Gets launch plan resources * :doc:`flytectl_get_project` - Gets project resources * :doc:`flytectl_get_task` - Gets task resources +* :doc:`flytectl_get_task-resource-attribute` - Gets matchable resources of task attributes * :doc:`flytectl_get_workflow` - Gets workflow resources diff --git a/flytectl/docs/source/gen/flytectl_get_execution.rst b/flytectl/docs/source/gen/flytectl_get_execution.rst index 48a01189cc..fad935ec01 100644 --- a/flytectl/docs/source/gen/flytectl_get_execution.rst +++ b/flytectl/docs/source/gen/flytectl_get_execution.rst @@ -58,20 +58,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_get_launchplan.rst b/flytectl/docs/source/gen/flytectl_get_launchplan.rst index 0cd64b8651..2f97de100f 100644 --- a/flytectl/docs/source/gen/flytectl_get_launchplan.rst +++ b/flytectl/docs/source/gen/flytectl_get_launchplan.rst @@ -85,20 +85,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_get_project.rst b/flytectl/docs/source/gen/flytectl_get_project.rst index 17c5d35b3a..e47f0061c2 100644 --- a/flytectl/docs/source/gen/flytectl_get_project.rst +++ b/flytectl/docs/source/gen/flytectl_get_project.rst @@ -58,20 +58,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_get_task-resource-attribute.rst b/flytectl/docs/source/gen/flytectl_get_task-resource-attribute.rst new file mode 100644 index 0000000000..05046e2469 --- /dev/null +++ b/flytectl/docs/source/gen/flytectl_get_task-resource-attribute.rst @@ -0,0 +1,110 @@ +.. _flytectl_get_task-resource-attribute: + +flytectl get task-resource-attribute +------------------------------------ + +Gets matchable resources of task attributes + +Synopsis +~~~~~~~~ + + + +Retrieves task resource attributes for given project,domain combination or additionally with workflow name. + +Retrieves task resource attribute for project and domain +Here the command get task resource attributes for project flytectldemo and development domain. +:: + + flytectl get task-resource-attribute -p flytectldemo -d development + +eg : O/P + +.. code-block:: json + + {"Project":"flytectldemo","Domain":"development","Workflow":"","defaults":{"cpu":"1","memory":"150Mi"},"limits":{"cpu":"2","memory":"450Mi"}} + +Writing the task resource attribute to a file. If there are no task resource attributes a file would be written with basic data populated. +Here the command gets task resource attributes and writes the config file to tra.yaml +eg: content of tra.yaml + +:: + + flytectl get task-resource-attribute --attrFile tra.yaml + + +.. code-block:: yaml + + domain: development + project: flytectldemo + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +Usage + + +:: + + flytectl get task-resource-attribute [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for generating attribute for the resource type. + -h, --help help for task-resource-attribute + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_get` - Used for fetching various flyte resources including tasks/workflows/launchplans/executions/project. + diff --git a/flytectl/docs/source/gen/flytectl_get_task.rst b/flytectl/docs/source/gen/flytectl_get_task.rst index 5ccddc30c9..8a770670f9 100644 --- a/flytectl/docs/source/gen/flytectl_get_task.rst +++ b/flytectl/docs/source/gen/flytectl_get_task.rst @@ -85,20 +85,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_get_workflow.rst b/flytectl/docs/source/gen/flytectl_get_workflow.rst index 4b4a996b2c..9a2288c75f 100644 --- a/flytectl/docs/source/gen/flytectl_get_workflow.rst +++ b/flytectl/docs/source/gen/flytectl_get_workflow.rst @@ -58,20 +58,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_register.rst b/flytectl/docs/source/gen/flytectl_register.rst index 19179afdfc..f02dbed437 100644 --- a/flytectl/docs/source/gen/flytectl_register.rst +++ b/flytectl/docs/source/gen/flytectl_register.rst @@ -29,20 +29,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_register_files.rst b/flytectl/docs/source/gen/flytectl_register_files.rst index 01074e4dc8..c3c90edb5e 100644 --- a/flytectl/docs/source/gen/flytectl_register_files.rst +++ b/flytectl/docs/source/gen/flytectl_register_files.rst @@ -75,20 +75,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_update.rst b/flytectl/docs/source/gen/flytectl_update.rst index 7db54b49a6..a30c1a9337 100644 --- a/flytectl/docs/source/gen/flytectl_update.rst +++ b/flytectl/docs/source/gen/flytectl_update.rst @@ -31,20 +31,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) @@ -76,5 +76,6 @@ SEE ALSO * :doc:`flytectl_update_launchplan` - Updates launch plan metadata * :doc:`flytectl_update_project` - Updates project resources * :doc:`flytectl_update_task` - Updates task metadata +* :doc:`flytectl_update_task-resource-attribute` - Updates matchable resources of task attributes * :doc:`flytectl_update_workflow` - Updates workflow metadata diff --git a/flytectl/docs/source/gen/flytectl_update_launchplan.rst b/flytectl/docs/source/gen/flytectl_update_launchplan.rst index 49bf4bffa9..cc76a1ab93 100644 --- a/flytectl/docs/source/gen/flytectl_update_launchplan.rst +++ b/flytectl/docs/source/gen/flytectl_update_launchplan.rst @@ -48,20 +48,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_update_project.rst b/flytectl/docs/source/gen/flytectl_update_project.rst index 4a6e7bee48..8a2e2525ca 100644 --- a/flytectl/docs/source/gen/flytectl_update_project.rst +++ b/flytectl/docs/source/gen/flytectl_update_project.rst @@ -73,20 +73,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_update_task-resource-attribute.rst b/flytectl/docs/source/gen/flytectl_update_task-resource-attribute.rst new file mode 100644 index 0000000000..a0aef81d04 --- /dev/null +++ b/flytectl/docs/source/gen/flytectl_update_task-resource-attribute.rst @@ -0,0 +1,117 @@ +.. _flytectl_update_task-resource-attribute: + +flytectl update task-resource-attribute +--------------------------------------- + +Updates matchable resources of task attributes + +Synopsis +~~~~~~~~ + + + +Updates task resource attributes for given project and domain combination or additionally with workflow name. + +Updating the task resource attribute is only available from a generated file. See the get section for generating this file. +Here the command updates takes the input for task resource attributes from the config file tra.yaml +eg: content of tra.yaml + +.. code-block:: yaml + + domain: development + project: flytectldemo + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +:: + + flytectl update task-resource-attribute -attrFile tra.yaml + +Updating task resource attribute for project and domain and workflow combination. This will take precedence over any other +resource attribute defined at project domain level. +Update the resource attributes for workflow core.control_flow.run_merge_sort.merge_sort in flytectldemo , development domain +.. code-block:: yaml + + domain: development + project: flytectldemo + workflow: core.control_flow.run_merge_sort.merge_sort + defaults: + cpu: "1" + memory: 150Mi + limits: + cpu: "2" + memory: 450Mi + +:: + + flytectl update task-resource-attribute -attrFile tra.yaml + +Usage + + + +:: + + flytectl update task-resource-attribute [flags] + +Options +~~~~~~~ + +:: + + --attrFile string attribute file name to be used for updating attribute for the resource type. + -h, --help help for task-resource-attribute + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --admin.authorizationHeader string Custom metadata header to pass JWT + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") + --admin.endpoint string For admin types, specify where the uri of the service is located. + --admin.insecure Use insecure connection. + --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") + --admin.maxRetries int Max number of gRPC retries (default 4) + --admin.perRetryTimeout string gRPC per retry timeout (default "15s") + --admin.scopes strings List of scopes to request + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. + --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) + --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) + --config string config file (default is $HOME/.flyte/config.yaml) + -d, --domain string Specifies the Flyte project's domain. + --logger.formatter.type string Sets logging format type. (default "json") + --logger.level int Sets the minimum logging level. (default 4) + --logger.mute Mutes all logs regardless of severity. Intended for benchmarks/tests only. + --logger.show-source Includes source code location in logs. + -o, --output string Specifies the output type - supported formats [TABLE JSON YAML] (default "TABLE") + -p, --project string Specifies the Flyte project. + --root.domain string Specified the domain to work on. + --root.output string Specified the output type. + --root.project string Specifies the project to work on. + --storage.cache.max_size_mbs int Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used + --storage.cache.target_gc_percent int Sets the garbage collection target percentage. + --storage.connection.access-key string Access key to use. Only required when authtype is set to accesskey. + --storage.connection.auth-type string Auth Type to use [iam, accesskey]. (default "iam") + --storage.connection.disable-ssl Disables SSL connection. Should only be used for development. + --storage.connection.endpoint string URL for storage client to connect to. + --storage.connection.region string Region to connect to. (default "us-east-1") + --storage.connection.secret-key string Secret to use when accesskey is set. + --storage.container string Initial container to create -if it doesn't exist-.' + --storage.defaultHttpClient.timeout string Sets time out on the http client. (default "0s") + --storage.enable-multicontainer If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered + --storage.limits.maxDownloadMBs int Maximum allowed download size (in MBs) per call. (default 2) + --storage.type string Sets the type of storage to configure [s3/minio/local/mem/stow]. (default "s3") + +SEE ALSO +~~~~~~~~ + +* :doc:`flytectl_update` - Used for updating flyte resources eg: project. + diff --git a/flytectl/docs/source/gen/flytectl_update_task.rst b/flytectl/docs/source/gen/flytectl_update_task.rst index 5a81366eb8..ba02121a13 100644 --- a/flytectl/docs/source/gen/flytectl_update_task.rst +++ b/flytectl/docs/source/gen/flytectl_update_task.rst @@ -48,20 +48,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_update_workflow.rst b/flytectl/docs/source/gen/flytectl_update_workflow.rst index 553446ca27..7320bb5423 100644 --- a/flytectl/docs/source/gen/flytectl_update_workflow.rst +++ b/flytectl/docs/source/gen/flytectl_update_workflow.rst @@ -48,20 +48,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/docs/source/gen/flytectl_version.rst b/flytectl/docs/source/gen/flytectl_version.rst index a56c061f65..84df72ed64 100644 --- a/flytectl/docs/source/gen/flytectl_version.rst +++ b/flytectl/docs/source/gen/flytectl_version.rst @@ -33,20 +33,20 @@ Options inherited from parent commands :: --admin.authorizationHeader string Custom metadata header to pass JWT - --admin.authorizationServerUrl string This is the URL to your IDP's authorization server' - --admin.clientId string Client ID - --admin.clientSecretLocation string File containing the client secret + --admin.authorizationServerUrl string This is the URL to your IdP's authorization server. It'll default to Endpoint + --admin.clientId string Client ID (default "flytepropeller") + --admin.clientSecretLocation string File containing the client secret (default "/etc/secrets/client_secret") --admin.endpoint string For admin types, specify where the uri of the service is located. --admin.insecure Use insecure connection. --admin.maxBackoffDelay string Max delay for grpc backoff (default "8s") --admin.maxRetries int Max number of gRPC retries (default 4) --admin.perRetryTimeout string gRPC per retry timeout (default "15s") --admin.scopes strings List of scopes to request - --admin.tokenUrl string Your IDPs token endpoint - --admin.useAuth Whether or not to try to authenticate with options below + --admin.tokenUrl string OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. + --admin.useAuth Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. --adminutils.batchSize int Maximum number of records to retrieve per call. (default 100) --adminutils.maxRecords int Maximum number of records to retrieve. (default 500) - --config string config file (default is $HOME/config.yaml) + --config string config file (default is $HOME/.flyte/config.yaml) -d, --domain string Specifies the Flyte project's domain. --logger.formatter.type string Sets logging format type. (default "json") --logger.level int Sets the minimum logging level. (default 4) diff --git a/flytectl/pkg/ext/attribute_match_deleter_test.go b/flytectl/pkg/ext/attribute_match_deleter_test.go new file mode 100644 index 0000000000..9c8c665b08 --- /dev/null +++ b/flytectl/pkg/ext/attribute_match_deleter_test.go @@ -0,0 +1,49 @@ +package ext + +import ( + "context" + "fmt" + "testing" + + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var adminDeleterExt AdminDeleterExtClient + +func deleteAttributeMatchFetcherSetup() { + ctx = context.Background() + adminClient = new(mocks.AdminServiceClient) + adminDeleterExt = AdminDeleterExtClient{AdminClient: adminClient} +} + +func TestDeleteWorkflowAttributes(t *testing.T) { + deleteAttributeMatchFetcherSetup() + adminClient.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything).Return(nil, nil) + err := adminDeleterExt.DeleteWorkflowAttributes(ctx, "dummyProject", "domainValue", "workflowName", admin.MatchableResource_TASK_RESOURCE) + assert.Nil(t, err) +} + +func TestDeleteWorkflowAttributesError(t *testing.T) { + deleteAttributeMatchFetcherSetup() + adminClient.OnDeleteWorkflowAttributesMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + err := adminDeleterExt.DeleteWorkflowAttributes(ctx, "dummyProject", "domainValue", "workflowName", admin.MatchableResource_TASK_RESOURCE) + assert.Equal(t, fmt.Errorf("failed"), err) +} + +func TestDeleteProjectDomainAttributes(t *testing.T) { + deleteAttributeMatchFetcherSetup() + adminClient.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything).Return(nil, nil) + err := adminDeleterExt.DeleteProjectDomainAttributes(ctx, "dummyProject", "domainValue", admin.MatchableResource_TASK_RESOURCE) + assert.Nil(t, err) +} + +func TestDeleteProjectDomainAttributesError(t *testing.T) { + deleteAttributeMatchFetcherSetup() + adminClient.OnDeleteProjectDomainAttributesMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + err := adminDeleterExt.DeleteProjectDomainAttributes(ctx, "dummyProject", "domainValue", admin.MatchableResource_TASK_RESOURCE) + assert.Equal(t, fmt.Errorf("failed"), err) +} diff --git a/flytectl/pkg/ext/attribute_match_fetcher.go b/flytectl/pkg/ext/attribute_match_fetcher.go new file mode 100644 index 0000000000..f3ea76b369 --- /dev/null +++ b/flytectl/pkg/ext/attribute_match_fetcher.go @@ -0,0 +1,29 @@ +package ext + +import ( + "context" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +func (a *AdminFetcherExtClient) FetchWorkflowAttributes(ctx context.Context, project, domain, name string, + rsType admin.MatchableResource) (*admin.WorkflowAttributesGetResponse, error) { + workflowAttr, err := a.AdminServiceClient().GetWorkflowAttributes(ctx, &admin.WorkflowAttributesGetRequest{ + Project: project, + Domain: domain, + Workflow: name, + ResourceType: rsType, + }) + return workflowAttr, err +} + +func (a *AdminFetcherExtClient) FetchProjectDomainAttributes(ctx context.Context, project, domain string, + rsType admin.MatchableResource) (*admin.ProjectDomainAttributesGetResponse, error) { + projectDomainAttr, err := a.AdminServiceClient().GetProjectDomainAttributes(ctx, + &admin.ProjectDomainAttributesGetRequest{ + Project: project, + Domain: domain, + ResourceType: rsType, + }) + return projectDomainAttr, err +} diff --git a/flytectl/pkg/ext/attribute_match_fetcher_test.go b/flytectl/pkg/ext/attribute_match_fetcher_test.go new file mode 100644 index 0000000000..b91d18fe3a --- /dev/null +++ b/flytectl/pkg/ext/attribute_match_fetcher_test.go @@ -0,0 +1,49 @@ +package ext + +import ( + "context" + "fmt" + "testing" + + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getAttributeMatchFetcherSetup() { + ctx = context.Background() + adminClient = new(mocks.AdminServiceClient) + adminFetcherExt = AdminFetcherExtClient{AdminClient: adminClient} +} + +func TestFetchWorkflowAttributes(t *testing.T) { + getAttributeMatchFetcherSetup() + resp := &admin.WorkflowAttributesGetResponse{} + adminClient.OnGetWorkflowAttributesMatch(mock.Anything, mock.Anything).Return(resp, nil) + _, err := adminFetcherExt.FetchWorkflowAttributes(ctx, "dummyProject", "domainValue", "workflowName", admin.MatchableResource_TASK_RESOURCE) + assert.Nil(t, err) +} + +func TestFetchWorkflowAttributesError(t *testing.T) { + getAttributeMatchFetcherSetup() + adminClient.OnGetWorkflowAttributesMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + _, err := adminFetcherExt.FetchWorkflowAttributes(ctx, "dummyProject", "domainValue", "workflowName", admin.MatchableResource_TASK_RESOURCE) + assert.Equal(t, fmt.Errorf("failed"), err) +} + +func TestFetchProjectDomainAttributes(t *testing.T) { + getAttributeMatchFetcherSetup() + resp := &admin.ProjectDomainAttributesGetResponse{} + adminClient.OnGetProjectDomainAttributesMatch(mock.Anything, mock.Anything).Return(resp, nil) + _, err := adminFetcherExt.FetchProjectDomainAttributes(ctx, "dummyProject", "domainValue", admin.MatchableResource_TASK_RESOURCE) + assert.Nil(t, err) +} + +func TestFetchProjectDomainAttributesError(t *testing.T) { + getAttributeMatchFetcherSetup() + adminClient.OnGetProjectDomainAttributesMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + _, err := adminFetcherExt.FetchProjectDomainAttributes(ctx, "dummyProject", "domainValue", admin.MatchableResource_TASK_RESOURCE) + assert.Equal(t, fmt.Errorf("failed"), err) +} diff --git a/flytectl/pkg/ext/attribute_match_updater.go b/flytectl/pkg/ext/attribute_match_updater.go new file mode 100644 index 0000000000..d06e22155e --- /dev/null +++ b/flytectl/pkg/ext/attribute_match_updater.go @@ -0,0 +1,31 @@ +package ext + +import ( + "context" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +func (a *AdminUpdaterExtClient) UpdateWorkflowAttributes(ctx context.Context, project, domain, name string, matchingAttr *admin.MatchingAttributes) error { + _, err := a.AdminServiceClient().UpdateWorkflowAttributes(ctx, &admin.WorkflowAttributesUpdateRequest{ + Attributes: &admin.WorkflowAttributes{ + Project: project, + Domain: domain, + Workflow: name, + MatchingAttributes: matchingAttr, + }, + }) + return err +} + +func (a *AdminUpdaterExtClient) UpdateProjectDomainAttributes(ctx context.Context, project, domain string, matchingAttr *admin.MatchingAttributes) error { + _, err := a.AdminServiceClient().UpdateProjectDomainAttributes(ctx, + &admin.ProjectDomainAttributesUpdateRequest{ + Attributes: &admin.ProjectDomainAttributes{ + Project: project, + Domain: domain, + MatchingAttributes: matchingAttr, + }, + }) + return err +} diff --git a/flytectl/pkg/ext/attribute_match_updater_test.go b/flytectl/pkg/ext/attribute_match_updater_test.go new file mode 100644 index 0000000000..f57d86a369 --- /dev/null +++ b/flytectl/pkg/ext/attribute_match_updater_test.go @@ -0,0 +1,55 @@ +package ext + +import ( + "context" + "fmt" + "testing" + + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var adminUpdaterExt AdminUpdaterExtClient + +func updateAttributeMatchFetcherSetup() { + ctx = context.Background() + adminClient = new(mocks.AdminServiceClient) + adminUpdaterExt = AdminUpdaterExtClient{AdminClient: adminClient} +} + +func TestUpdateWorkflowAttributes(t *testing.T) { + updateAttributeMatchFetcherSetup() + matchingAttr := &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_TaskResourceAttributes{}, + } + adminClient.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything).Return(nil, nil) + err := adminUpdaterExt.UpdateWorkflowAttributes(ctx, "dummyProject", "domainValue", "workflowName", matchingAttr) + assert.Nil(t, err) +} + +func TestUpdateWorkflowAttributesError(t *testing.T) { + updateAttributeMatchFetcherSetup() + adminClient.OnUpdateWorkflowAttributesMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + err := adminUpdaterExt.UpdateWorkflowAttributes(ctx, "dummyProject", "domainValue", "workflowName", nil) + assert.Equal(t, fmt.Errorf("failed"), err) +} + +func TestUpdateProjectDomainAttributes(t *testing.T) { + updateAttributeMatchFetcherSetup() + matchingAttr := &admin.MatchingAttributes{ + Target: &admin.MatchingAttributes_TaskResourceAttributes{}, + } + adminClient.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything).Return(nil, nil) + err := adminUpdaterExt.UpdateProjectDomainAttributes(ctx, "dummyProject", "domainValue", matchingAttr) + assert.Nil(t, err) +} + +func TestUpdateProjectDomainAttributesError(t *testing.T) { + updateAttributeMatchFetcherSetup() + adminClient.OnUpdateProjectDomainAttributesMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + err := adminUpdaterExt.UpdateProjectDomainAttributes(ctx, "dummyProject", "domainValue", nil) + assert.Equal(t, fmt.Errorf("failed"), err) +} diff --git a/flytectl/pkg/ext/attribute_matcher_deleter.go b/flytectl/pkg/ext/attribute_matcher_deleter.go new file mode 100644 index 0000000000..b7fdaec9b2 --- /dev/null +++ b/flytectl/pkg/ext/attribute_matcher_deleter.go @@ -0,0 +1,26 @@ +package ext + +import ( + "context" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" +) + +func (a *AdminDeleterExtClient) DeleteWorkflowAttributes(ctx context.Context, project, domain, name string, rsType admin.MatchableResource) error { + _, err := a.AdminServiceClient().DeleteWorkflowAttributes(ctx, &admin.WorkflowAttributesDeleteRequest{ + Project: project, + Domain: domain, + Workflow: name, + ResourceType: rsType, + }) + return err +} + +func (a *AdminDeleterExtClient) DeleteProjectDomainAttributes(ctx context.Context, project, domain string, rsType admin.MatchableResource) error { + _, err := a.AdminServiceClient().DeleteProjectDomainAttributes(ctx, &admin.ProjectDomainAttributesDeleteRequest{ + Project: project, + Domain: domain, + ResourceType: rsType, + }) + return err +} diff --git a/flytectl/pkg/ext/deleter.go b/flytectl/pkg/ext/deleter.go new file mode 100644 index 0000000000..03827d1079 --- /dev/null +++ b/flytectl/pkg/ext/deleter.go @@ -0,0 +1,33 @@ +package ext + +import ( + "context" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" +) + +//go:generate mockery -all -case=underscore + +// AdminDeleterExtInterface Interface for exposing the update capabilities from the admin +type AdminDeleterExtInterface interface { + AdminServiceClient() service.AdminServiceClient + + // DeleteWorkflowAttributes deletes workflow attributes within a project, domain for a particular matchable resource + DeleteWorkflowAttributes(ctx context.Context, project, domain, name string, rsType admin.MatchableResource) error + + // DeleteProjectDomainAttributes deletes project domain attributes for a particular matchable resource + DeleteProjectDomainAttributes(ctx context.Context, project, domain string, rsType admin.MatchableResource) error +} + +// AdminDeleterExtClient is used for interacting with extended features used for deleting/archiving data in admin service +type AdminDeleterExtClient struct { + AdminClient service.AdminServiceClient +} + +func (a *AdminDeleterExtClient) AdminServiceClient() service.AdminServiceClient { + if a == nil { + return nil + } + return a.AdminClient +} diff --git a/flytectl/pkg/ext/doc.go b/flytectl/pkg/ext/doc.go new file mode 100644 index 0000000000..aa5073ed52 --- /dev/null +++ b/flytectl/pkg/ext/doc.go @@ -0,0 +1,2 @@ +// Package ext Provides Fetch,Update and Delete extensions to the admin API's whose interface directly relates to flytectl commands +package ext diff --git a/flytectl/pkg/ext/execution_fetcher_ext.go b/flytectl/pkg/ext/execution_fetcher.go similarity index 100% rename from flytectl/pkg/ext/execution_fetcher_ext.go rename to flytectl/pkg/ext/execution_fetcher.go diff --git a/flytectl/pkg/ext/execution_fetcher_test.go b/flytectl/pkg/ext/execution_fetcher_test.go new file mode 100644 index 0000000000..add301a6da --- /dev/null +++ b/flytectl/pkg/ext/execution_fetcher_test.go @@ -0,0 +1,68 @@ +package ext + +import ( + "context" + "fmt" + "testing" + + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + executionResponse *admin.Execution +) + +func getExecutionFetcherSetup() { + ctx = context.Background() + adminClient = new(mocks.AdminServiceClient) + adminFetcherExt = AdminFetcherExtClient{AdminClient: adminClient} + projectValue := "dummyProject" + domainValue := "domainValue" + executionNameValue := "execName" + launchPlanNameValue := "launchPlanNameValue" + launchPlanVersionValue := "launchPlanVersionValue" + workflowNameValue := "workflowNameValue" + workflowVersionValue := "workflowVersionValue" + executionResponse = &admin.Execution{ + Id: &core.WorkflowExecutionIdentifier{ + Project: projectValue, + Domain: domainValue, + Name: executionNameValue, + }, + Spec: &admin.ExecutionSpec{ + LaunchPlan: &core.Identifier{ + Project: projectValue, + Domain: domainValue, + Name: launchPlanNameValue, + Version: launchPlanVersionValue, + }, + }, + Closure: &admin.ExecutionClosure{ + WorkflowId: &core.Identifier{ + Project: projectValue, + Domain: domainValue, + Name: workflowNameValue, + Version: workflowVersionValue, + }, + Phase: core.WorkflowExecution_SUCCEEDED, + }, + } +} + +func TestFetchExecutionVersion(t *testing.T) { + getExecutionFetcherSetup() + adminClient.OnGetExecutionMatch(mock.Anything, mock.Anything).Return(executionResponse, nil) + _, err := adminFetcherExt.FetchExecution(ctx, "execName", "dummyProject", "domainValue") + assert.Nil(t, err) +} + +func TestFetchExecutionError(t *testing.T) { + getExecutionFetcherSetup() + adminClient.OnGetExecutionMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + _, err := adminFetcherExt.FetchExecution(ctx, "execName", "dummyProject", "domainValue") + assert.Equal(t, fmt.Errorf("failed"), err) +} diff --git a/flytectl/pkg/ext/fetcher_ext_client.go b/flytectl/pkg/ext/fetcher.go similarity index 79% rename from flytectl/pkg/ext/fetcher_ext_client.go rename to flytectl/pkg/ext/fetcher.go index ebe34ca9d3..d32421156b 100644 --- a/flytectl/pkg/ext/fetcher_ext_client.go +++ b/flytectl/pkg/ext/fetcher.go @@ -34,6 +34,12 @@ type AdminFetcherExtInterface interface { // FetchTaskVersion fetches particular version of task in a project, domain FetchTaskVersion(ctx context.Context, name, version, project, domain string) (*admin.Task, error) + + // FetchWorkflowAttributes fetches workflow attributes particular resource type in a project, domain and workflow + FetchWorkflowAttributes(ctx context.Context, project, domain, name string, rsType admin.MatchableResource) (*admin.WorkflowAttributesGetResponse, error) + + // FetchProjectDomainAttributes fetches project domain attributes particular resource type in a project, domain + FetchProjectDomainAttributes(ctx context.Context, project, domain string, rsType admin.MatchableResource) (*admin.ProjectDomainAttributesGetResponse, error) } // AdminFetcherExtClient is used for interacting with extended features used for fetching data from admin service diff --git a/flytectl/pkg/ext/launch_plan_fetcher_test.go b/flytectl/pkg/ext/launch_plan_fetcher_test.go new file mode 100644 index 0000000000..7b40fbde89 --- /dev/null +++ b/flytectl/pkg/ext/launch_plan_fetcher_test.go @@ -0,0 +1,151 @@ +package ext + +import ( + "context" + "fmt" + "testing" + + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + launchPlanListResponse *admin.LaunchPlanList +) + +func getLaunchPlanFetcherSetup() { + ctx = context.Background() + adminClient = new(mocks.AdminServiceClient) + adminFetcherExt = AdminFetcherExtClient{AdminClient: adminClient} + + parameterMap := map[string]*core.Parameter{ + "numbers": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + }, + }, + "numbers_count": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + "run_local_at_count": { + Var: &core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + Behavior: &core.Parameter_Default{ + Default: &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: 10, + }, + }, + }, + }, + }, + }, + }, + }, + } + launchPlan1 := &admin.LaunchPlan{ + Id: &core.Identifier{ + Name: "launchplan1", + Version: "v1", + }, + Spec: &admin.LaunchPlanSpec{ + DefaultInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + Closure: &admin.LaunchPlanClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 0, Nanos: 0}, + ExpectedInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + } + launchPlan2 := &admin.LaunchPlan{ + Id: &core.Identifier{ + Name: "launchplan1", + Version: "v2", + }, + Spec: &admin.LaunchPlanSpec{ + DefaultInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + Closure: &admin.LaunchPlanClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + ExpectedInputs: &core.ParameterMap{ + Parameters: parameterMap, + }, + }, + } + + launchPlans := []*admin.LaunchPlan{launchPlan2, launchPlan1} + + launchPlanListResponse = &admin.LaunchPlanList{ + LaunchPlans: launchPlans, + } +} + +func TestFetchAllVerOfLP(t *testing.T) { + getLaunchPlanFetcherSetup() + adminClient.OnListLaunchPlansMatch(mock.Anything, mock.Anything).Return(launchPlanListResponse, nil) + _, err := adminFetcherExt.FetchAllVerOfLP(ctx, "lpName", "project", "domain") + assert.Nil(t, err) +} + +func TestFetchAllVerOfLPError(t *testing.T) { + getLaunchPlanFetcherSetup() + adminClient.OnListLaunchPlansMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + _, err := adminFetcherExt.FetchAllVerOfLP(ctx, "lpName", "project", "domain") + assert.Equal(t, fmt.Errorf("failed"), err) +} + +func TestFetchAllVerOfLPEmptyResponse(t *testing.T) { + launchPlanListResponse := &admin.LaunchPlanList{} + getLaunchPlanFetcherSetup() + adminClient.OnListLaunchPlansMatch(mock.Anything, mock.Anything).Return(launchPlanListResponse, nil) + _, err := adminFetcherExt.FetchAllVerOfLP(ctx, "lpName", "project", "domain") + assert.Equal(t, fmt.Errorf("no launchplans retrieved for lpName"), err) +} + +func TestFetchLPLatestVersion(t *testing.T) { + getLaunchPlanFetcherSetup() + adminClient.OnListLaunchPlansMatch(mock.Anything, mock.Anything).Return(launchPlanListResponse, nil) + _, err := adminFetcherExt.FetchLPLatestVersion(ctx, "lpName", "project", "domain") + assert.Nil(t, err) +} + +func TestFetchLPLatestVersionError(t *testing.T) { + launchPlanListResponse := &admin.LaunchPlanList{} + getLaunchPlanFetcherSetup() + adminClient.OnListLaunchPlansMatch(mock.Anything, mock.Anything).Return(launchPlanListResponse, nil) + _, err := adminFetcherExt.FetchLPLatestVersion(ctx, "lpName", "project", "domain") + assert.Equal(t, fmt.Errorf("no launchplans retrieved for lpName"), err) +} diff --git a/flytectl/pkg/ext/mocks/admin_deleter_ext_interface.go b/flytectl/pkg/ext/mocks/admin_deleter_ext_interface.go new file mode 100644 index 0000000000..2f55293a80 --- /dev/null +++ b/flytectl/pkg/ext/mocks/admin_deleter_ext_interface.go @@ -0,0 +1,116 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + admin "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + mock "github.com/stretchr/testify/mock" + + service "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" +) + +// AdminDeleterExtInterface is an autogenerated mock type for the AdminDeleterExtInterface type +type AdminDeleterExtInterface struct { + mock.Mock +} + +type AdminDeleterExtInterface_AdminServiceClient struct { + *mock.Call +} + +func (_m AdminDeleterExtInterface_AdminServiceClient) Return(_a0 service.AdminServiceClient) *AdminDeleterExtInterface_AdminServiceClient { + return &AdminDeleterExtInterface_AdminServiceClient{Call: _m.Call.Return(_a0)} +} + +func (_m *AdminDeleterExtInterface) OnAdminServiceClient() *AdminDeleterExtInterface_AdminServiceClient { + c := _m.On("AdminServiceClient") + return &AdminDeleterExtInterface_AdminServiceClient{Call: c} +} + +func (_m *AdminDeleterExtInterface) OnAdminServiceClientMatch(matchers ...interface{}) *AdminDeleterExtInterface_AdminServiceClient { + c := _m.On("AdminServiceClient", matchers...) + return &AdminDeleterExtInterface_AdminServiceClient{Call: c} +} + +// AdminServiceClient provides a mock function with given fields: +func (_m *AdminDeleterExtInterface) AdminServiceClient() service.AdminServiceClient { + ret := _m.Called() + + var r0 service.AdminServiceClient + if rf, ok := ret.Get(0).(func() service.AdminServiceClient); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(service.AdminServiceClient) + } + } + + return r0 +} + +type AdminDeleterExtInterface_DeleteProjectDomainAttributes struct { + *mock.Call +} + +func (_m AdminDeleterExtInterface_DeleteProjectDomainAttributes) Return(_a0 error) *AdminDeleterExtInterface_DeleteProjectDomainAttributes { + return &AdminDeleterExtInterface_DeleteProjectDomainAttributes{Call: _m.Call.Return(_a0)} +} + +func (_m *AdminDeleterExtInterface) OnDeleteProjectDomainAttributes(ctx context.Context, project string, domain string, rsType admin.MatchableResource) *AdminDeleterExtInterface_DeleteProjectDomainAttributes { + c := _m.On("DeleteProjectDomainAttributes", ctx, project, domain, rsType) + return &AdminDeleterExtInterface_DeleteProjectDomainAttributes{Call: c} +} + +func (_m *AdminDeleterExtInterface) OnDeleteProjectDomainAttributesMatch(matchers ...interface{}) *AdminDeleterExtInterface_DeleteProjectDomainAttributes { + c := _m.On("DeleteProjectDomainAttributes", matchers...) + return &AdminDeleterExtInterface_DeleteProjectDomainAttributes{Call: c} +} + +// DeleteProjectDomainAttributes provides a mock function with given fields: ctx, project, domain, rsType +func (_m *AdminDeleterExtInterface) DeleteProjectDomainAttributes(ctx context.Context, project string, domain string, rsType admin.MatchableResource) error { + ret := _m.Called(ctx, project, domain, rsType) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, admin.MatchableResource) error); ok { + r0 = rf(ctx, project, domain, rsType) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type AdminDeleterExtInterface_DeleteWorkflowAttributes struct { + *mock.Call +} + +func (_m AdminDeleterExtInterface_DeleteWorkflowAttributes) Return(_a0 error) *AdminDeleterExtInterface_DeleteWorkflowAttributes { + return &AdminDeleterExtInterface_DeleteWorkflowAttributes{Call: _m.Call.Return(_a0)} +} + +func (_m *AdminDeleterExtInterface) OnDeleteWorkflowAttributes(ctx context.Context, project string, domain string, name string, rsType admin.MatchableResource) *AdminDeleterExtInterface_DeleteWorkflowAttributes { + c := _m.On("DeleteWorkflowAttributes", ctx, project, domain, name, rsType) + return &AdminDeleterExtInterface_DeleteWorkflowAttributes{Call: c} +} + +func (_m *AdminDeleterExtInterface) OnDeleteWorkflowAttributesMatch(matchers ...interface{}) *AdminDeleterExtInterface_DeleteWorkflowAttributes { + c := _m.On("DeleteWorkflowAttributes", matchers...) + return &AdminDeleterExtInterface_DeleteWorkflowAttributes{Call: c} +} + +// DeleteWorkflowAttributes provides a mock function with given fields: ctx, project, domain, name, rsType +func (_m *AdminDeleterExtInterface) DeleteWorkflowAttributes(ctx context.Context, project string, domain string, name string, rsType admin.MatchableResource) error { + ret := _m.Called(ctx, project, domain, name, rsType) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, admin.MatchableResource) error); ok { + r0 = rf(ctx, project, domain, name, rsType) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/flytectl/pkg/ext/mocks/admin_fetcher_ext_interface.go b/flytectl/pkg/ext/mocks/admin_fetcher_ext_interface.go index 9f8d7512ff..95d3c99c92 100644 --- a/flytectl/pkg/ext/mocks/admin_fetcher_ext_interface.go +++ b/flytectl/pkg/ext/mocks/admin_fetcher_ext_interface.go @@ -256,6 +256,47 @@ func (_m *AdminFetcherExtInterface) FetchLPVersion(ctx context.Context, name str return r0, r1 } +type AdminFetcherExtInterface_FetchProjectDomainAttributes struct { + *mock.Call +} + +func (_m AdminFetcherExtInterface_FetchProjectDomainAttributes) Return(_a0 *admin.ProjectDomainAttributesGetResponse, _a1 error) *AdminFetcherExtInterface_FetchProjectDomainAttributes { + return &AdminFetcherExtInterface_FetchProjectDomainAttributes{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *AdminFetcherExtInterface) OnFetchProjectDomainAttributes(ctx context.Context, project string, domain string, rsType admin.MatchableResource) *AdminFetcherExtInterface_FetchProjectDomainAttributes { + c := _m.On("FetchProjectDomainAttributes", ctx, project, domain, rsType) + return &AdminFetcherExtInterface_FetchProjectDomainAttributes{Call: c} +} + +func (_m *AdminFetcherExtInterface) OnFetchProjectDomainAttributesMatch(matchers ...interface{}) *AdminFetcherExtInterface_FetchProjectDomainAttributes { + c := _m.On("FetchProjectDomainAttributes", matchers...) + return &AdminFetcherExtInterface_FetchProjectDomainAttributes{Call: c} +} + +// FetchProjectDomainAttributes provides a mock function with given fields: ctx, project, domain, rsType +func (_m *AdminFetcherExtInterface) FetchProjectDomainAttributes(ctx context.Context, project string, domain string, rsType admin.MatchableResource) (*admin.ProjectDomainAttributesGetResponse, error) { + ret := _m.Called(ctx, project, domain, rsType) + + var r0 *admin.ProjectDomainAttributesGetResponse + if rf, ok := ret.Get(0).(func(context.Context, string, string, admin.MatchableResource) *admin.ProjectDomainAttributesGetResponse); ok { + r0 = rf(ctx, project, domain, rsType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*admin.ProjectDomainAttributesGetResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string, admin.MatchableResource) error); ok { + r1 = rf(ctx, project, domain, rsType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type AdminFetcherExtInterface_FetchTaskLatestVersion struct { *mock.Call } @@ -337,3 +378,44 @@ func (_m *AdminFetcherExtInterface) FetchTaskVersion(ctx context.Context, name s return r0, r1 } + +type AdminFetcherExtInterface_FetchWorkflowAttributes struct { + *mock.Call +} + +func (_m AdminFetcherExtInterface_FetchWorkflowAttributes) Return(_a0 *admin.WorkflowAttributesGetResponse, _a1 error) *AdminFetcherExtInterface_FetchWorkflowAttributes { + return &AdminFetcherExtInterface_FetchWorkflowAttributes{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *AdminFetcherExtInterface) OnFetchWorkflowAttributes(ctx context.Context, project string, domain string, name string, rsType admin.MatchableResource) *AdminFetcherExtInterface_FetchWorkflowAttributes { + c := _m.On("FetchWorkflowAttributes", ctx, project, domain, name, rsType) + return &AdminFetcherExtInterface_FetchWorkflowAttributes{Call: c} +} + +func (_m *AdminFetcherExtInterface) OnFetchWorkflowAttributesMatch(matchers ...interface{}) *AdminFetcherExtInterface_FetchWorkflowAttributes { + c := _m.On("FetchWorkflowAttributes", matchers...) + return &AdminFetcherExtInterface_FetchWorkflowAttributes{Call: c} +} + +// FetchWorkflowAttributes provides a mock function with given fields: ctx, project, domain, name, rsType +func (_m *AdminFetcherExtInterface) FetchWorkflowAttributes(ctx context.Context, project string, domain string, name string, rsType admin.MatchableResource) (*admin.WorkflowAttributesGetResponse, error) { + ret := _m.Called(ctx, project, domain, name, rsType) + + var r0 *admin.WorkflowAttributesGetResponse + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, admin.MatchableResource) *admin.WorkflowAttributesGetResponse); ok { + r0 = rf(ctx, project, domain, name, rsType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*admin.WorkflowAttributesGetResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, admin.MatchableResource) error); ok { + r1 = rf(ctx, project, domain, name, rsType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/flytectl/pkg/ext/mocks/admin_updater_ext_interface.go b/flytectl/pkg/ext/mocks/admin_updater_ext_interface.go new file mode 100644 index 0000000000..d944ff5a82 --- /dev/null +++ b/flytectl/pkg/ext/mocks/admin_updater_ext_interface.go @@ -0,0 +1,116 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + admin "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + + mock "github.com/stretchr/testify/mock" + + service "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" +) + +// AdminUpdaterExtInterface is an autogenerated mock type for the AdminUpdaterExtInterface type +type AdminUpdaterExtInterface struct { + mock.Mock +} + +type AdminUpdaterExtInterface_AdminServiceClient struct { + *mock.Call +} + +func (_m AdminUpdaterExtInterface_AdminServiceClient) Return(_a0 service.AdminServiceClient) *AdminUpdaterExtInterface_AdminServiceClient { + return &AdminUpdaterExtInterface_AdminServiceClient{Call: _m.Call.Return(_a0)} +} + +func (_m *AdminUpdaterExtInterface) OnAdminServiceClient() *AdminUpdaterExtInterface_AdminServiceClient { + c := _m.On("AdminServiceClient") + return &AdminUpdaterExtInterface_AdminServiceClient{Call: c} +} + +func (_m *AdminUpdaterExtInterface) OnAdminServiceClientMatch(matchers ...interface{}) *AdminUpdaterExtInterface_AdminServiceClient { + c := _m.On("AdminServiceClient", matchers...) + return &AdminUpdaterExtInterface_AdminServiceClient{Call: c} +} + +// AdminServiceClient provides a mock function with given fields: +func (_m *AdminUpdaterExtInterface) AdminServiceClient() service.AdminServiceClient { + ret := _m.Called() + + var r0 service.AdminServiceClient + if rf, ok := ret.Get(0).(func() service.AdminServiceClient); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(service.AdminServiceClient) + } + } + + return r0 +} + +type AdminUpdaterExtInterface_UpdateProjectDomainAttributes struct { + *mock.Call +} + +func (_m AdminUpdaterExtInterface_UpdateProjectDomainAttributes) Return(_a0 error) *AdminUpdaterExtInterface_UpdateProjectDomainAttributes { + return &AdminUpdaterExtInterface_UpdateProjectDomainAttributes{Call: _m.Call.Return(_a0)} +} + +func (_m *AdminUpdaterExtInterface) OnUpdateProjectDomainAttributes(ctx context.Context, project string, domain string, matchingAttr *admin.MatchingAttributes) *AdminUpdaterExtInterface_UpdateProjectDomainAttributes { + c := _m.On("UpdateProjectDomainAttributes", ctx, project, domain, matchingAttr) + return &AdminUpdaterExtInterface_UpdateProjectDomainAttributes{Call: c} +} + +func (_m *AdminUpdaterExtInterface) OnUpdateProjectDomainAttributesMatch(matchers ...interface{}) *AdminUpdaterExtInterface_UpdateProjectDomainAttributes { + c := _m.On("UpdateProjectDomainAttributes", matchers...) + return &AdminUpdaterExtInterface_UpdateProjectDomainAttributes{Call: c} +} + +// UpdateProjectDomainAttributes provides a mock function with given fields: ctx, project, domain, matchingAttr +func (_m *AdminUpdaterExtInterface) UpdateProjectDomainAttributes(ctx context.Context, project string, domain string, matchingAttr *admin.MatchingAttributes) error { + ret := _m.Called(ctx, project, domain, matchingAttr) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, *admin.MatchingAttributes) error); ok { + r0 = rf(ctx, project, domain, matchingAttr) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type AdminUpdaterExtInterface_UpdateWorkflowAttributes struct { + *mock.Call +} + +func (_m AdminUpdaterExtInterface_UpdateWorkflowAttributes) Return(_a0 error) *AdminUpdaterExtInterface_UpdateWorkflowAttributes { + return &AdminUpdaterExtInterface_UpdateWorkflowAttributes{Call: _m.Call.Return(_a0)} +} + +func (_m *AdminUpdaterExtInterface) OnUpdateWorkflowAttributes(ctx context.Context, project string, domain string, name string, matchingAttr *admin.MatchingAttributes) *AdminUpdaterExtInterface_UpdateWorkflowAttributes { + c := _m.On("UpdateWorkflowAttributes", ctx, project, domain, name, matchingAttr) + return &AdminUpdaterExtInterface_UpdateWorkflowAttributes{Call: c} +} + +func (_m *AdminUpdaterExtInterface) OnUpdateWorkflowAttributesMatch(matchers ...interface{}) *AdminUpdaterExtInterface_UpdateWorkflowAttributes { + c := _m.On("UpdateWorkflowAttributes", matchers...) + return &AdminUpdaterExtInterface_UpdateWorkflowAttributes{Call: c} +} + +// UpdateWorkflowAttributes provides a mock function with given fields: ctx, project, domain, name, matchingAttr +func (_m *AdminUpdaterExtInterface) UpdateWorkflowAttributes(ctx context.Context, project string, domain string, name string, matchingAttr *admin.MatchingAttributes) error { + ret := _m.Called(ctx, project, domain, name, matchingAttr) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, *admin.MatchingAttributes) error); ok { + r0 = rf(ctx, project, domain, name, matchingAttr) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/flytectl/pkg/ext/task_fetcher_test.go b/flytectl/pkg/ext/task_fetcher_test.go new file mode 100644 index 0000000000..7f7476b2c0 --- /dev/null +++ b/flytectl/pkg/ext/task_fetcher_test.go @@ -0,0 +1,125 @@ +package ext + +import ( + "context" + "fmt" + "testing" + + "github.com/flyteorg/flyteidl/clients/go/admin/mocks" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + adminFetcherExt AdminFetcherExtClient + adminClient *mocks.AdminServiceClient + ctx context.Context + taskListResponse *admin.TaskList +) + +func getTaskFetcherSetup() { + ctx = context.Background() + adminClient = new(mocks.AdminServiceClient) + adminFetcherExt = AdminFetcherExtClient{AdminClient: adminClient} + + sortedListLiteralType := core.Variable{ + Type: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_INTEGER, + }, + }, + }, + }, + } + variableMap := map[string]*core.Variable{ + "sorted_list1": &sortedListLiteralType, + "sorted_list2": &sortedListLiteralType, + } + + task1 := &admin.Task{ + Id: &core.Identifier{ + Name: "task1", + Version: "v1", + }, + Closure: &admin.TaskClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 0, Nanos: 0}, + CompiledTask: &core.CompiledTask{ + Template: &core.TaskTemplate{ + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: variableMap, + }, + }, + }, + }, + }, + } + + task2 := &admin.Task{ + Id: &core.Identifier{ + Name: "task1", + Version: "v2", + }, + Closure: &admin.TaskClosure{ + CreatedAt: ×tamppb.Timestamp{Seconds: 1, Nanos: 0}, + CompiledTask: &core.CompiledTask{ + Template: &core.TaskTemplate{ + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: variableMap, + }, + }, + }, + }, + }, + } + + tasks := []*admin.Task{task2, task1} + + taskListResponse = &admin.TaskList{ + Tasks: tasks, + } +} + +func TestFetchAllVerOfTask(t *testing.T) { + getTaskFetcherSetup() + adminClient.OnListTasksMatch(mock.Anything, mock.Anything).Return(taskListResponse, nil) + _, err := adminFetcherExt.FetchAllVerOfTask(ctx, "taskName", "project", "domain") + assert.Nil(t, err) +} + +func TestFetchAllVerOfTaskError(t *testing.T) { + getTaskFetcherSetup() + adminClient.OnListTasksMatch(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed")) + _, err := adminFetcherExt.FetchAllVerOfTask(ctx, "taskName", "project", "domain") + assert.Equal(t, fmt.Errorf("failed"), err) +} + +func TestFetchAllVerOfTaskEmptyResponse(t *testing.T) { + taskListResponse := &admin.TaskList{} + getTaskFetcherSetup() + adminClient.OnListTasksMatch(mock.Anything, mock.Anything).Return(taskListResponse, nil) + _, err := adminFetcherExt.FetchAllVerOfTask(ctx, "taskName", "project", "domain") + assert.Equal(t, fmt.Errorf("no tasks retrieved for taskName"), err) +} + +func TestFetchTaskLatestVersion(t *testing.T) { + getTaskFetcherSetup() + adminClient.OnListTasksMatch(mock.Anything, mock.Anything).Return(taskListResponse, nil) + _, err := adminFetcherExt.FetchTaskLatestVersion(ctx, "taskName", "project", "domain") + assert.Nil(t, err) +} + +func TestFetchTaskLatestVersionError(t *testing.T) { + taskListResponse := &admin.TaskList{} + getTaskFetcherSetup() + adminClient.OnListTasksMatch(mock.Anything, mock.Anything).Return(taskListResponse, nil) + _, err := adminFetcherExt.FetchTaskLatestVersion(ctx, "taskName", "project", "domain") + assert.Equal(t, fmt.Errorf("no tasks retrieved for taskName"), err) +} diff --git a/flytectl/pkg/ext/updater.go b/flytectl/pkg/ext/updater.go new file mode 100644 index 0000000000..ddf9f5841e --- /dev/null +++ b/flytectl/pkg/ext/updater.go @@ -0,0 +1,33 @@ +package ext + +import ( + "context" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" +) + +//go:generate mockery -all -case=underscore + +// AdminUpdaterExtInterface Interface for exposing the update capabilities from the admin +type AdminUpdaterExtInterface interface { + AdminServiceClient() service.AdminServiceClient + + // UpdateWorkflowAttributes updates workflow attributes within a project, domain for a particular matchable resource + UpdateWorkflowAttributes(ctx context.Context, project, domain, name string, matchingAttr *admin.MatchingAttributes) error + + // UpdateProjectDomainAttributes updates project domain attributes for a particular matchable resource + UpdateProjectDomainAttributes(ctx context.Context, project, domain string, matchingAttr *admin.MatchingAttributes) error +} + +// AdminUpdaterExtClient is used for interacting with extended features used for updating data in admin service +type AdminUpdaterExtClient struct { + AdminClient service.AdminServiceClient +} + +func (a *AdminUpdaterExtClient) AdminServiceClient() service.AdminServiceClient { + if a == nil { + return nil + } + return a.AdminClient +}