From 7569de6dd871cd7b498875bab036ab7a54499b8b Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Fri, 30 Aug 2024 23:55:14 +0800 Subject: [PATCH 01/11] [Docs] Echo Task (#5707) * add echo tasks Signed-off-by: Future-Outlier * update nikki's advice Signed-off-by: Future-Outlier Co-authored-by: nikki everett --------- Signed-off-by: Future-Outlier Co-authored-by: nikki everett --- .../advanced_composition/conditionals.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/user_guide/advanced_composition/conditionals.md b/docs/user_guide/advanced_composition/conditionals.md index bbaf3dc27ae..3afca887721 100644 --- a/docs/user_guide/advanced_composition/conditionals.md +++ b/docs/user_guide/advanced_composition/conditionals.md @@ -124,6 +124,25 @@ You can run the workflow locally as follows: :lines: 181-188 ``` +## Running a noop task in a conditional + +In some cases, you may want to skip the execution of a conditional workflow if a certain condition is not met. +You can achieve this by using the `echo` task, which simply returns the input value. + +:::{note} +To enable the echo plugin in the backend, add the plugin to Flyte's configuration file. +```yaml +task-plugins: + enabled-plugins: + - echo +``` +::: + +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/89bf7bc7788802097904c5f9ffb75ba70ef980a6/examples/advanced_composition/advanced_composition/conditional.py +:caption: advanced_composition/conditional.py +:lines: 197-209 +``` + ## Run the example on the Flyte cluster To run the provided workflows on the Flyte cluster, use the following commands: @@ -170,4 +189,10 @@ pyflyte run --remote \ consume_task_output --radius 0.4 --seed 7 ``` +``` +pyflyte run --remote \ + https://raw.githubusercontent.com/flyteorg/flytesnacks/89bf7bc7788802097904c5f9ffb75ba70ef980a6/examples/advanced_composition/advanced_composition/conditional.py \ + noop_in_conditional --radius 0.4 --seed 5 +``` + [flytesnacks]: https://github.com/flyteorg/flytesnacks/tree/master/examples/advanced_composition From 2ed24086f3d5eac5b779c1755642d4ac1b1d0f76 Mon Sep 17 00:00:00 2001 From: Wei-Yu Kao <115421902+wayner0628@users.noreply.github.com> Date: Sat, 31 Aug 2024 01:49:42 +0800 Subject: [PATCH 02/11] Improve execution name readability (#5637) Signed-off-by: wayner0628 Signed-off-by: Kevin Su Co-authored-by: Kevin Su --- flyteadmin/go.mod | 1 + flyteadmin/go.sum | 2 + .../async/schedule/aws/workflow_executor.go | 5 +- flyteadmin/pkg/common/executions.go | 13 ---- flyteadmin/pkg/common/executions_test.go | 23 ------- .../pkg/common/naming/execution_name.go | 30 +++++++++ .../pkg/common/naming/execution_name_test.go | 64 +++++++++++++++++++ flyteadmin/pkg/manager/impl/util/shared.go | 3 +- .../pkg/manager/impl/util/shared_test.go | 4 +- .../interfaces/application_configuration.go | 3 +- .../scheduler/executor/executor_impl.go | 21 ++---- go.mod | 1 + go.sum | 2 + 13 files changed, 112 insertions(+), 60 deletions(-) delete mode 100644 flyteadmin/pkg/common/executions_test.go create mode 100644 flyteadmin/pkg/common/naming/execution_name.go create mode 100644 flyteadmin/pkg/common/naming/execution_name_test.go diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index ac743842505..b9eba5b83a3 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -48,6 +48,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/wI2L/jsondiff v0.5.0 + github.com/wolfeidau/humanhash v1.1.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 golang.org/x/oauth2 v0.16.0 diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index dba9da2e86f..049add4bbce 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1297,6 +1297,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wolfeidau/humanhash v1.1.0 h1:06KgtyyABJGBbrfMONrW7S+b5TTYVyrNB/jss5n7F3E= +github.com/wolfeidau/humanhash v1.1.0/go.mod h1:jkpynR1bfyfkmKEQudIC0osWKynFAoayRjzH9OJdVIg= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= diff --git a/flyteadmin/pkg/async/schedule/aws/workflow_executor.go b/flyteadmin/pkg/async/schedule/aws/workflow_executor.go index 918402836be..13d2041f9da 100644 --- a/flyteadmin/pkg/async/schedule/aws/workflow_executor.go +++ b/flyteadmin/pkg/async/schedule/aws/workflow_executor.go @@ -15,7 +15,7 @@ import ( "github.com/flyteorg/flyte/flyteadmin/pkg/async" scheduleInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/async/schedule/interfaces" - "github.com/flyteorg/flyte/flyteadmin/pkg/common" + "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/interfaces" runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" @@ -129,7 +129,7 @@ func generateExecutionName(launchPlan admin.LaunchPlan, kickoffTime time.Time) s Name: launchPlan.Id.Name, }) randomSeed := kickoffTime.UnixNano() + int64(hashedIdentifier) - return common.GetExecutionName(randomSeed) + return naming.GetExecutionName(randomSeed) } func (e *workflowExecutor) formulateExecutionCreateRequest( @@ -207,7 +207,6 @@ func (e *workflowExecutor) run() error { continue } executionRequest := e.formulateExecutionCreateRequest(launchPlan, scheduledWorkflowExecutionRequest.KickoffTime) - ctx = contextutils.WithWorkflowID(ctx, fmt.Sprintf(workflowIdentifierFmt, executionRequest.Project, executionRequest.Domain, executionRequest.Name)) err = e.resolveKickoffTimeArg(scheduledWorkflowExecutionRequest, launchPlan, &executionRequest) diff --git a/flyteadmin/pkg/common/executions.go b/flyteadmin/pkg/common/executions.go index fbb5bdd6bd9..4ac1ec73006 100644 --- a/flyteadmin/pkg/common/executions.go +++ b/flyteadmin/pkg/common/executions.go @@ -1,22 +1,9 @@ package common import ( - "fmt" - - "k8s.io/apimachinery/pkg/util/rand" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) -const ExecutionIDLength = 20 -const ExecutionStringFormat = "a%s" - -/* #nosec */ -func GetExecutionName(seed int64) string { - rand.Seed(seed) - return fmt.Sprintf(ExecutionStringFormat, rand.String(ExecutionIDLength-1)) -} - var terminalExecutionPhases = map[core.WorkflowExecution_Phase]bool{ core.WorkflowExecution_SUCCEEDED: true, core.WorkflowExecution_FAILED: true, diff --git a/flyteadmin/pkg/common/executions_test.go b/flyteadmin/pkg/common/executions_test.go deleted file mode 100644 index 628abd6e9da..00000000000 --- a/flyteadmin/pkg/common/executions_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package common - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const AllowedExecutionIDStartCharStr = "abcdefghijklmnopqrstuvwxyz" -const AllowedExecutionIDStr = "abcdefghijklmnopqrstuvwxyz1234567890" - -var AllowedExecutionIDStartChars = []rune(AllowedExecutionIDStartCharStr) -var AllowedExecutionIDChars = []rune(AllowedExecutionIDStr) - -func TestGetExecutionName(t *testing.T) { - randString := GetExecutionName(time.Now().UnixNano()) - assert.Len(t, randString, ExecutionIDLength) - assert.Contains(t, AllowedExecutionIDStartChars, rune(randString[0])) - for i := 1; i < len(randString); i++ { - assert.Contains(t, AllowedExecutionIDChars, rune(randString[i])) - } -} diff --git a/flyteadmin/pkg/common/naming/execution_name.go b/flyteadmin/pkg/common/naming/execution_name.go new file mode 100644 index 00000000000..01aa3fe8b6a --- /dev/null +++ b/flyteadmin/pkg/common/naming/execution_name.go @@ -0,0 +1,30 @@ +package naming + +import ( + "fmt" + + "github.com/wolfeidau/humanhash" + "k8s.io/apimachinery/pkg/util/rand" + + "github.com/flyteorg/flyte/flyteadmin/pkg/runtime" + runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" +) + +const ExecutionIDLength = 20 +const ExecutionIDLengthLimit = 63 +const ExecutionStringFormat = "a%s" + +var configProvider runtimeInterfaces.ApplicationConfiguration = runtime.NewApplicationConfigurationProvider() + +/* #nosec */ +func GetExecutionName(seed int64) string { + rand.Seed(seed) + config := configProvider.GetTopLevelConfig() + if config.FeatureGates.EnableFriendlyNames { + hashKey := []byte(rand.String(ExecutionIDLength)) + // Ignoring the error as it's guaranteed hash key longer than result in this context. + result, _ := humanhash.Humanize(hashKey, 4) + return result + } + return fmt.Sprintf(ExecutionStringFormat, rand.String(ExecutionIDLength-1)) +} diff --git a/flyteadmin/pkg/common/naming/execution_name_test.go b/flyteadmin/pkg/common/naming/execution_name_test.go new file mode 100644 index 00000000000..22729dbb9ba --- /dev/null +++ b/flyteadmin/pkg/common/naming/execution_name_test.go @@ -0,0 +1,64 @@ +package naming + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" + runtimeMocks "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/mocks" +) + +const AllowedExecutionIDAlphabetStr = "abcdefghijklmnopqrstuvwxyz" +const AllowedExecutionIDAlphanumericStr = "abcdefghijklmnopqrstuvwxyz1234567890" +const AllowedExecutionIDFriendlyNameStr = "abcdefghijklmnopqrstuvwxyz-" + +var AllowedExecutionIDAlphabets = []rune(AllowedExecutionIDAlphabetStr) +var AllowedExecutionIDAlphanumerics = []rune(AllowedExecutionIDAlphanumericStr) +var AllowedExecutionIDFriendlyNameChars = []rune(AllowedExecutionIDFriendlyNameStr) + +func TestGetExecutionName(t *testing.T) { + originalConfigProvider := configProvider + defer func() { configProvider = originalConfigProvider }() + + mockConfigProvider := &runtimeMocks.MockApplicationProvider{} + configProvider = mockConfigProvider + + t.Run("general name", func(t *testing.T) { + appConfig := runtimeInterfaces.ApplicationConfig{ + FeatureGates: runtimeInterfaces.FeatureGates{ + EnableFriendlyNames: false, + }, + } + mockConfigProvider.SetTopLevelConfig(appConfig) + + randString := GetExecutionName(time.Now().UnixNano()) + assert.Len(t, randString, ExecutionIDLength) + assert.Contains(t, AllowedExecutionIDAlphabets, rune(randString[0])) + for i := 1; i < len(randString); i++ { + assert.Contains(t, AllowedExecutionIDAlphanumerics, rune(randString[i])) + } + }) + + t.Run("friendly name", func(t *testing.T) { + appConfig := runtimeInterfaces.ApplicationConfig{ + FeatureGates: runtimeInterfaces.FeatureGates{ + EnableFriendlyNames: true, + }, + } + mockConfigProvider.SetTopLevelConfig(appConfig) + + randString := GetExecutionName(time.Now().UnixNano()) + assert.LessOrEqual(t, len(randString), ExecutionIDLengthLimit) + for i := 0; i < len(randString); i++ { + assert.Contains(t, AllowedExecutionIDFriendlyNameChars, rune(randString[i])) + } + hyphenCount := strings.Count(randString, "-") + assert.Equal(t, 3, hyphenCount, "FriendlyName should contain exactly three hyphens") + words := strings.Split(randString, "-") + assert.Equal(t, 4, len(words), "FriendlyName should be split into exactly four words") + }) + +} diff --git a/flyteadmin/pkg/manager/impl/util/shared.go b/flyteadmin/pkg/manager/impl/util/shared.go index 24c97f416a3..b1395697ef0 100644 --- a/flyteadmin/pkg/manager/impl/util/shared.go +++ b/flyteadmin/pkg/manager/impl/util/shared.go @@ -8,6 +8,7 @@ import ( "google.golang.org/grpc/codes" "github.com/flyteorg/flyte/flyteadmin/pkg/common" + "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/impl/shared" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/impl/validation" @@ -25,7 +26,7 @@ func GetExecutionName(request admin.ExecutionCreateRequest) string { if request.Name != "" { return request.Name } - return common.GetExecutionName(time.Now().UnixNano()) + return naming.GetExecutionName(time.Now().UnixNano()) } func GetTask(ctx context.Context, repo repoInterfaces.Repository, identifier core.Identifier) ( diff --git a/flyteadmin/pkg/manager/impl/util/shared_test.go b/flyteadmin/pkg/manager/impl/util/shared_test.go index 21a78997c30..75759485db8 100644 --- a/flyteadmin/pkg/manager/impl/util/shared_test.go +++ b/flyteadmin/pkg/manager/impl/util/shared_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/assert" "google.golang.org/grpc/codes" - "github.com/flyteorg/flyte/flyteadmin/pkg/common" commonMocks "github.com/flyteorg/flyte/flyteadmin/pkg/common/mocks" + "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" flyteAdminErrors "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/impl/testutils" managerInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/manager/interfaces" @@ -42,7 +42,7 @@ func TestPopulateExecutionID(t *testing.T) { Domain: "domain", }) assert.NotEmpty(t, name) - assert.Len(t, name, common.ExecutionIDLength) + assert.Len(t, name, naming.ExecutionIDLength) } func TestPopulateExecutionID_ExistingName(t *testing.T) { diff --git a/flyteadmin/pkg/runtime/interfaces/application_configuration.go b/flyteadmin/pkg/runtime/interfaces/application_configuration.go index ca6dc609231..8be59abe14f 100644 --- a/flyteadmin/pkg/runtime/interfaces/application_configuration.go +++ b/flyteadmin/pkg/runtime/interfaces/application_configuration.go @@ -49,7 +49,8 @@ type PostgresConfig struct { } type FeatureGates struct { - EnableArtifacts bool `json:"enableArtifacts" pflag:",Enable artifacts feature."` + EnableArtifacts bool `json:"enableArtifacts" pflag:",Enable artifacts feature."` + EnableFriendlyNames bool `json:"enableFriendlyNames" pflag:",Enable generation of friendly execution names feature."` } // ApplicationConfig is the base configuration to start admin diff --git a/flyteadmin/scheduler/executor/executor_impl.go b/flyteadmin/scheduler/executor/executor_impl.go index dffb98e1b6a..f3fd86c6cf0 100644 --- a/flyteadmin/scheduler/executor/executor_impl.go +++ b/flyteadmin/scheduler/executor/executor_impl.go @@ -2,7 +2,6 @@ package executor import ( "context" - "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -12,7 +11,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" - "github.com/flyteorg/flyte/flyteadmin/scheduler/identifier" + "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" "github.com/flyteorg/flyte/flyteadmin/scheduler/repositories/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" @@ -53,23 +52,11 @@ func (w *executor) Execute(ctx context.Context, scheduledTime time.Time, s model } } - // Making the identifier deterministic using the hash of the identifier and scheduled time - executionIdentifier, err := identifier.GetExecutionIdentifier(ctx, core.Identifier{ - Project: s.Project, - Domain: s.Domain, - Name: s.Name, - Version: s.Version, - }, scheduledTime) - - if err != nil { - logger.Errorf(ctx, "failed to generate execution identifier for schedule %+v due to %v", s, err) - return err - } - + executionName := naming.GetExecutionName(time.Now().UnixNano()) executionRequest := &admin.ExecutionCreateRequest{ Project: s.Project, Domain: s.Domain, - Name: "f" + strings.ReplaceAll(executionIdentifier.String(), "-", "")[:19], + Name: executionName, Spec: &admin.ExecutionSpec{ LaunchPlan: &core.Identifier{ ResourceType: core.ResourceType_LAUNCH_PLAN, @@ -97,7 +84,7 @@ func (w *executor) Execute(ctx context.Context, scheduledTime time.Time, s model // Do maximum of 30 retries on failures with constant backoff factor opts := wait.Backoff{Duration: 3000, Factor: 2.0, Steps: 30} - err = retry.OnError(opts, + err := retry.OnError(opts, func(err error) bool { // For idempotent behavior ignore the AlreadyExists error which happens if we try to schedule a launchplan // for execution at the same time which is already available in admin. diff --git a/go.mod b/go.mod index 3a7098d3c0f..8fd55ed61a5 100644 --- a/go.mod +++ b/go.mod @@ -178,6 +178,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/wI2L/jsondiff v0.5.0 // indirect + github.com/wolfeidau/humanhash v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index 05db1b9c1c7..ae60f268004 100644 --- a/go.sum +++ b/go.sum @@ -1333,6 +1333,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= +github.com/wolfeidau/humanhash v1.1.0 h1:06KgtyyABJGBbrfMONrW7S+b5TTYVyrNB/jss5n7F3E= +github.com/wolfeidau/humanhash v1.1.0/go.mod h1:jkpynR1bfyfkmKEQudIC0osWKynFAoayRjzH9OJdVIg= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= From 7a917995ca4000bc4767d40d61c6bbb65e427f65 Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Fri, 30 Aug 2024 12:56:32 -0700 Subject: [PATCH 03/11] Configure imagePullPolicy to be Always pull on flyte sandbox environment (#5709) --- flyte-single-binary-local.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/flyte-single-binary-local.yaml b/flyte-single-binary-local.yaml index 4cb63e8d4d9..e7350212a2f 100644 --- a/flyte-single-binary-local.yaml +++ b/flyte-single-binary-local.yaml @@ -57,6 +57,7 @@ plugins: - FLYTE_AWS_ENDPOINT: http://flyte-sandbox-minio.flyte:9000 - FLYTE_AWS_ACCESS_KEY_ID: minio - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + image-pull-policy: Always # Helps in better iteration of flytekit changes k8s-array: logs: config: From 79a1e463e3656aaf8d9dd73a574e7bfe3f85cb78 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sun, 1 Sep 2024 18:30:33 -0700 Subject: [PATCH 04/11] should not set echo plugin as default (#5713) Signed-off-by: Kevin Su --- flyteplugins/go/tasks/plugins/testing/echo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyteplugins/go/tasks/plugins/testing/echo.go b/flyteplugins/go/tasks/plugins/testing/echo.go index 7c55d3862f5..09c4dc53b16 100644 --- a/flyteplugins/go/tasks/plugins/testing/echo.go +++ b/flyteplugins/go/tasks/plugins/testing/echo.go @@ -181,7 +181,7 @@ func init() { taskStartTimes: make(map[string]time.Time), }, nil }, - IsDefault: true, + IsDefault: false, }, ) } From 96c799c5ccad301d42e8c4266cbaeab217692016 Mon Sep 17 00:00:00 2001 From: Wei-Yu Kao <115421902+wayner0628@users.noreply.github.com> Date: Mon, 2 Sep 2024 16:20:06 +0800 Subject: [PATCH 05/11] Move default execution name generation to flyteadmin (#5714) Signed-off-by: wayner0628 --- flytectl/cmd/create/execution_util.go | 5 ----- flytectl/go.mod | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/flytectl/cmd/create/execution_util.go b/flytectl/cmd/create/execution_util.go index 7ebb757a291..bcb5c5639f3 100644 --- a/flytectl/cmd/create/execution_util.go +++ b/flytectl/cmd/create/execution_util.go @@ -4,14 +4,12 @@ import ( "context" "fmt" "io/ioutil" - "strings" cmdCore "github.com/flyteorg/flyte/flytectl/cmd/core" cmdGet "github.com/flyteorg/flyte/flytectl/cmd/get" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/logger" - "github.com/google/uuid" "sigs.k8s.io/yaml" ) @@ -149,9 +147,6 @@ func recoverExecution(ctx context.Context, executionName string, project string, func createExecutionRequest(ID *core.Identifier, inputs *core.LiteralMap, envs *admin.Envs, securityContext *core.SecurityContext, authRole *admin.AuthRole, targetExecName string, targetExecutionCluster string) *admin.ExecutionCreateRequest { - if len(targetExecName) == 0 { - targetExecName = "f" + strings.ReplaceAll(uuid.New().String(), "-", "")[:19] - } var clusterAssignment *admin.ClusterAssignment if executionConfig.ClusterPool != "" { clusterAssignment = &admin.ClusterAssignment{ClusterPoolName: executionConfig.ClusterPool} diff --git a/flytectl/go.mod b/flytectl/go.mod index 9e4baeea63f..0298ad38f0d 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -21,7 +21,6 @@ require ( github.com/go-ozzo/ozzo-validation/v4 v4.3.0 github.com/golang/protobuf v1.5.3 github.com/google/go-github/v42 v42.0.0 - github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.3.0 github.com/hexops/gotextdiff v1.0.3 github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 @@ -102,6 +101,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect From 6add5f86df5028eaaf31ab6812e7f756bc207753 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Mon, 2 Sep 2024 21:10:05 -0700 Subject: [PATCH 06/11] Update helm/docs per changes in supported task discovery (#5694) * Update helm/docs per changes in supported task discovery Signed-off-by: Jason Parraga * Cleanup flyte-binary Signed-off-by: Jason Parraga * fixes Signed-off-by: Jason Parraga --------- Signed-off-by: Jason Parraga --- charts/flyte-binary/README.md | 1 - charts/flyte-binary/values.yaml | 2 -- charts/flyte-core/README.md | 4 ++-- charts/flyte-core/values.yaml | 8 +++----- .../manifests/complete-agent.yaml | 8 +++----- docker/sandbox-bundled/manifests/complete.yaml | 4 ++-- docker/sandbox-bundled/manifests/dev.yaml | 4 ++-- docs/deployment/agents/airflow.rst | 11 +---------- docs/deployment/agents/bigquery.rst | 11 +---------- docs/deployment/agents/chatgpt.rst | 6 +----- docs/deployment/agents/databricks.rst | 18 +++--------------- docs/deployment/agents/mmcloud.rst | 2 -- docs/deployment/agents/openai_batch.rst | 12 ++---------- docs/deployment/agents/sagemaker_inference.rst | 14 ++------------ docs/deployment/agents/sensor.rst | 11 +---------- docs/deployment/agents/snowflake.rst | 11 +---------- .../generated/flytepropeller_config.rst | 3 --- docs/flyte_agents/developing_agents.md | 6 +----- 18 files changed, 25 insertions(+), 111 deletions(-) diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md index cc39b38cdc3..43e0a714371 100644 --- a/charts/flyte-binary/README.md +++ b/charts/flyte-binary/README.md @@ -25,7 +25,6 @@ Chart for basic single Flyte executable deployment | configuration.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | | | configuration.agentService.defaultAgent.insecure | bool | `true` | | | configuration.agentService.defaultAgent.timeouts.GetTask | string | `"10s"` | | -| configuration.agentService.supportedTaskTypes[0] | string | `"default_task"` | | | configuration.annotations | object | `{}` | | | configuration.auth.authorizedUris | list | `[]` | | | configuration.auth.clientSecretsExternalSecretRef | string | `""` | | diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml index 0f7261d86e7..3f87556c3ed 100644 --- a/charts/flyte-binary/values.yaml +++ b/charts/flyte-binary/values.yaml @@ -168,8 +168,6 @@ configuration: timeouts: GetTask: 10s defaultTimeout: 10s - supportedTaskTypes: - - default_task # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte # If set, no Flyte configuration will be generated by this chart externalConfigMap: "" diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 673ba7b6ef5..ba11205264f 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -195,11 +195,11 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteagent.enabled | bool | `false` | | -| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":["sensor"]}` | Agent service configuration for propeller. | +| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":[]}` | Agent service configuration for propeller. | | flyteagent.plugin_config.plugins.agent-service.defaultAgent | object | `{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true}` | The default agent service to use for plugin tasks. | | flyteagent.plugin_config.plugins.agent-service.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | The agent service endpoint propeller should connect to. | | flyteagent.plugin_config.plugins.agent-service.defaultAgent.insecure | bool | `true` | Whether the connection from propeller to the agent service should use TLS. | -| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `["sensor"]` | The task types supported by the default agent. | +| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `[]` | The task types supported by the default agent. As of #5460 these are discovered automatically and don't need to be configured. | | flyteagent.podLabels | object | `{}` | Labels for flyteagent pods | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.enabled | bool | `true` | | diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 93c0d9b3894..7442222e9bc 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -287,11 +287,9 @@ flyteagent: endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000" # -- Whether the connection from propeller to the agent service should use TLS. insecure: true - # -- The task types supported by the default agent. - supportedTaskTypes: - - sensor - # -- Uncomment to enable task type that uses Flyte Agent - # - bigquery_query_job_task + # -- The task types supported by the default agent. As of #5460 these are discovered automatically and don't + # need to be configured. + supportedTaskTypes: [] # -- Labels for flyteagent pods podLabels: {} diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 78a678ae34f..1279b64c601 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -485,8 +485,6 @@ data: insecure: true timeouts: GetTask: 10s - supportedTaskTypes: - - default_task 002-database.yaml: | database: postgres: @@ -818,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: cWlOc1c1bnl5ZGI3YTlzSw== + haSharedSecret: WVl4Y1pFNm1JTkxOMjJpZQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1249,7 +1247,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 6e8a4cc6177037f26cee65d09c37c010437ea3f0989a2a2dfef380fed9f468c2 + checksum/configuration: f746817691b502fb50f04992e148847f89c0a32d7df822dbda7f1f5fdf84f420 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1415,7 +1413,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 7f8247a0b84f43018fdf11a598132b8a67ed9fde6573ffce801b725a6f955012 + checksum/secret: f0211dde23276e14b7254edd85e3afd32044562e409b56d79b27202e23dc224c labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 5d46b89edf6..e9e3007dc9b 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -798,7 +798,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: UUxqaW5SeGlBbFNlQzVoag== + haSharedSecret: Q3BDekMwb0JHM1pxaXAycg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1362,7 +1362,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: bea0c8f293b54e309a353e0e8563e709ad817d372d2b1dce1114188693aa3f12 + checksum/secret: 65e52b61d9d5af16c4217a1ea9f625d8d18e3251ede28f012cc14ef665412685 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 917645af33f..d0983c6f0d6 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: ZmdJNWs5RUg4cWNVTVBzRw== + haSharedSecret: RU9mMGpZWGNCcnp4cEJ3bg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: a896f2c43dff6c05c154b51e4c9ec21c9e2f03ecaf4c1fed045d84523219cf63 + checksum/secret: 90ba33230f60ed1ee81e3088a4c88b0c9408c36ad5d40270cf05f503149f25a8 labels: app: docker-registry release: flyte-sandbox diff --git a/docs/deployment/agents/airflow.rst b/docs/deployment/agents/airflow.rst index ad6a6dab360..174967e20c4 100644 --- a/docs/deployment/agents/airflow.rst +++ b/docs/deployment/agents/airflow.rst @@ -21,7 +21,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -35,11 +35,6 @@ Specify agent configuration - container_array: k8s-array - airflow: agent-service - plugins: - agent-service: - supportedTaskTypes: - - airflow - .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it. @@ -63,10 +58,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array airflow: agent-service - plugins: - agent-service: - supportedTaskTypes: - - airflow Upgrade the Flyte Helm release diff --git a/docs/deployment/agents/bigquery.rst b/docs/deployment/agents/bigquery.rst index d3e4ee490e3..0d30d0d3b35 100644 --- a/docs/deployment/agents/bigquery.rst +++ b/docs/deployment/agents/bigquery.rst @@ -29,7 +29,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -42,11 +42,6 @@ Specify agent configuration - container: container - container_array: k8s-array - bigquery_query_job_task: agent-service - - plugins: - agent-service: - supportedTaskTypes: - - bigquery_query_job_task .. group-tab:: Flyte core @@ -71,10 +66,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array bigquery_query_job_task: agent-service - plugins: - agent-service: - supportedTaskTypes: - - bigquery_query_job_task Ensure that the propeller has the correct service account for BigQuery. diff --git a/docs/deployment/agents/chatgpt.rst b/docs/deployment/agents/chatgpt.rst index afc569222f4..cb0b44fa390 100644 --- a/docs/deployment/agents/chatgpt.rst +++ b/docs/deployment/agents/chatgpt.rst @@ -20,7 +20,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -36,8 +36,6 @@ Specify agent configuration plugins: agent-service: - supportedTaskTypes: - - chatgpt # Configuring the timeout is optional. # Tasks like using ChatGPT with a large model might require a longer time, # so we have the option to adjust the timeout setting here. @@ -70,8 +68,6 @@ Specify agent configuration chatgpt: agent-service plugins: agent-service: - supportedTaskTypes: - - chatgpt # Configuring the timeout is optional. # Tasks like using ChatGPT with a large model might require a longer time, # so we have the option to adjust the timeout setting here. diff --git a/docs/deployment/agents/databricks.rst b/docs/deployment/agents/databricks.rst index b21fab3c571..0458fb36678 100644 --- a/docs/deployment/agents/databricks.rst +++ b/docs/deployment/agents/databricks.rst @@ -139,7 +139,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,12,16 + :emphasize-lines: 7,12 tasks: task-plugins: @@ -153,17 +153,13 @@ Specify agent configuration - sidecar - k8s-array - agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark .. group-tab:: Helm chart Edit the relevant YAML file to specify the plugin. .. code-block:: yaml - :emphasize-lines: 7,11,15 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -176,17 +172,13 @@ Specify agent configuration - container: container - container_array: k8s-array - spark: agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following config to it: .. code-block:: yaml - :emphasize-lines: 9,14-17 + :emphasize-lines: 9 enabled_plugins: tasks: @@ -201,10 +193,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array spark: agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark Add the Databricks access token ------------------------------- diff --git a/docs/deployment/agents/mmcloud.rst b/docs/deployment/agents/mmcloud.rst index 422162af27f..9ccb101aa2f 100644 --- a/docs/deployment/agents/mmcloud.rst +++ b/docs/deployment/agents/mmcloud.rst @@ -71,8 +71,6 @@ Enable the MMCloud agent by adding the following config to the relevant YAML fil mmcloud-agent: endpoint: insecure: true - supportedTaskTypes: - - mmcloud_task agentForTaskTypes: - mmcloud_task: mmcloud-agent diff --git a/docs/deployment/agents/openai_batch.rst b/docs/deployment/agents/openai_batch.rst index 7aff9d262e8..2cfa70471aa 100644 --- a/docs/deployment/agents/openai_batch.rst +++ b/docs/deployment/agents/openai_batch.rst @@ -19,7 +19,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,15 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -32,17 +32,13 @@ Specify agent configuration - container: container - container_array: k8s-array - openai-batch: agent-service - plugins: - agent-service: - supportedTaskTypes: - - openai-batch .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it: .. code-block:: yaml - :emphasize-lines: 9,14,18 + :emphasize-lines: 9,14 configmap: enabled_plugins: @@ -58,10 +54,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array openai-batch: agent-service - plugins: - agent-service: - supportedTaskTypes: - - openai-batch Add the OpenAI API token ------------------------ diff --git a/docs/deployment/agents/sagemaker_inference.rst b/docs/deployment/agents/sagemaker_inference.rst index 5ceb248c2d9..3f03e08f55c 100644 --- a/docs/deployment/agents/sagemaker_inference.rst +++ b/docs/deployment/agents/sagemaker_inference.rst @@ -19,7 +19,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11-12,16-17 + :emphasize-lines: 7,11-12 tasks: task-plugins: @@ -33,18 +33,13 @@ Specify agent configuration - container_array: k8s-array - boto: agent-service - sagemaker-endpoint: agent-service - plugins: - agent-service: - supportedTaskTypes: - - boto - - sagemaker-endpoint .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it: .. code-block:: yaml - :emphasize-lines: 9,14-15,19-20 + :emphasize-lines: 9,14-15 configmap: enabled_plugins: @@ -61,11 +56,6 @@ Specify agent configuration container_array: k8s-array boto: agent-service sagemaker-endpoint: agent-service - plugins: - agent-service: - supportedTaskTypes: - - boto - - sagemaker-endpoint AWS credentials --------------- diff --git a/docs/deployment/agents/sensor.rst b/docs/deployment/agents/sensor.rst index 958e5d896ac..312e34bcd1a 100644 --- a/docs/deployment/agents/sensor.rst +++ b/docs/deployment/agents/sensor.rst @@ -56,7 +56,7 @@ Enable the sensor agent by adding the following config to the relevant YAML file kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -69,11 +69,6 @@ Enable the sensor agent by adding the following config to the relevant YAML file - container: container - container_array: k8s-array - sensor: agent-service - - plugins: - agent-service: - supportedTaskTypes: - - sensor .. group-tab:: Flyte core @@ -98,10 +93,6 @@ Enable the sensor agent by adding the following config to the relevant YAML file sidecar: sidecar container_array: k8s-array sensor: agent-service - plugins: - agent-service: - supportedTaskTypes: - - sensor Upgrade the deployment diff --git a/docs/deployment/agents/snowflake.rst b/docs/deployment/agents/snowflake.rst index d6ee74125bd..dad62a3795c 100644 --- a/docs/deployment/agents/snowflake.rst +++ b/docs/deployment/agents/snowflake.rst @@ -35,7 +35,7 @@ Specify agent configuration kubectl edit configmap flyte-sandbox-config -n flyte .. code-block:: yaml - :emphasize-lines: 7,11,16 + :emphasize-lines: 7,11 tasks: task-plugins: @@ -49,11 +49,6 @@ Specify agent configuration - container_array: k8s-array - snowflake: agent-service - plugins: - agent-service: - supportedTaskTypes: - - snowflake - .. group-tab:: Flyte core Create a file named ``values-override.yaml`` and add the following configuration to it. @@ -77,10 +72,6 @@ Specify agent configuration sidecar: sidecar container_array: k8s-array snowflake: agent-service - plugins: - agent-service: - supportedTaskTypes: - - snowflake Ensure that the propeller has the correct service account for Snowflake. diff --git a/docs/deployment/configuration/generated/flytepropeller_config.rst b/docs/deployment/configuration/generated/flytepropeller_config.rst index be6f7ee7f05..554ebec849d 100644 --- a/docs/deployment/configuration/generated/flytepropeller_config.rst +++ b/docs/deployment/configuration/generated/flytepropeller_config.rst @@ -994,9 +994,6 @@ agent-service (`agent.Config`_) Value: 50 ProjectScopeResourceConstraint: Value: 100 - supportedTaskTypes: - - task_type_1 - - task_type_2 webApi: caching: maxSystemFailures: 5 diff --git a/docs/flyte_agents/developing_agents.md b/docs/flyte_agents/developing_agents.md index ee989b812f8..fd310088efd 100644 --- a/docs/flyte_agents/developing_agents.md +++ b/docs/flyte_agents/developing_agents.md @@ -197,10 +197,6 @@ you can route particular task requests to designated agent services by adjusting ```yaml plugins: agent-service: - supportedTaskTypes: - - bigquery_query_job_task - - default_task - - custom_task # By default, all requests will be sent to the default agent. defaultAgent: endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000" @@ -224,4 +220,4 @@ you can route particular task requests to designated agent services by adjusting agentForTaskTypes: # It will override the default agent for custom_task, which means propeller will send the request to this agent. - custom_task: custom_agent -``` \ No newline at end of file +``` From 61e75da759bbbdaa0cfd3ccbff0cd495fcfc5910 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 3 Sep 2024 13:11:26 +0800 Subject: [PATCH 07/11] [flyteagent] Add Logging for Agent Supported Task Types (#5718) * Add Logging for Agent-Supported Task Types Signed-off-by: Future-Outlier * use make(map[string]struct{}) Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier --- flyteplugins/go/tasks/plugins/webapi/agent/client.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flyteplugins/go/tasks/plugins/webapi/agent/client.go b/flyteplugins/go/tasks/plugins/webapi/agent/client.go index 35e66621074..148113fb384 100644 --- a/flyteplugins/go/tasks/plugins/webapi/agent/client.go +++ b/flyteplugins/go/tasks/plugins/webapi/agent/client.go @@ -3,6 +3,7 @@ package agent import ( "context" "crypto/x509" + "strings" "golang.org/x/exp/maps" "google.golang.org/grpc" @@ -127,19 +128,26 @@ func getAgentRegistry(ctx context.Context, cs *ClientSet) Registry { continue } + agentSupportedTaskCategories := make(map[string]struct{}) for _, agent := range res.GetAgents() { deprecatedSupportedTaskTypes := agent.SupportedTaskTypes for _, supportedTaskType := range deprecatedSupportedTaskTypes { agent := &Agent{AgentDeployment: agentDeployment, IsSync: agent.IsSync} newAgentRegistry[supportedTaskType] = map[int32]*Agent{defaultTaskTypeVersion: agent} + agentSupportedTaskCategories[supportedTaskType] = struct{}{} } supportedTaskCategories := agent.SupportedTaskCategories for _, supportedCategory := range supportedTaskCategories { agent := &Agent{AgentDeployment: agentDeployment, IsSync: agent.IsSync} - newAgentRegistry[supportedCategory.GetName()] = map[int32]*Agent{supportedCategory.GetVersion(): agent} + supportedCategoryName := supportedCategory.GetName() + newAgentRegistry[supportedCategoryName] = map[int32]*Agent{supportedCategory.GetVersion(): agent} + agentSupportedTaskCategories[supportedCategoryName] = struct{}{} } + } + logger.Infof(ctx, "AgentDeployment [%v] supports the following task types: [%v]", agentDeployment.Endpoint, + strings.Join(maps.Keys(agentSupportedTaskCategories), ", ")) } // If the agent doesn't implement the metadata service, we construct the registry based on the configuration @@ -160,6 +168,7 @@ func getAgentRegistry(ctx context.Context, cs *ClientSet) Registry { } } + logger.Infof(ctx, "AgentDeployments support the following task types: [%v]", strings.Join(maps.Keys(newAgentRegistry), ", ")) return newAgentRegistry } From d4db50cb29fc6c1000260587720b4275f0dda765 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 4 Sep 2024 02:36:46 +0530 Subject: [PATCH 08/11] extend pod customization to include init containers (#5685) * apply pod config to init containers Signed-off-by: Samhita Alla * remove container Signed-off-by: Samhita Alla * add test Signed-off-by: Samhita Alla * Add bool to check init containers Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Samhita Alla Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .../tasks/pluginmachinery/flytek8s/pod_helper.go | 9 +++++++++ .../go/tasks/plugins/k8s/pod/container_test.go | 16 ++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index db62aeb4e7b..e8252090df0 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -348,6 +348,15 @@ func ApplyFlytePodConfiguration(ctx context.Context, tCtx pluginsCore.TaskExecut IncludeConsoleURL: hasExternalLinkType(taskTemplate), } + // iterate over the initContainers first + for index := range podSpec.InitContainers { + var resourceMode = ResourceCustomizationModeEnsureExistingResourcesInRange + + if err := AddFlyteCustomizationsToContainer(ctx, templateParameters, resourceMode, &podSpec.InitContainers[index]); err != nil { + return nil, nil, err + } + } + resourceRequests := make([]v1.ResourceRequirements, 0, len(podSpec.Containers)) var primaryContainer *v1.Container for index, container := range podSpec.Containers { diff --git a/flyteplugins/go/tasks/plugins/k8s/pod/container_test.go b/flyteplugins/go/tasks/plugins/k8s/pod/container_test.go index 5d89e2f0eca..f9d49b2448e 100644 --- a/flyteplugins/go/tasks/plugins/k8s/pod/container_test.go +++ b/flyteplugins/go/tasks/plugins/k8s/pod/container_test.go @@ -49,6 +49,13 @@ func dummyContainerTaskTemplate(command []string, args []string) *core.TaskTempl func dummyContainerTaskTemplateWithPodSpec(command []string, args []string) *core.TaskTemplate { podSpec := v1.PodSpec{ + InitContainers: []v1.Container{ + v1.Container{ + Name: "test-image", + Command: command, + Args: args, + }, + }, Containers: []v1.Container{ v1.Container{ Name: "test-image", @@ -174,24 +181,28 @@ func TestContainerTaskExecutor_BuildResource(t *testing.T) { taskTemplate *core.TaskTemplate taskMetadata pluginsCore.TaskExecutionMetadata expectServiceAccount string + checkInitContainer bool }{ { name: "BuildResource", taskTemplate: dummyContainerTaskTemplate(command, args), taskMetadata: dummyContainerTaskMetadata(containerResourceRequirements, nil, true, ""), expectServiceAccount: serviceAccount, + checkInitContainer: false, }, { name: "BuildResource_PodTemplate", taskTemplate: dummyContainerTaskTemplateWithPodSpec(command, args), taskMetadata: dummyContainerTaskMetadata(containerResourceRequirements, nil, true, ""), expectServiceAccount: podTemplateServiceAccount, + checkInitContainer: true, }, { name: "BuildResource_SecurityContext", taskTemplate: dummyContainerTaskTemplate(command, args), taskMetadata: dummyContainerTaskMetadata(containerResourceRequirements, nil, false, ""), expectServiceAccount: securityContextServiceAccount, + checkInitContainer: false, }, } for _, tc := range testCases { @@ -213,6 +224,11 @@ func TestContainerTaskExecutor_BuildResource(t *testing.T) { assert.Equal(t, command, j.Spec.Containers[0].Command) assert.Equal(t, []string{"test-data-reference"}, j.Spec.Containers[0].Args) + if tc.checkInitContainer { + assert.Equal(t, command, j.Spec.InitContainers[0].Command) + assert.Equal(t, []string{"test-data-reference"}, j.Spec.InitContainers[0].Args) + } + assert.Equal(t, tc.expectServiceAccount, j.Spec.ServiceAccountName) }) } From 5b6bd52a9c9e9cd06f0ff84b8310ee33f7bc3c9c Mon Sep 17 00:00:00 2001 From: Nikki Everett Date: Thu, 5 Sep 2024 01:58:20 -0500 Subject: [PATCH 09/11] Update "Try Serverless" language in Quickstart guide (#5698) * update langauge Signed-off-by: nikki everett * Update docs/quickstart_guide.md Co-authored-by: Haytham Abuelfutuh Signed-off-by: Nikki Everett --------- Signed-off-by: nikki everett Signed-off-by: Nikki Everett Co-authored-by: Haytham Abuelfutuh --- docs/quickstart_guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/quickstart_guide.md b/docs/quickstart_guide.md index e2e03cc8c55..244f0934c6c 100644 --- a/docs/quickstart_guide.md +++ b/docs/quickstart_guide.md @@ -6,12 +6,12 @@ In this guide, you will create and run a Flyte workflow in a local Python enviro ````{dropdown} Try Flyte in your browser :animate: fade-in-slide-down -Union Serverless is a version of Flyte hosted by [Union](https://www.union.ai/) that you can try in your browser. Sign up for the waitlist below and check out the [Union Serverless Quickstart guide](https://docs.union.ai/serverless/quick-start). +Union Serverless is a SaaS offering by [Union](https://www.union.ai/) built on Flyte. Sign up below and check out the [Union Serverless Quickstart guide](https://docs.union.ai/serverless/quick-start). ```{button-link} https://signup.union.ai/ :color: warning -Join the Union Serverless waitlist +Sign up for Union Serverless ``` ```` From 5f695895ed6a7fa45d980023ab874591184b0fc1 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Thu, 5 Sep 2024 04:46:10 -0700 Subject: [PATCH 10/11] Refactor flyteadmin to pass proto structs as pointers (#5717) --- flyteadmin/.golangci.yml | 5 +- flyteadmin/dataproxy/service.go | 32 ++-- flyteadmin/dataproxy/service_test.go | 16 +- .../implementations/cloudevent_publisher.go | 12 +- .../node_execution_event_writer.go | 6 +- .../node_execution_event_writer_test.go | 2 +- .../workflow_execution_event_writer.go | 6 +- .../workflow_execution_event_writer_test.go | 2 +- .../async/events/interfaces/node_execution.go | 2 +- .../events/interfaces/workflow_execution.go | 2 +- .../mocks/node_execution_event_writer.go | 2 +- .../mocks/workflow_execution_event_writer.go | 2 +- flyteadmin/pkg/async/notifications/email.go | 34 ++-- .../pkg/async/notifications/email_test.go | 10 +- .../implementations/aws_emailer.go | 4 +- .../implementations/aws_emailer_test.go | 8 +- .../implementations/aws_processor.go | 4 +- .../implementations/aws_processor_test.go | 4 +- .../implementations/gcp_processor.go | 4 +- .../implementations/gcp_processor_test.go | 4 +- .../implementations/noop_notifications.go | 2 +- .../implementations/sandbox_processor.go | 4 +- .../implementations/sandbox_processor_test.go | 6 +- .../implementations/sendgrid_emailer.go | 4 +- .../implementations/sendgrid_emailer_test.go | 2 +- .../async/notifications/interfaces/emailer.go | 2 +- .../async/notifications/mocks/processor.go | 4 +- .../schedule/aws/cloud_watch_scheduler.go | 12 +- .../aws/cloud_watch_scheduler_test.go | 16 +- .../pkg/async/schedule/aws/serialization.go | 8 +- .../async/schedule/aws/serialization_test.go | 4 +- flyteadmin/pkg/async/schedule/aws/shared.go | 2 +- .../pkg/async/schedule/aws/shared_test.go | 2 +- .../async/schedule/aws/workflow_executor.go | 24 +-- .../schedule/aws/workflow_executor_test.go | 30 +-- .../schedule/interfaces/event_scheduler.go | 8 +- .../schedule/mocks/mock_event_scheduler.go | 6 +- .../async/schedule/noop/event_scheduler.go | 2 +- flyteadmin/pkg/common/flyte_url.go | 8 +- flyteadmin/pkg/common/flyte_url_test.go | 8 +- .../data/implementations/aws_remote_url.go | 10 +- .../data/implementations/gcp_remote_url.go | 10 +- .../data/implementations/noop_remote_url.go | 6 +- flyteadmin/pkg/data/interfaces/remote.go | 2 +- flyteadmin/pkg/data/mocks/remote.go | 6 +- .../impl/description_entity_manager.go | 6 +- .../impl/description_entity_manager_test.go | 16 +- .../pkg/manager/impl/execution_manager.go | 108 +++++------ .../manager/impl/execution_manager_test.go | 174 +++++++++--------- .../executions/quality_of_service_test.go | 2 +- .../pkg/manager/impl/executions/queues.go | 4 +- .../manager/impl/executions/queues_test.go | 16 +- .../pkg/manager/impl/launch_plan_manager.go | 58 +++--- .../manager/impl/launch_plan_manager_test.go | 92 ++++----- .../pkg/manager/impl/metrics_manager.go | 28 +-- .../pkg/manager/impl/metrics_manager_test.go | 10 +- .../pkg/manager/impl/named_entity_manager.go | 12 +- .../manager/impl/named_entity_manager_test.go | 12 +- .../manager/impl/node_execution_manager.go | 34 ++-- .../impl/node_execution_manager_test.go | 88 ++++----- .../pkg/manager/impl/project_manager.go | 18 +- .../pkg/manager/impl/project_manager_test.go | 30 +-- .../impl/resources/resource_manager.go | 40 ++-- .../impl/resources/resource_manager_test.go | 34 ++-- flyteadmin/pkg/manager/impl/signal_manager.go | 10 +- .../pkg/manager/impl/signal_manager_test.go | 20 +- .../manager/impl/task_execution_manager.go | 28 +-- .../impl/task_execution_manager_test.go | 24 +-- flyteadmin/pkg/manager/impl/task_manager.go | 20 +- .../pkg/manager/impl/task_manager_test.go | 16 +- .../manager/impl/testutils/mock_requests.go | 39 ++-- flyteadmin/pkg/manager/impl/util/data.go | 16 +- flyteadmin/pkg/manager/impl/util/data_test.go | 34 ++-- flyteadmin/pkg/manager/impl/util/filters.go | 6 +- .../pkg/manager/impl/util/filters_test.go | 4 +- flyteadmin/pkg/manager/impl/util/shared.go | 30 +-- .../pkg/manager/impl/util/shared_test.go | 72 ++++---- .../impl/util/single_task_execution.go | 22 +-- .../impl/util/single_task_execution_test.go | 8 +- .../impl/validation/attributes_validator.go | 16 +- .../validation/attributes_validator_test.go | 50 ++--- .../impl/validation/execution_validator.go | 4 +- .../validation/execution_validator_test.go | 12 +- .../impl/validation/launch_plan_validator.go | 4 +- .../validation/launch_plan_validator_test.go | 12 +- .../impl/validation/named_entity_validator.go | 6 +- .../validation/named_entity_validator_test.go | 36 ++-- .../validation/node_execution_validator.go | 4 +- .../node_execution_validator_test.go | 14 +- .../impl/validation/project_validator.go | 8 +- .../impl/validation/project_validator_test.go | 44 ++--- .../impl/validation/signal_validator.go | 12 +- .../impl/validation/signal_validator_test.go | 26 +-- .../validation/task_execution_validator.go | 4 +- .../task_execution_validator_test.go | 16 +- .../manager/impl/validation/task_validator.go | 16 +- .../impl/validation/task_validator_test.go | 14 +- .../pkg/manager/impl/validation/validation.go | 12 +- .../impl/validation/validation_test.go | 54 +++--- .../impl/validation/workflow_validator.go | 6 +- .../validation/workflow_validator_test.go | 4 +- .../pkg/manager/impl/workflow_manager.go | 40 ++-- .../pkg/manager/impl/workflow_manager_test.go | 30 +-- .../manager/interfaces/description_entity.go | 4 +- .../pkg/manager/interfaces/execution.go | 18 +- .../pkg/manager/interfaces/launch_plan.go | 14 +- flyteadmin/pkg/manager/interfaces/metrics.go | 2 +- .../pkg/manager/interfaces/named_entity.go | 6 +- .../pkg/manager/interfaces/node_execution.go | 12 +- flyteadmin/pkg/manager/interfaces/project.go | 10 +- flyteadmin/pkg/manager/interfaces/resource.go | 20 +- flyteadmin/pkg/manager/interfaces/signal.go | 6 +- flyteadmin/pkg/manager/interfaces/task.go | 8 +- .../pkg/manager/interfaces/task_execution.go | 8 +- flyteadmin/pkg/manager/interfaces/workflow.go | 8 +- flyteadmin/pkg/manager/mocks/execution.go | 36 ++-- flyteadmin/pkg/manager/mocks/launch_plan.go | 28 +-- .../pkg/manager/mocks/metrics_interface.go | 8 +- flyteadmin/pkg/manager/mocks/named_entity.go | 12 +- .../pkg/manager/mocks/node_execution.go | 22 +-- flyteadmin/pkg/manager/mocks/project.go | 20 +- flyteadmin/pkg/manager/mocks/resource.go | 34 ++-- .../pkg/manager/mocks/signal_interface.go | 48 ++--- flyteadmin/pkg/manager/mocks/task.go | 12 +- .../pkg/manager/mocks/task_execution.go | 16 +- flyteadmin/pkg/manager/mocks/workflow.go | 12 +- .../gormimpl/node_execution_repo_test.go | 8 +- .../gormimpl/task_execution_repo_test.go | 2 +- .../interfaces/node_execution_repo.go | 2 +- .../interfaces/task_execution_repo.go | 2 +- .../transformers/description_entity.go | 2 +- .../transformers/description_entity_test.go | 2 +- .../repositories/transformers/execution.go | 4 +- .../transformers/execution_event.go | 2 +- .../transformers/execution_event_test.go | 2 +- .../transformers/execution_test.go | 30 +-- .../repositories/transformers/launch_plan.go | 8 +- .../transformers/launch_plan_test.go | 12 +- .../transformers/node_execution_event.go | 2 +- .../transformers/node_execution_event_test.go | 2 +- .../pkg/repositories/transformers/project.go | 8 +- .../repositories/transformers/project_test.go | 2 +- .../pkg/repositories/transformers/resource.go | 16 +- .../transformers/resource_test.go | 10 +- .../pkg/repositories/transformers/signal.go | 10 +- .../repositories/transformers/signal_test.go | 14 +- .../pkg/repositories/transformers/task.go | 6 +- .../repositories/transformers/task_test.go | 2 +- .../pkg/repositories/transformers/workflow.go | 2 +- flyteadmin/pkg/rpc/adminservice/attributes.go | 56 +----- .../rpc/adminservice/description_entity.go | 13 +- flyteadmin/pkg/rpc/adminservice/execution.go | 53 +----- .../pkg/rpc/adminservice/launch_plan.go | 39 +--- .../pkg/rpc/adminservice/named_entity.go | 21 +-- .../pkg/rpc/adminservice/node_execution.go | 34 +--- flyteadmin/pkg/rpc/adminservice/project.go | 28 +-- flyteadmin/pkg/rpc/adminservice/task.go | 23 +-- .../pkg/rpc/adminservice/task_execution.go | 26 +-- .../rpc/adminservice/tests/execution_test.go | 41 ++--- .../adminservice/tests/launch_plan_test.go | 12 +- .../adminservice/tests/node_execution_test.go | 18 +- .../adminservice/tests/project_domain_test.go | 8 +- .../rpc/adminservice/tests/project_test.go | 10 +- .../adminservice/tests/task_execution_test.go | 34 ++-- .../pkg/rpc/adminservice/tests/task_test.go | 10 +- .../rpc/adminservice/tests/workflow_test.go | 4 +- flyteadmin/pkg/rpc/adminservice/workflow.go | 24 +-- flyteadmin/pkg/rpc/signal_service.go | 17 +- flyteadmin/pkg/rpc/signal_service_test.go | 36 ---- .../interfaces/application_configuration.go | 4 +- .../mocks/quality_of_service_configuration.go | 10 +- .../quality_of_service_configuration.go | 2 +- .../mocks/mock_configuration_provider.go | 2 +- .../runtime/quality_of_service_provider.go | 6 +- .../workflowengine/impl/interface_provider.go | 20 +- .../impl/interface_provider_test.go | 6 +- .../workflowengine/impl/prepare_execution.go | 5 +- .../scheduler/dbapi/event_scheduler_impl.go | 4 +- .../dbapi/event_scheduler_impl_test.go | 16 +- flyteadmin/scheduler/identifier/identifier.go | 8 +- flyteadmin/tests/task_test.go | 24 +-- flyteadmin/tests/workflow_test.go | 2 +- flytectl/cmd/compile/compile.go | 4 +- flytepropeller/pkg/compiler/admin.go | 14 +- flytepropeller/pkg/compiler/admin_test.go | 4 +- flytepropeller/pkg/compiler/builders.go | 4 +- flytepropeller/pkg/compiler/common/id_set.go | 2 +- flytepropeller/pkg/compiler/common/index.go | 2 +- .../pkg/compiler/common/mocks/task.go | 12 +- .../pkg/compiler/common/mocks/workflow.go | 32 ++-- .../compiler/common/mocks/workflow_builder.go | 34 ++-- flytepropeller/pkg/compiler/requirements.go | 4 +- .../pkg/compiler/validators/bindings_test.go | 8 +- .../pkg/compiler/validators/interface.go | 6 +- .../pkg/compiler/validators/interface_test.go | 12 +- .../pkg/compiler/validators/node.go | 4 +- .../pkg/compiler/workflow_compiler_test.go | 4 +- .../nodes/dynamic/dynamic_workflow.go | 5 +- 198 files changed, 1472 insertions(+), 1733 deletions(-) diff --git a/flyteadmin/.golangci.yml b/flyteadmin/.golangci.yml index 76977148dcf..4dbb031812b 100644 --- a/flyteadmin/.golangci.yml +++ b/flyteadmin/.golangci.yml @@ -36,5 +36,6 @@ linters-settings: - prefix(github.com/flyteorg) skip-generated: true issues: - exclude: - - copylocks + exclude-rules: + - path: pkg/workflowengine/impl/prepare_execution.go + text: "copies lock" diff --git a/flyteadmin/dataproxy/service.go b/flyteadmin/dataproxy/service.go index 5bb7a16632e..d61998835f5 100644 --- a/flyteadmin/dataproxy/service.go +++ b/flyteadmin/dataproxy/service.go @@ -161,7 +161,7 @@ func (s Service) CreateDownloadLink(ctx context.Context, req *service.CreateDown // Lookup task, node, workflow execution var nativeURL string if nodeExecutionIDEnvelope, casted := req.GetSource().(*service.CreateDownloadLinkRequest_NodeExecutionId); casted { - node, err := s.nodeExecutionManager.GetNodeExecution(ctx, admin.NodeExecutionGetRequest{ + node, err := s.nodeExecutionManager.GetNodeExecution(ctx, &admin.NodeExecutionGetRequest{ Id: nodeExecutionIDEnvelope.NodeExecutionId, }) @@ -309,9 +309,9 @@ func (s Service) validateResolveArtifactRequest(req *service.GetDataRequest) err // GetCompleteTaskExecutionID returns the task execution identifier for the task execution with the Task ID filled in. // The one coming from the node execution doesn't have this as this is not data encapsulated in the flyte url. -func (s Service) GetCompleteTaskExecutionID(ctx context.Context, taskExecID core.TaskExecutionIdentifier) (*core.TaskExecutionIdentifier, error) { +func (s Service) GetCompleteTaskExecutionID(ctx context.Context, taskExecID *core.TaskExecutionIdentifier) (*core.TaskExecutionIdentifier, error) { - taskExecs, err := s.taskExecutionManager.ListTaskExecutions(ctx, admin.TaskExecutionListRequest{ + taskExecs, err := s.taskExecutionManager.ListTaskExecutions(ctx, &admin.TaskExecutionListRequest{ NodeExecutionId: taskExecID.GetNodeExecutionId(), Limit: 1, Filters: fmt.Sprintf("eq(retry_attempt,%s)", strconv.Itoa(int(taskExecID.RetryAttempt))), @@ -326,9 +326,9 @@ func (s Service) GetCompleteTaskExecutionID(ctx context.Context, taskExecID core return taskExec.Id, nil } -func (s Service) GetTaskExecutionID(ctx context.Context, attempt int, nodeExecID core.NodeExecutionIdentifier) (*core.TaskExecutionIdentifier, error) { - taskExecs, err := s.taskExecutionManager.ListTaskExecutions(ctx, admin.TaskExecutionListRequest{ - NodeExecutionId: &nodeExecID, +func (s Service) GetTaskExecutionID(ctx context.Context, attempt int, nodeExecID *core.NodeExecutionIdentifier) (*core.TaskExecutionIdentifier, error) { + taskExecs, err := s.taskExecutionManager.ListTaskExecutions(ctx, &admin.TaskExecutionListRequest{ + NodeExecutionId: nodeExecID, Limit: 1, Filters: fmt.Sprintf("eq(retry_attempt,%s)", strconv.Itoa(attempt)), }) @@ -342,11 +342,11 @@ func (s Service) GetTaskExecutionID(ctx context.Context, attempt int, nodeExecID return taskExec.Id, nil } -func (s Service) GetDataFromNodeExecution(ctx context.Context, nodeExecID core.NodeExecutionIdentifier, ioType common.ArtifactType, name string) ( +func (s Service) GetDataFromNodeExecution(ctx context.Context, nodeExecID *core.NodeExecutionIdentifier, ioType common.ArtifactType, name string) ( *service.GetDataResponse, error) { - resp, err := s.nodeExecutionManager.GetNodeExecutionData(ctx, admin.NodeExecutionGetDataRequest{ - Id: &nodeExecID, + resp, err := s.nodeExecutionManager.GetNodeExecutionData(ctx, &admin.NodeExecutionGetDataRequest{ + Id: nodeExecID, }) if err != nil { return nil, err @@ -361,7 +361,7 @@ func (s Service) GetDataFromNodeExecution(ctx context.Context, nodeExecID core.N // Assume deck, and create a download link request dlRequest := service.CreateDownloadLinkRequest{ ArtifactType: service.ArtifactType_ARTIFACT_TYPE_DECK, - Source: &service.CreateDownloadLinkRequest_NodeExecutionId{NodeExecutionId: &nodeExecID}, + Source: &service.CreateDownloadLinkRequest_NodeExecutionId{NodeExecutionId: nodeExecID}, } resp, err := s.CreateDownloadLink(ctx, &dlRequest) if err != nil { @@ -391,12 +391,12 @@ func (s Service) GetDataFromNodeExecution(ctx context.Context, nodeExecID core.N }, nil } -func (s Service) GetDataFromTaskExecution(ctx context.Context, taskExecID core.TaskExecutionIdentifier, ioType common.ArtifactType, name string) ( +func (s Service) GetDataFromTaskExecution(ctx context.Context, taskExecID *core.TaskExecutionIdentifier, ioType common.ArtifactType, name string) ( *service.GetDataResponse, error) { var lm *core.LiteralMap - reqT := admin.TaskExecutionGetDataRequest{ - Id: &taskExecID, + reqT := &admin.TaskExecutionGetDataRequest{ + Id: taskExecID, } resp, err := s.taskExecutionManager.GetTaskExecutionData(ctx, reqT) if err != nil { @@ -445,13 +445,13 @@ func (s Service) GetData(ctx context.Context, req *service.GetDataRequest) ( } if execution.NodeExecID != nil { - return s.GetDataFromNodeExecution(ctx, *execution.NodeExecID, execution.IOType, execution.LiteralName) + return s.GetDataFromNodeExecution(ctx, execution.NodeExecID, execution.IOType, execution.LiteralName) } else if execution.PartialTaskExecID != nil { - taskExecID, err := s.GetCompleteTaskExecutionID(ctx, *execution.PartialTaskExecID) + taskExecID, err := s.GetCompleteTaskExecutionID(ctx, execution.PartialTaskExecID) if err != nil { return nil, err } - return s.GetDataFromTaskExecution(ctx, *taskExecID, execution.IOType, execution.LiteralName) + return s.GetDataFromTaskExecution(ctx, taskExecID, execution.IOType, execution.LiteralName) } return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "failed to parse get data request %v", req) diff --git a/flyteadmin/dataproxy/service_test.go b/flyteadmin/dataproxy/service_test.go index 3716b989149..81193e106be 100644 --- a/flyteadmin/dataproxy/service_test.go +++ b/flyteadmin/dataproxy/service_test.go @@ -160,7 +160,7 @@ func TestCreateUploadLocationMore(t *testing.T) { func TestCreateDownloadLink(t *testing.T) { dataStore := commonMocks.GetMockStorageClient() nodeExecutionManager := &mocks.MockNodeExecutionManager{} - nodeExecutionManager.SetGetNodeExecutionFunc(func(ctx context.Context, request admin.NodeExecutionGetRequest) (*admin.NodeExecution, error) { + nodeExecutionManager.SetGetNodeExecutionFunc(func(ctx context.Context, request *admin.NodeExecutionGetRequest) (*admin.NodeExecution, error) { return &admin.NodeExecution{ Closure: &admin.NodeExecutionClosure{ DeckUri: "s3://something/something", @@ -282,14 +282,14 @@ func TestService_GetData(t *testing.T) { } nodeExecutionManager.SetGetNodeExecutionDataFunc( - func(ctx context.Context, request admin.NodeExecutionGetDataRequest) (*admin.NodeExecutionGetDataResponse, error) { + func(ctx context.Context, request *admin.NodeExecutionGetDataRequest) (*admin.NodeExecutionGetDataResponse, error) { return &admin.NodeExecutionGetDataResponse{ FullInputs: inputsLM, FullOutputs: outputsLM, }, nil }, ) - taskExecutionManager.SetListTaskExecutionsCallback(func(ctx context.Context, request admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { + taskExecutionManager.SetListTaskExecutionsCallback(func(ctx context.Context, request *admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { return &admin.TaskExecutionList{ TaskExecutions: []*admin.TaskExecution{ { @@ -315,7 +315,7 @@ func TestService_GetData(t *testing.T) { }, }, nil }) - taskExecutionManager.SetGetTaskExecutionDataCallback(func(ctx context.Context, request admin.TaskExecutionGetDataRequest) (*admin.TaskExecutionGetDataResponse, error) { + taskExecutionManager.SetGetTaskExecutionDataCallback(func(ctx context.Context, request *admin.TaskExecutionGetDataRequest) (*admin.TaskExecutionGetDataResponse, error) { return &admin.TaskExecutionGetDataResponse{ FullInputs: inputsLM, FullOutputs: outputsLM, @@ -388,10 +388,10 @@ func TestService_Error(t *testing.T) { assert.NoError(t, err) t.Run("get a working set of urls without retry attempt", func(t *testing.T) { - taskExecutionManager.SetListTaskExecutionsCallback(func(ctx context.Context, request admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { + taskExecutionManager.SetListTaskExecutionsCallback(func(ctx context.Context, request *admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { return nil, errors.NewFlyteAdminErrorf(1, "not found") }) - nodeExecID := core.NodeExecutionIdentifier{ + nodeExecID := &core.NodeExecutionIdentifier{ NodeId: "n0", ExecutionId: &core.WorkflowExecutionIdentifier{ Project: "proj", @@ -404,13 +404,13 @@ func TestService_Error(t *testing.T) { }) t.Run("get a working set of urls without retry attempt", func(t *testing.T) { - taskExecutionManager.SetListTaskExecutionsCallback(func(ctx context.Context, request admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { + taskExecutionManager.SetListTaskExecutionsCallback(func(ctx context.Context, request *admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) { return &admin.TaskExecutionList{ TaskExecutions: nil, Token: "", }, nil }) - nodeExecID := core.NodeExecutionIdentifier{ + nodeExecID := &core.NodeExecutionIdentifier{ NodeId: "n0", ExecutionId: &core.WorkflowExecutionIdentifier{ Project: "proj", diff --git a/flyteadmin/pkg/async/cloudevent/implementations/cloudevent_publisher.go b/flyteadmin/pkg/async/cloudevent/implementations/cloudevent_publisher.go index 46bd0f0ede5..228db852d0c 100644 --- a/flyteadmin/pkg/async/cloudevent/implementations/cloudevent_publisher.go +++ b/flyteadmin/pkg/async/cloudevent/implementations/cloudevent_publisher.go @@ -204,8 +204,8 @@ func getNodeExecutionContext(ctx context.Context, identifier *core.NodeExecution // This is a rough copy of the ListTaskExecutions function in TaskExecutionManager. It can be deprecated once we move the processing out of Admin itself. // Just return the highest retry attempt. -func (c *CloudEventWrappedPublisher) getLatestTaskExecutions(ctx context.Context, nodeExecutionID core.NodeExecutionIdentifier) (*admin.TaskExecution, error) { - ctx = getNodeExecutionContext(ctx, &nodeExecutionID) +func (c *CloudEventWrappedPublisher) getLatestTaskExecutions(ctx context.Context, nodeExecutionID *core.NodeExecutionIdentifier) (*admin.TaskExecution, error) { + ctx = getNodeExecutionContext(ctx, nodeExecutionID) identifierFilters, err := util.GetNodeExecutionIdentifierFilters(ctx, nodeExecutionID) if err != nil { @@ -283,7 +283,7 @@ func (c *CloudEventWrappedPublisher) TransformNodeExecutionEvent(ctx context.Con var taskExecID *core.TaskExecutionIdentifier var typedInterface *core.TypedInterface - lte, err := c.getLatestTaskExecutions(ctx, *rawEvent.Id) + lte, err := c.getLatestTaskExecutions(ctx, rawEvent.Id) if err != nil { logger.Errorf(ctx, "failed to get latest task execution for node exec id [%+v] with err: %v", rawEvent.Id, err) return nil, err @@ -353,7 +353,7 @@ func (c *CloudEventWrappedPublisher) Publish(ctx context.Context, notificationTy phase = e.Phase.String() eventTime = e.OccurredAt.AsTime() - dummyNodeExecutionID := core.NodeExecutionIdentifier{ + dummyNodeExecutionID := &core.NodeExecutionIdentifier{ NodeId: "end-node", ExecutionId: e.ExecutionId, } @@ -378,7 +378,7 @@ func (c *CloudEventWrappedPublisher) Publish(ctx context.Context, notificationTy if e.ParentNodeExecutionId == nil { return fmt.Errorf("parent node execution id is nil for task execution [%+v]", e) } - eventSource = common.FlyteURLKeyFromNodeExecutionIDRetry(*e.ParentNodeExecutionId, + eventSource = common.FlyteURLKeyFromNodeExecutionIDRetry(e.ParentNodeExecutionId, int(e.RetryAttempt)) finalMsg, err = c.TransformTaskExecutionEvent(ctx, e) if err != nil { @@ -392,7 +392,7 @@ func (c *CloudEventWrappedPublisher) Publish(ctx context.Context, notificationTy phase = e.Phase.String() eventTime = e.OccurredAt.AsTime() eventID = fmt.Sprintf("%v.%v", executionID, phase) - eventSource = common.FlyteURLKeyFromNodeExecutionID(*msgType.Event.Id) + eventSource = common.FlyteURLKeyFromNodeExecutionID(msgType.Event.Id) finalMsg, err = c.TransformNodeExecutionEvent(ctx, e) if err != nil { logger.Errorf(ctx, "Failed to transform node execution event with error: %v", err) diff --git a/flyteadmin/pkg/async/events/implementations/node_execution_event_writer.go b/flyteadmin/pkg/async/events/implementations/node_execution_event_writer.go index 623baf354dc..3f59496626c 100644 --- a/flyteadmin/pkg/async/events/implementations/node_execution_event_writer.go +++ b/flyteadmin/pkg/async/events/implementations/node_execution_event_writer.go @@ -14,10 +14,10 @@ import ( // events, node execution processing doesn't have to wait on these to be committed. type nodeExecutionEventWriter struct { db repositoryInterfaces.Repository - events chan admin.NodeExecutionEventRequest + events chan *admin.NodeExecutionEventRequest } -func (w *nodeExecutionEventWriter) Write(event admin.NodeExecutionEventRequest) { +func (w *nodeExecutionEventWriter) Write(event *admin.NodeExecutionEventRequest) { w.events <- event } @@ -40,6 +40,6 @@ func (w *nodeExecutionEventWriter) Run() { func NewNodeExecutionEventWriter(db repositoryInterfaces.Repository, bufferSize int) interfaces.NodeExecutionEventWriter { return &nodeExecutionEventWriter{ db: db, - events: make(chan admin.NodeExecutionEventRequest, bufferSize), + events: make(chan *admin.NodeExecutionEventRequest, bufferSize), } } diff --git a/flyteadmin/pkg/async/events/implementations/node_execution_event_writer_test.go b/flyteadmin/pkg/async/events/implementations/node_execution_event_writer_test.go index 4fafd19f693..3271ff54527 100644 --- a/flyteadmin/pkg/async/events/implementations/node_execution_event_writer_test.go +++ b/flyteadmin/pkg/async/events/implementations/node_execution_event_writer_test.go @@ -12,7 +12,7 @@ import ( func TestNodeExecutionEventWriter(t *testing.T) { db := mocks.NewMockRepository() - event := admin.NodeExecutionEventRequest{ + event := &admin.NodeExecutionEventRequest{ RequestId: "request_id", Event: &event2.NodeExecutionEvent{ Id: &core.NodeExecutionIdentifier{ diff --git a/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer.go b/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer.go index 7521dee4b8b..e4f63b44f99 100644 --- a/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer.go +++ b/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer.go @@ -14,10 +14,10 @@ import ( // events, workflow execution processing doesn't have to wait on these to be committed. type workflowExecutionEventWriter struct { db repositoryInterfaces.Repository - events chan admin.WorkflowExecutionEventRequest + events chan *admin.WorkflowExecutionEventRequest } -func (w *workflowExecutionEventWriter) Write(event admin.WorkflowExecutionEventRequest) { +func (w *workflowExecutionEventWriter) Write(event *admin.WorkflowExecutionEventRequest) { w.events <- event } @@ -40,6 +40,6 @@ func (w *workflowExecutionEventWriter) Run() { func NewWorkflowExecutionEventWriter(db repositoryInterfaces.Repository, bufferSize int) interfaces.WorkflowExecutionEventWriter { return &workflowExecutionEventWriter{ db: db, - events: make(chan admin.WorkflowExecutionEventRequest, bufferSize), + events: make(chan *admin.WorkflowExecutionEventRequest, bufferSize), } } diff --git a/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer_test.go b/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer_test.go index db52cb809f4..ce8dd390df2 100644 --- a/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer_test.go +++ b/flyteadmin/pkg/async/events/implementations/workflow_execution_event_writer_test.go @@ -12,7 +12,7 @@ import ( func TestWorkflowExecutionEventWriter(t *testing.T) { db := mocks.NewMockRepository() - event := admin.WorkflowExecutionEventRequest{ + event := &admin.WorkflowExecutionEventRequest{ RequestId: "request_id", Event: &event2.WorkflowExecutionEvent{ ExecutionId: &core.WorkflowExecutionIdentifier{ diff --git a/flyteadmin/pkg/async/events/interfaces/node_execution.go b/flyteadmin/pkg/async/events/interfaces/node_execution.go index d6163db526d..5e62e3b69b9 100644 --- a/flyteadmin/pkg/async/events/interfaces/node_execution.go +++ b/flyteadmin/pkg/async/events/interfaces/node_execution.go @@ -8,5 +8,5 @@ import ( type NodeExecutionEventWriter interface { Run() - Write(nodeExecutionEvent admin.NodeExecutionEventRequest) + Write(nodeExecutionEvent *admin.NodeExecutionEventRequest) } diff --git a/flyteadmin/pkg/async/events/interfaces/workflow_execution.go b/flyteadmin/pkg/async/events/interfaces/workflow_execution.go index f730ddc993c..d503ea09346 100644 --- a/flyteadmin/pkg/async/events/interfaces/workflow_execution.go +++ b/flyteadmin/pkg/async/events/interfaces/workflow_execution.go @@ -8,5 +8,5 @@ import ( type WorkflowExecutionEventWriter interface { Run() - Write(workflowExecutionEvent admin.WorkflowExecutionEventRequest) + Write(workflowExecutionEvent *admin.WorkflowExecutionEventRequest) } diff --git a/flyteadmin/pkg/async/events/mocks/node_execution_event_writer.go b/flyteadmin/pkg/async/events/mocks/node_execution_event_writer.go index b51639c6099..f50149b6db4 100644 --- a/flyteadmin/pkg/async/events/mocks/node_execution_event_writer.go +++ b/flyteadmin/pkg/async/events/mocks/node_execution_event_writer.go @@ -19,6 +19,6 @@ func (_m *NodeExecutionEventWriter) Run() { } // Write provides a mock function with given fields: nodeExecutionEvent -func (_m *NodeExecutionEventWriter) Write(nodeExecutionEvent admin.NodeExecutionEventRequest) { +func (_m *NodeExecutionEventWriter) Write(nodeExecutionEvent *admin.NodeExecutionEventRequest) { _m.Called(nodeExecutionEvent) } diff --git a/flyteadmin/pkg/async/events/mocks/workflow_execution_event_writer.go b/flyteadmin/pkg/async/events/mocks/workflow_execution_event_writer.go index 1c366f4cd5a..616f3334f9c 100644 --- a/flyteadmin/pkg/async/events/mocks/workflow_execution_event_writer.go +++ b/flyteadmin/pkg/async/events/mocks/workflow_execution_event_writer.go @@ -19,6 +19,6 @@ func (_m *WorkflowExecutionEventWriter) Run() { } // Write provides a mock function with given fields: workflowExecutionEvent -func (_m *WorkflowExecutionEventWriter) Write(workflowExecutionEvent admin.WorkflowExecutionEventRequest) { +func (_m *WorkflowExecutionEventWriter) Write(workflowExecutionEvent *admin.WorkflowExecutionEventRequest) { _m.Called(workflowExecutionEvent) } diff --git a/flyteadmin/pkg/async/notifications/email.go b/flyteadmin/pkg/async/notifications/email.go index 94eb71719cf..a89210cead7 100644 --- a/flyteadmin/pkg/async/notifications/email.go +++ b/flyteadmin/pkg/async/notifications/email.go @@ -8,7 +8,7 @@ import ( "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" ) -type GetTemplateValue func(admin.WorkflowExecutionEventRequest, *admin.Execution) string +type GetTemplateValue func(*admin.WorkflowExecutionEventRequest, *admin.Execution) string const executionError = " The execution failed with error: [%s]." @@ -29,58 +29,58 @@ const launchPlanName = "launch_plan.name" const launchPlanVersion = "launch_plan.version" const replaceAllInstances = -1 -func getProject(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getProject(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Id.Project } -func getDomain(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getDomain(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Id.Domain } -func getName(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getName(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Id.Name } -func getPhase(request admin.WorkflowExecutionEventRequest, _ *admin.Execution) string { +func getPhase(request *admin.WorkflowExecutionEventRequest, _ *admin.Execution) string { return strings.ToLower(request.Event.Phase.String()) } -func getError(request admin.WorkflowExecutionEventRequest, _ *admin.Execution) string { +func getError(request *admin.WorkflowExecutionEventRequest, _ *admin.Execution) string { if request.Event.GetError() != nil { return fmt.Sprintf(executionError, request.Event.GetError().Message) } return "" } -func getWorkflowProject(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getWorkflowProject(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Closure.WorkflowId.Project } -func getWorkflowDomain(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getWorkflowDomain(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Closure.WorkflowId.Domain } -func getWorkflowName(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getWorkflowName(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Closure.WorkflowId.Name } -func getWorkflowVersion(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getWorkflowVersion(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Closure.WorkflowId.Version } -func getLaunchPlanProject(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getLaunchPlanProject(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Spec.LaunchPlan.Project } -func getLaunchPlanDomain(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getLaunchPlanDomain(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Spec.LaunchPlan.Domain } -func getLaunchPlanName(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getLaunchPlanName(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Spec.LaunchPlan.Name } -func getLaunchPlanVersion(_ admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { +func getLaunchPlanVersion(_ *admin.WorkflowExecutionEventRequest, exec *admin.Execution) string { return exec.Spec.LaunchPlan.Version } @@ -100,7 +100,7 @@ var getTemplateValueFuncs = map[string]GetTemplateValue{ launchPlanVersion: getLaunchPlanVersion, } -func substituteEmailParameters(message string, request admin.WorkflowExecutionEventRequest, execution *admin.Execution) string { +func substituteEmailParameters(message string, request *admin.WorkflowExecutionEventRequest, execution *admin.Execution) string { for template, function := range getTemplateValueFuncs { message = strings.Replace(message, fmt.Sprintf(substitutionParam, template), function(request, execution), replaceAllInstances) message = strings.Replace(message, fmt.Sprintf(substitutionParamNoSpaces, template), function(request, execution), replaceAllInstances) @@ -112,8 +112,8 @@ func substituteEmailParameters(message string, request admin.WorkflowExecutionEv // in customizable email fields set in the flyteadmin application notifications config. func ToEmailMessageFromWorkflowExecutionEvent( config runtimeInterfaces.NotificationsConfig, - emailNotification admin.EmailNotification, - request admin.WorkflowExecutionEventRequest, + emailNotification *admin.EmailNotification, + request *admin.WorkflowExecutionEventRequest, execution *admin.Execution) *admin.EmailMessage { return &admin.EmailMessage{ diff --git a/flyteadmin/pkg/async/notifications/email_test.go b/flyteadmin/pkg/async/notifications/email_test.go index 5a9cf01c36c..35f351a45d5 100644 --- a/flyteadmin/pkg/async/notifications/email_test.go +++ b/flyteadmin/pkg/async/notifications/email_test.go @@ -52,7 +52,7 @@ var workflowExecution = &admin.Execution{ func TestSubstituteEmailParameters(t *testing.T) { message := "{{ unused }}. {{project }} and {{ domain }} and {{ name }} ended up in {{ phase }}.{{ error }}" - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_SUCCEEDED, }, @@ -88,7 +88,7 @@ func TestSubstituteAllTemplates(t *testing.T) { messageTemplate = append(messageTemplate, template) desiredResult = append(desiredResult, result) } - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_SUCCEEDED, }, @@ -117,7 +117,7 @@ func TestSubstituteAllTemplatesNoSpaces(t *testing.T) { messageTemplate = append(messageTemplate, template) desiredResult = append(desiredResult, result) } - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_SUCCEEDED, }, @@ -136,12 +136,12 @@ func TestToEmailMessageFromWorkflowExecutionEvent(t *testing.T) { Subject: "Notice: Execution \"{{ name }}\" has succeeded in \"{{ domain }}\".", }, } - emailNotification := admin.EmailNotification{ + emailNotification := &admin.EmailNotification{ RecipientsEmail: []string{ "a@example.com", "b@example.org", }, } - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_ABORTED, }, diff --git a/flyteadmin/pkg/async/notifications/implementations/aws_emailer.go b/flyteadmin/pkg/async/notifications/implementations/aws_emailer.go index 72985c95486..712bd7080dd 100644 --- a/flyteadmin/pkg/async/notifications/implementations/aws_emailer.go +++ b/flyteadmin/pkg/async/notifications/implementations/aws_emailer.go @@ -21,7 +21,7 @@ type AwsEmailer struct { awsEmail sesiface.SESAPI } -func FlyteEmailToSesEmailInput(email admin.EmailMessage) ses.SendEmailInput { +func FlyteEmailToSesEmailInput(email *admin.EmailMessage) ses.SendEmailInput { var toAddress []*string for _, toEmail := range email.RecipientsEmail { // SES email input takes an array of pointers to strings so we have to create a new one for each email @@ -51,7 +51,7 @@ func FlyteEmailToSesEmailInput(email admin.EmailMessage) ses.SendEmailInput { } } -func (e *AwsEmailer) SendEmail(ctx context.Context, email admin.EmailMessage) error { +func (e *AwsEmailer) SendEmail(ctx context.Context, email *admin.EmailMessage) error { emailInput := FlyteEmailToSesEmailInput(email) _, err := e.awsEmail.SendEmail(&emailInput) e.systemMetrics.SendTotal.Inc() diff --git a/flyteadmin/pkg/async/notifications/implementations/aws_emailer_test.go b/flyteadmin/pkg/async/notifications/implementations/aws_emailer_test.go index c06d818eeca..01a2a062731 100644 --- a/flyteadmin/pkg/async/notifications/implementations/aws_emailer_test.go +++ b/flyteadmin/pkg/async/notifications/implementations/aws_emailer_test.go @@ -32,7 +32,7 @@ func TestAwsEmailer_SendEmail(t *testing.T) { mockAwsEmail := mocks.SESClient{} var awsSES sesiface.SESAPI = &mockAwsEmail expectedSenderEmail := "no-reply@example.com" - emailNotification := admin.EmailMessage{ + emailNotification := &admin.EmailMessage{ SubjectLine: "Notice: Execution \"name\" has succeeded in \"domain\".", SenderEmail: "no-reply@example.com", RecipientsEmail: []string{ @@ -67,7 +67,7 @@ func TestAwsEmailer_SendEmail(t *testing.T) { } func TestFlyteEmailToSesEmailInput(t *testing.T) { - emailNotification := admin.EmailMessage{ + emailNotification := &admin.EmailMessage{ SubjectLine: "Notice: Execution \"name\" has succeeded in \"domain\".", SenderEmail: "no-reply@example.com", RecipientsEmail: []string{ @@ -97,7 +97,7 @@ func TestAwsEmailer_SendEmailError(t *testing.T) { testEmail := NewAwsEmailer(getNotificationsConfig(), promutils.NewTestScope(), awsSES) - emailNotification := admin.EmailMessage{ + emailNotification := &admin.EmailMessage{ SubjectLine: "Notice: Execution \"name\" has succeeded in \"domain\".", SenderEmail: "no-reply@example.com", RecipientsEmail: []string{ @@ -125,7 +125,7 @@ func TestAwsEmailer_SendEmailEmailOutput(t *testing.T) { testEmail := NewAwsEmailer(getNotificationsConfig(), promutils.NewTestScope(), awsSES) - emailNotification := admin.EmailMessage{ + emailNotification := &admin.EmailMessage{ SubjectLine: "Notice: Execution \"name\" has succeeded in \"domain\".", SenderEmail: "no-reply@example.com", RecipientsEmail: []string{ diff --git a/flyteadmin/pkg/async/notifications/implementations/aws_processor.go b/flyteadmin/pkg/async/notifications/implementations/aws_processor.go index fb3b3c2a1ba..0e1dceb53c2 100644 --- a/flyteadmin/pkg/async/notifications/implementations/aws_processor.go +++ b/flyteadmin/pkg/async/notifications/implementations/aws_processor.go @@ -36,7 +36,7 @@ func (p *Processor) StartProcessing() { } func (p *Processor) run() error { - var emailMessage admin.EmailMessage + emailMessage := &admin.EmailMessage{} var err error for msg := range p.sub.Start() { p.systemMetrics.MessageTotal.Inc() @@ -83,7 +83,7 @@ func (p *Processor) run() error { continue } - if err = proto.Unmarshal(notificationBytes, &emailMessage); err != nil { + if err = proto.Unmarshal(notificationBytes, emailMessage); err != nil { logger.Debugf(context.Background(), "failed to unmarshal to notification object from decoded string[%s] from message [%s] with err: %v", valueString, stringMsg, err) p.systemMetrics.MessageDecodingError.Inc() p.markMessageDone(msg) diff --git a/flyteadmin/pkg/async/notifications/implementations/aws_processor_test.go b/flyteadmin/pkg/async/notifications/implementations/aws_processor_test.go index ef27f1f3a84..e566fdd7407 100644 --- a/flyteadmin/pkg/async/notifications/implementations/aws_processor_test.go +++ b/flyteadmin/pkg/async/notifications/implementations/aws_processor_test.go @@ -30,7 +30,7 @@ func TestProcessor_StartProcessing(t *testing.T) { // Because the message stored in Amazon SQS is a JSON of the SNS output, store the test output in the JSON Messages. testSubscriber.JSONMessages = append(testSubscriber.JSONMessages, testSubscriberMessage) - sendEmailValidationFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailValidationFunc := func(ctx context.Context, email *admin.EmailMessage) error { assert.Equal(t, email.Body, testEmail.Body) assert.Equal(t, email.RecipientsEmail, testEmail.RecipientsEmail) assert.Equal(t, email.SubjectLine, testEmail.SubjectLine) @@ -115,7 +115,7 @@ func TestProcessor_StartProcessingError(t *testing.T) { func TestProcessor_StartProcessingEmailError(t *testing.T) { initializeProcessor() emailError := errors.New("error sending email") - sendEmailErrorFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailErrorFunc := func(ctx context.Context, email *admin.EmailMessage) error { return emailError } mockEmailer.SetSendEmailFunc(sendEmailErrorFunc) diff --git a/flyteadmin/pkg/async/notifications/implementations/gcp_processor.go b/flyteadmin/pkg/async/notifications/implementations/gcp_processor.go index 54e4f4a5926..b1e97ca7a1e 100644 --- a/flyteadmin/pkg/async/notifications/implementations/gcp_processor.go +++ b/flyteadmin/pkg/async/notifications/implementations/gcp_processor.go @@ -39,12 +39,12 @@ func (p *GcpProcessor) StartProcessing() { } func (p *GcpProcessor) run() error { - var emailMessage admin.EmailMessage + emailMessage := &admin.EmailMessage{} for msg := range p.sub.Start() { p.systemMetrics.MessageTotal.Inc() - if err := proto.Unmarshal(msg.Message(), &emailMessage); err != nil { + if err := proto.Unmarshal(msg.Message(), emailMessage); err != nil { logger.Debugf(context.Background(), "failed to unmarshal to notification object message [%s] with err: %v", string(msg.Message()), err) p.systemMetrics.MessageDecodingError.Inc() p.markMessageDone(msg) diff --git a/flyteadmin/pkg/async/notifications/implementations/gcp_processor_test.go b/flyteadmin/pkg/async/notifications/implementations/gcp_processor_test.go index da5bda2610d..5ad49a72577 100644 --- a/flyteadmin/pkg/async/notifications/implementations/gcp_processor_test.go +++ b/flyteadmin/pkg/async/notifications/implementations/gcp_processor_test.go @@ -34,7 +34,7 @@ func TestGcpProcessor_StartProcessing(t *testing.T) { testGcpProcessor := NewGcpProcessor(&testGcpSubscriber, &mockGcpEmailer, promutils.NewTestScope()) - sendEmailValidationFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailValidationFunc := func(ctx context.Context, email *admin.EmailMessage) error { assert.Equal(t, email.Body, testEmail.Body) assert.Equal(t, email.RecipientsEmail, testEmail.RecipientsEmail) assert.Equal(t, email.SubjectLine, testEmail.SubjectLine) @@ -81,7 +81,7 @@ func TestGcpProcessor_StartProcessingError(t *testing.T) { func TestGcpProcessor_StartProcessingEmailError(t *testing.T) { initializeGcpSubscriber() emailError := errors.New("error sending email") - sendEmailErrorFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailErrorFunc := func(ctx context.Context, email *admin.EmailMessage) error { return emailError } mockGcpEmailer.SetSendEmailFunc(sendEmailErrorFunc) diff --git a/flyteadmin/pkg/async/notifications/implementations/noop_notifications.go b/flyteadmin/pkg/async/notifications/implementations/noop_notifications.go index 4da316f6b28..03dfa063ea0 100644 --- a/flyteadmin/pkg/async/notifications/implementations/noop_notifications.go +++ b/flyteadmin/pkg/async/notifications/implementations/noop_notifications.go @@ -14,7 +14,7 @@ import ( // Email to use when there is no email configuration. type NoopEmail struct{} -func (n *NoopEmail) SendEmail(ctx context.Context, email admin.EmailMessage) error { +func (n *NoopEmail) SendEmail(ctx context.Context, email *admin.EmailMessage) error { logger.Debugf(ctx, "received noop SendEmail request with subject [%s] and recipient [%s]", email.SubjectLine, strings.Join(email.RecipientsEmail, ",")) return nil diff --git a/flyteadmin/pkg/async/notifications/implementations/sandbox_processor.go b/flyteadmin/pkg/async/notifications/implementations/sandbox_processor.go index 2cb83da4069..9fb5e34bc75 100644 --- a/flyteadmin/pkg/async/notifications/implementations/sandbox_processor.go +++ b/flyteadmin/pkg/async/notifications/implementations/sandbox_processor.go @@ -27,12 +27,12 @@ func (p *SandboxProcessor) StartProcessing() { } func (p *SandboxProcessor) run() error { - var emailMessage admin.EmailMessage + emailMessage := &admin.EmailMessage{} for { select { case msg := <-p.subChan: - err := proto.Unmarshal(msg, &emailMessage) + err := proto.Unmarshal(msg, emailMessage) if err != nil { logger.Errorf(context.Background(), "error with unmarshalling message [%v]", err) return err diff --git a/flyteadmin/pkg/async/notifications/implementations/sandbox_processor_test.go b/flyteadmin/pkg/async/notifications/implementations/sandbox_processor_test.go index d0ee9ee31bc..83594284a91 100644 --- a/flyteadmin/pkg/async/notifications/implementations/sandbox_processor_test.go +++ b/flyteadmin/pkg/async/notifications/implementations/sandbox_processor_test.go @@ -19,7 +19,7 @@ func TestSandboxProcessor_StartProcessingSuccess(t *testing.T) { msgChan <- msg testSandboxProcessor := NewSandboxProcessor(msgChan, &mockSandboxEmailer) - sendEmailValidationFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailValidationFunc := func(ctx context.Context, email *admin.EmailMessage) error { assert.Equal(t, testEmail.Body, email.Body) assert.Equal(t, testEmail.RecipientsEmail, email.RecipientsEmail) assert.Equal(t, testEmail.SubjectLine, email.SubjectLine) @@ -43,7 +43,7 @@ func TestSandboxProcessor_StartProcessingError(t *testing.T) { msgChan <- msg emailError := errors.New("error running processor") - sendEmailValidationFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailValidationFunc := func(ctx context.Context, email *admin.EmailMessage) error { return emailError } mockSandboxEmailer.SetSendEmailFunc(sendEmailValidationFunc) @@ -70,7 +70,7 @@ func TestSandboxProcessor_StartProcessingEmailError(t *testing.T) { testSandboxProcessor := NewSandboxProcessor(msgChan, &mockSandboxEmailer) emailError := errors.New("error sending email") - sendEmailValidationFunc := func(ctx context.Context, email admin.EmailMessage) error { + sendEmailValidationFunc := func(ctx context.Context, email *admin.EmailMessage) error { return emailError } diff --git a/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer.go b/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer.go index 54f53859f3a..c8386bd41e6 100644 --- a/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer.go +++ b/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer.go @@ -30,7 +30,7 @@ func getEmailAddresses(addresses []string) []*mail.Email { return sendgridAddresses } -func getSendgridEmail(adminEmail admin.EmailMessage) *mail.SGMailV3 { +func getSendgridEmail(adminEmail *admin.EmailMessage) *mail.SGMailV3 { m := mail.NewV3Mail() // This from email address is really here as a formality. For sendgrid specifically, the sender email is determined // from the api key that's used, not what you send along here. @@ -60,7 +60,7 @@ func getAPIKey(config runtimeInterfaces.EmailServerConfig) string { return strings.TrimSpace(string(apiKeyFile)) } -func (s SendgridEmailer) SendEmail(ctx context.Context, email admin.EmailMessage) error { +func (s SendgridEmailer) SendEmail(ctx context.Context, email *admin.EmailMessage) error { m := getSendgridEmail(email) s.systemMetrics.SendTotal.Inc() response, err := s.client.Send(m) diff --git a/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer_test.go b/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer_test.go index bfedb152d07..eafad84b2c3 100644 --- a/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer_test.go +++ b/flyteadmin/pkg/async/notifications/implementations/sendgrid_emailer_test.go @@ -21,7 +21,7 @@ func TestAddresses(t *testing.T) { } func TestGetEmail(t *testing.T) { - emailNotification := admin.EmailMessage{ + emailNotification := &admin.EmailMessage{ SubjectLine: "Notice: Execution \"name\" has succeeded in \"domain\".", SenderEmail: "no-reply@example.com", RecipientsEmail: []string{ diff --git a/flyteadmin/pkg/async/notifications/interfaces/emailer.go b/flyteadmin/pkg/async/notifications/interfaces/emailer.go index 54b6ad55749..f6874cf5808 100644 --- a/flyteadmin/pkg/async/notifications/interfaces/emailer.go +++ b/flyteadmin/pkg/async/notifications/interfaces/emailer.go @@ -9,5 +9,5 @@ import ( // The implementation of Emailer needs to be passed to the implementation of Processor // in order for emails to be sent. type Emailer interface { - SendEmail(ctx context.Context, email admin.EmailMessage) error + SendEmail(ctx context.Context, email *admin.EmailMessage) error } diff --git a/flyteadmin/pkg/async/notifications/mocks/processor.go b/flyteadmin/pkg/async/notifications/mocks/processor.go index a60bb26f966..178d68490eb 100644 --- a/flyteadmin/pkg/async/notifications/mocks/processor.go +++ b/flyteadmin/pkg/async/notifications/mocks/processor.go @@ -29,7 +29,7 @@ func (m *MockSubscriber) Stop() error { return nil } -type SendEmailFunc func(ctx context.Context, email admin.EmailMessage) error +type SendEmailFunc func(ctx context.Context, email *admin.EmailMessage) error type MockEmailer struct { sendEmailFunc SendEmailFunc @@ -39,7 +39,7 @@ func (m *MockEmailer) SetSendEmailFunc(sendEmail SendEmailFunc) { m.sendEmailFunc = sendEmail } -func (m *MockEmailer) SendEmail(ctx context.Context, email admin.EmailMessage) error { +func (m *MockEmailer) SendEmail(ctx context.Context, email *admin.EmailMessage) error { if m.sendEmailFunc != nil { return m.sendEmailFunc(ctx, email) } diff --git a/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler.go b/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler.go index 768d1957662..9c3cb166b57 100644 --- a/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler.go +++ b/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler.go @@ -68,7 +68,7 @@ type cloudWatchScheduler struct { metrics cloudWatchSchedulerMetrics } -func getScheduleName(scheduleNamePrefix string, identifier core.Identifier) string { +func getScheduleName(scheduleNamePrefix string, identifier *core.Identifier) string { hashedIdentifier := hashIdentifier(identifier) if len(scheduleNamePrefix) > 0 { return fmt.Sprintf(scheduleNameFormat, scheduleNamePrefix, hashedIdentifier) @@ -76,12 +76,12 @@ func getScheduleName(scheduleNamePrefix string, identifier core.Identifier) stri return fmt.Sprintf("%d", hashedIdentifier) } -func getScheduleDescription(identifier core.Identifier) string { +func getScheduleDescription(identifier *core.Identifier) string { return fmt.Sprintf(scheduleDescriptionFormat, identifier.Project, identifier.Domain, identifier.Name) } -func getScheduleExpression(schedule admin.Schedule) (string, error) { +func getScheduleExpression(schedule *admin.Schedule) (string, error) { if schedule.GetCronExpression() != "" { return fmt.Sprintf(cronExpression, schedule.GetCronExpression()), nil } @@ -171,11 +171,11 @@ func (s *cloudWatchScheduler) AddSchedule(ctx context.Context, input scheduleInt } func (s *cloudWatchScheduler) CreateScheduleInput(ctx context.Context, appConfig *appInterfaces.SchedulerConfig, - identifier core.Identifier, schedule *admin.Schedule) (scheduleInterfaces.AddScheduleInput, error) { + identifier *core.Identifier, schedule *admin.Schedule) (scheduleInterfaces.AddScheduleInput, error) { payload, err := SerializeScheduleWorkflowPayload( schedule.GetKickoffTimeInputArg(), - admin.NamedEntityIdentifier{ + &admin.NamedEntityIdentifier{ Project: identifier.Project, Domain: identifier.Domain, Name: identifier.Name, @@ -194,7 +194,7 @@ func (s *cloudWatchScheduler) CreateScheduleInput(ctx context.Context, appConfig addScheduleInput := scheduleInterfaces.AddScheduleInput{ Identifier: identifier, - ScheduleExpression: *schedule, + ScheduleExpression: schedule, Payload: payload, ScheduleNamePrefix: scheduleNamePrefix, } diff --git a/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler_test.go b/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler_test.go index bb32163d1f0..9b24e1baa98 100644 --- a/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler_test.go +++ b/flyteadmin/pkg/async/schedule/aws/cloud_watch_scheduler_test.go @@ -26,7 +26,7 @@ var expectedError = flyteAdminErrors.NewFlyteAdminError(codes.Internal, "foo") var testSerializedPayload = fmt.Sprintf("event triggered at '%s'", awsTimestampPlaceholder) -var testSchedulerIdentifier = core.Identifier{ +var testSchedulerIdentifier = &core.Identifier{ Project: "project", Domain: "domain", Name: "name", @@ -55,7 +55,7 @@ func TestGetScheduleDescription(t *testing.T) { } func TestGetScheduleExpression(t *testing.T) { - expression, err := getScheduleExpression(admin.Schedule{ + expression, err := getScheduleExpression(&admin.Schedule{ ScheduleExpression: &admin.Schedule_CronExpression{ CronExpression: "foo", }, @@ -63,7 +63,7 @@ func TestGetScheduleExpression(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "cron(foo)", expression) - expression, err = getScheduleExpression(admin.Schedule{ + expression, err = getScheduleExpression(&admin.Schedule{ ScheduleExpression: &admin.Schedule_Rate{ Rate: &admin.FixedRate{ Value: 1, @@ -74,7 +74,7 @@ func TestGetScheduleExpression(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "rate(1 day)", expression) - expression, err = getScheduleExpression(admin.Schedule{ + expression, err = getScheduleExpression(&admin.Schedule{ ScheduleExpression: &admin.Schedule_Rate{ Rate: &admin.FixedRate{ Value: 2, @@ -85,7 +85,7 @@ func TestGetScheduleExpression(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "rate(2 hours)", expression) - _, err = getScheduleExpression(admin.Schedule{}) + _, err = getScheduleExpression(&admin.Schedule{}) assert.Equal(t, codes.InvalidArgument, err.(flyteAdminErrors.FlyteAdminError).Code()) } @@ -133,7 +133,7 @@ func TestAddSchedule(t *testing.T) { assert.Nil(t, scheduler.AddSchedule(context.Background(), scheduleInterfaces.AddScheduleInput{ Identifier: testSchedulerIdentifier, - ScheduleExpression: admin.Schedule{ + ScheduleExpression: &admin.Schedule{ ScheduleExpression: &admin.Schedule_Rate{ Rate: &admin.FixedRate{ Value: 1, @@ -168,7 +168,7 @@ func TestAddSchedule_PutRuleError(t *testing.T) { err := scheduler.AddSchedule(context.Background(), scheduleInterfaces.AddScheduleInput{ Identifier: testSchedulerIdentifier, - ScheduleExpression: admin.Schedule{ + ScheduleExpression: &admin.Schedule{ ScheduleExpression: &admin.Schedule_Rate{ Rate: &admin.FixedRate{ Value: 1, @@ -195,7 +195,7 @@ func TestAddSchedule_PutTargetsError(t *testing.T) { err := scheduler.AddSchedule(context.Background(), scheduleInterfaces.AddScheduleInput{ Identifier: testSchedulerIdentifier, - ScheduleExpression: admin.Schedule{ + ScheduleExpression: &admin.Schedule{ ScheduleExpression: &admin.Schedule_Rate{ Rate: &admin.FixedRate{ Value: 1, diff --git a/flyteadmin/pkg/async/schedule/aws/serialization.go b/flyteadmin/pkg/async/schedule/aws/serialization.go index 833235f4fc2..8162a9a265a 100644 --- a/flyteadmin/pkg/async/schedule/aws/serialization.go +++ b/flyteadmin/pkg/async/schedule/aws/serialization.go @@ -35,13 +35,13 @@ type ScheduledWorkflowExecutionRequest struct { // The name of the kickoff time input argument in the workflow definition. This will be filled with kickoff time. KickoffTimeArg string // The desired launch plan identifier to trigger on schedule event firings. - LaunchPlanIdentifier admin.NamedEntityIdentifier + LaunchPlanIdentifier *admin.NamedEntityIdentifier } // This produces a function that is used to serialize messages enqueued on the cloudwatch scheduler. func SerializeScheduleWorkflowPayload( - kickoffTimeArg string, launchPlanIdentifier admin.NamedEntityIdentifier) (*string, error) { - payload, err := proto.Marshal(&launchPlanIdentifier) + kickoffTimeArg string, launchPlanIdentifier *admin.NamedEntityIdentifier) (*string, error) { + payload, err := proto.Marshal(launchPlanIdentifier) if err != nil { return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "failed to marshall launch plan with err: %v", err) } @@ -81,6 +81,6 @@ func DeserializeScheduleWorkflowPayload(payload []byte) (ScheduledWorkflowExecut return ScheduledWorkflowExecutionRequest{ KickoffTime: kickoffTime, KickoffTimeArg: scheduleWorkflowPayload.KickoffTimeArg, - LaunchPlanIdentifier: launchPlanIdentifier, + LaunchPlanIdentifier: &launchPlanIdentifier, }, nil } diff --git a/flyteadmin/pkg/async/schedule/aws/serialization_test.go b/flyteadmin/pkg/async/schedule/aws/serialization_test.go index 5b148662476..ae6a293ecf3 100644 --- a/flyteadmin/pkg/async/schedule/aws/serialization_test.go +++ b/flyteadmin/pkg/async/schedule/aws/serialization_test.go @@ -15,7 +15,7 @@ import ( const testKickoffTimeArg = "kickoff time arg" -var testLaunchPlanIdentifier = admin.NamedEntityIdentifier{ +var testLaunchPlanIdentifier = &admin.NamedEntityIdentifier{ Name: "name", Project: "project", Domain: "domain", @@ -38,7 +38,7 @@ func TestDeserializeScheduleWorkflowPayload(t *testing.T) { time.Date(2017, 12, 22, 18, 43, 48, 0, time.UTC), scheduledWorkflowExecutionRequest.KickoffTime) assert.Equal(t, testKickoffTimeArg, scheduledWorkflowExecutionRequest.KickoffTimeArg) - assert.True(t, proto.Equal(&testLaunchPlanIdentifier, &scheduledWorkflowExecutionRequest.LaunchPlanIdentifier), + assert.True(t, proto.Equal(testLaunchPlanIdentifier, scheduledWorkflowExecutionRequest.LaunchPlanIdentifier), fmt.Sprintf("scheduledWorkflowExecutionRequest.LaunchPlanIdentifier %v", &scheduledWorkflowExecutionRequest.LaunchPlanIdentifier)) } diff --git a/flyteadmin/pkg/async/schedule/aws/shared.go b/flyteadmin/pkg/async/schedule/aws/shared.go index 2a0590cff6e..3868e057994 100644 --- a/flyteadmin/pkg/async/schedule/aws/shared.go +++ b/flyteadmin/pkg/async/schedule/aws/shared.go @@ -9,7 +9,7 @@ import ( "github.com/flyteorg/flyte/flytestdlib/logger" ) -func hashIdentifier(identifier core.Identifier) uint64 { +func hashIdentifier(identifier *core.Identifier) uint64 { h := fnv.New64() _, err := h.Write([]byte(fmt.Sprintf(scheduleNameInputsFormat, identifier.Project, identifier.Domain, identifier.Name))) diff --git a/flyteadmin/pkg/async/schedule/aws/shared_test.go b/flyteadmin/pkg/async/schedule/aws/shared_test.go index 4fb0ccd515d..7833d0980df 100644 --- a/flyteadmin/pkg/async/schedule/aws/shared_test.go +++ b/flyteadmin/pkg/async/schedule/aws/shared_test.go @@ -9,7 +9,7 @@ import ( ) func TestHashIdentifier(t *testing.T) { - identifier := core.Identifier{ + identifier := &core.Identifier{ Project: "project", Domain: "domain", Name: "name", diff --git a/flyteadmin/pkg/async/schedule/aws/workflow_executor.go b/flyteadmin/pkg/async/schedule/aws/workflow_executor.go index 13d2041f9da..523fdd077e3 100644 --- a/flyteadmin/pkg/async/schedule/aws/workflow_executor.go +++ b/flyteadmin/pkg/async/schedule/aws/workflow_executor.go @@ -61,7 +61,7 @@ var doNotconsumeBase64 = false // The kickoff time argument isn't required for scheduled workflows. However, if it exists we substitute the kick-off // time value for the input argument. func (e *workflowExecutor) resolveKickoffTimeArg( - request ScheduledWorkflowExecutionRequest, launchPlan admin.LaunchPlan, + request ScheduledWorkflowExecutionRequest, launchPlan *admin.LaunchPlan, executionRequest *admin.ExecutionCreateRequest) error { if request.KickoffTimeArg == "" || launchPlan.Closure.ExpectedInputs == nil { logger.Debugf(context.Background(), "No kickoff time to resolve for scheduled workflow execution: [%s/%s/%s]", @@ -100,8 +100,8 @@ func (e *workflowExecutor) resolveKickoffTimeArg( return nil } -func (e *workflowExecutor) getActiveLaunchPlanVersion(launchPlanIdentifier *admin.NamedEntityIdentifier) (admin.LaunchPlan, error) { - launchPlans, err := e.launchPlanManager.ListLaunchPlans(context.Background(), admin.ResourceListRequest{ +func (e *workflowExecutor) getActiveLaunchPlanVersion(launchPlanIdentifier *admin.NamedEntityIdentifier) (*admin.LaunchPlan, error) { + launchPlans, err := e.launchPlanManager.ListLaunchPlans(context.Background(), &admin.ResourceListRequest{ Id: launchPlanIdentifier, Filters: activeLaunchPlanFilter, Limit: 1, @@ -110,20 +110,20 @@ func (e *workflowExecutor) getActiveLaunchPlanVersion(launchPlanIdentifier *admi logger.Warningf(context.Background(), "failed to find active launch plan with identifier [%+v]", launchPlanIdentifier) e.metrics.NoActiveLaunchPlanVersionsFound.Inc() - return admin.LaunchPlan{}, err + return &admin.LaunchPlan{}, err } if len(launchPlans.LaunchPlans) != 1 { e.metrics.GreaterThan1LaunchPlanVersionsFound.Inc() logger.Warningf(context.Background(), "failed to get exactly one active launch plan for identifier: %+v", launchPlanIdentifier) - return admin.LaunchPlan{}, errors.NewFlyteAdminErrorf(codes.Internal, + return &admin.LaunchPlan{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to get exactly one active launch plan for identifier: %+v", launchPlanIdentifier) } - return *launchPlans.LaunchPlans[0], nil + return launchPlans.LaunchPlans[0], nil } -func generateExecutionName(launchPlan admin.LaunchPlan, kickoffTime time.Time) string { - hashedIdentifier := hashIdentifier(core.Identifier{ +func generateExecutionName(launchPlan *admin.LaunchPlan, kickoffTime time.Time) string { + hashedIdentifier := hashIdentifier(&core.Identifier{ Project: launchPlan.Id.Project, Domain: launchPlan.Id.Domain, Name: launchPlan.Id.Name, @@ -133,7 +133,7 @@ func generateExecutionName(launchPlan admin.LaunchPlan, kickoffTime time.Time) s } func (e *workflowExecutor) formulateExecutionCreateRequest( - launchPlan admin.LaunchPlan, kickoffTime time.Time) admin.ExecutionCreateRequest { + launchPlan *admin.LaunchPlan, kickoffTime time.Time) *admin.ExecutionCreateRequest { // Deterministically assign a name based on the schedule kickoff time/launch plan definition. name := generateExecutionName(launchPlan, kickoffTime) logger.Debugf(context.Background(), "generated name [%s] for scheduled execution with launch plan [%+v]", @@ -147,7 +147,7 @@ func (e *workflowExecutor) formulateExecutionCreateRequest( logger.Warningf(context.Background(), "failed to serialize kickoff time [%v] to proto with err: %v", kickoffTime, err) } - executionRequest := admin.ExecutionCreateRequest{ + executionRequest := &admin.ExecutionCreateRequest{ Project: launchPlan.Id.Project, Domain: launchPlan.Id.Domain, Name: name, @@ -189,7 +189,7 @@ func (e *workflowExecutor) run() error { logger.Debugf(context.Background(), "Processing scheduled workflow execution event: %+v", scheduledWorkflowExecutionRequest) - launchPlan, err := e.getActiveLaunchPlanVersion(&scheduledWorkflowExecutionRequest.LaunchPlanIdentifier) + launchPlan, err := e.getActiveLaunchPlanVersion(scheduledWorkflowExecutionRequest.LaunchPlanIdentifier) if err != nil { // In the rare case that a scheduled event fires right before a user disables the currently active launch // plan version (and triggers deleting the schedule rule) there may be no active launch plans. This is fine, @@ -209,7 +209,7 @@ func (e *workflowExecutor) run() error { executionRequest := e.formulateExecutionCreateRequest(launchPlan, scheduledWorkflowExecutionRequest.KickoffTime) ctx = contextutils.WithWorkflowID(ctx, fmt.Sprintf(workflowIdentifierFmt, executionRequest.Project, executionRequest.Domain, executionRequest.Name)) - err = e.resolveKickoffTimeArg(scheduledWorkflowExecutionRequest, launchPlan, &executionRequest) + err = e.resolveKickoffTimeArg(scheduledWorkflowExecutionRequest, launchPlan, executionRequest) if err != nil { e.metrics.FailedResolveKickoffTimeArg.Inc() logger.Error(context.Background(), err.Error()) diff --git a/flyteadmin/pkg/async/schedule/aws/workflow_executor_test.go b/flyteadmin/pkg/async/schedule/aws/workflow_executor_test.go index 0479e890732..f6fc9b96931 100644 --- a/flyteadmin/pkg/async/schedule/aws/workflow_executor_test.go +++ b/flyteadmin/pkg/async/schedule/aws/workflow_executor_test.go @@ -26,14 +26,14 @@ import ( const testKickoffTime = "kickoff time arg" var testKickoffTimestamp = time.Date(2017, 12, 22, 18, 43, 48, 0, time.UTC) -var testIdentifier = admin.NamedEntityIdentifier{ +var testIdentifier = &admin.NamedEntityIdentifier{ Name: "name", Project: "project", Domain: "domain", } var protoTestTimestamp, _ = ptypes.TimestampProto(testKickoffTimestamp) -var testKickoffTimeProtoLiteral = core.Literal{ +var testKickoffTimeProtoLiteral = &core.Literal{ Value: &core.Literal_Scalar{ Scalar: &core.Scalar{ Value: &core.Scalar_Primitive{ @@ -72,7 +72,7 @@ func TestResolveKickoffTimeArg(t *testing.T) { KickoffTimeArg: testKickoffTime, KickoffTime: testKickoffTimestamp, } - launchPlan := admin.LaunchPlan{ + launchPlan := &admin.LaunchPlan{ Closure: &admin.LaunchPlanClosure{ ExpectedInputs: &core.ParameterMap{ Parameters: map[string]*core.Parameter{ @@ -81,7 +81,7 @@ func TestResolveKickoffTimeArg(t *testing.T) { }, }, } - executionRequest := admin.ExecutionCreateRequest{ + executionRequest := &admin.ExecutionCreateRequest{ Project: testIdentifier.Project, Domain: testIdentifier.Domain, Name: testIdentifier.Name, @@ -90,11 +90,11 @@ func TestResolveKickoffTimeArg(t *testing.T) { }, } testExecutor := newWorkflowExecutorForTest(nil, nil, nil) - err := testExecutor.resolveKickoffTimeArg(scheduleRequest, launchPlan, &executionRequest) + err := testExecutor.resolveKickoffTimeArg(scheduleRequest, launchPlan, executionRequest) assert.Nil(t, err) assert.Contains(t, executionRequest.Inputs.Literals, testKickoffTime) assert.Equal(t, testKickoffTimeProtoLiteral, - *executionRequest.Inputs.Literals[testKickoffTime]) + executionRequest.Inputs.Literals[testKickoffTime]) } func TestResolveKickoffTimeArg_NoKickoffTimeArg(t *testing.T) { @@ -102,7 +102,7 @@ func TestResolveKickoffTimeArg_NoKickoffTimeArg(t *testing.T) { KickoffTimeArg: testKickoffTime, KickoffTime: testKickoffTimestamp, } - launchPlan := admin.LaunchPlan{ + launchPlan := &admin.LaunchPlan{ Closure: &admin.LaunchPlanClosure{ ExpectedInputs: &core.ParameterMap{ Parameters: map[string]*core.Parameter{ @@ -111,7 +111,7 @@ func TestResolveKickoffTimeArg_NoKickoffTimeArg(t *testing.T) { }, }, } - executionRequest := admin.ExecutionCreateRequest{ + executionRequest := &admin.ExecutionCreateRequest{ Project: testIdentifier.Project, Domain: testIdentifier.Domain, Name: testIdentifier.Name, @@ -120,7 +120,7 @@ func TestResolveKickoffTimeArg_NoKickoffTimeArg(t *testing.T) { }, } testExecutor := newWorkflowExecutorForTest(nil, nil, nil) - err := testExecutor.resolveKickoffTimeArg(scheduleRequest, launchPlan, &executionRequest) + err := testExecutor.resolveKickoffTimeArg(scheduleRequest, launchPlan, executionRequest) assert.Nil(t, err) assert.NotContains(t, executionRequest.Inputs.Literals, testKickoffTime) } @@ -140,7 +140,7 @@ func TestGetActiveLaunchPlanVersion(t *testing.T) { launchPlanManager := mocks.NewMockLaunchPlanManager() launchPlanManager.(*mocks.MockLaunchPlanManager).SetListLaunchPlansCallback( - func(ctx context.Context, request admin.ResourceListRequest) ( + func(ctx context.Context, request *admin.ResourceListRequest) ( *admin.LaunchPlanList, error) { assert.True(t, proto.Equal(launchPlanNamedIdentifier, request.Id)) assert.Equal(t, "eq(state,1)", request.Filters) @@ -169,7 +169,7 @@ func TestGetActiveLaunchPlanVersion_ManagerError(t *testing.T) { expectedErr := errors.New("expected error") launchPlanManager := mocks.NewMockLaunchPlanManager() launchPlanManager.(*mocks.MockLaunchPlanManager).SetListLaunchPlansCallback( - func(ctx context.Context, request admin.ResourceListRequest) ( + func(ctx context.Context, request *admin.ResourceListRequest) ( *admin.LaunchPlanList, error) { return nil, expectedErr }) @@ -185,7 +185,7 @@ func TestFormulateExecutionCreateRequest(t *testing.T) { Name: "baz", Version: "12345", } - launchPlan := admin.LaunchPlan{ + launchPlan := &admin.LaunchPlan{ Spec: &admin.LaunchPlanSpec{ WorkflowId: &core.Identifier{ Project: "project", @@ -232,21 +232,21 @@ func TestRun(t *testing.T) { testExecutionManager := mocks.MockExecutionManager{} var messagesSeen int testExecutionManager.SetCreateCallback(func( - ctx context.Context, request admin.ExecutionCreateRequest, requestedAt time.Time) ( + ctx context.Context, request *admin.ExecutionCreateRequest, requestedAt time.Time) ( *admin.ExecutionCreateResponse, error) { assert.Equal(t, "project", request.Project) assert.Equal(t, "domain", request.Domain) assert.Equal(t, "ar8fphnlc5wh9dksjncj", request.Name) if messagesSeen == 0 { assert.Contains(t, request.Inputs.Literals, testKickoffTime) - assert.Equal(t, testKickoffTimeProtoLiteral, *request.Inputs.Literals[testKickoffTime]) + assert.Equal(t, testKickoffTimeProtoLiteral, request.Inputs.Literals[testKickoffTime]) } messagesSeen++ return &admin.ExecutionCreateResponse{}, nil }) launchPlanManager := mocks.NewMockLaunchPlanManager() launchPlanManager.(*mocks.MockLaunchPlanManager).SetListLaunchPlansCallback( - func(ctx context.Context, request admin.ResourceListRequest) ( + func(ctx context.Context, request *admin.ResourceListRequest) ( *admin.LaunchPlanList, error) { assert.Equal(t, "project", request.Id.Project) assert.Equal(t, "domain", request.Id.Domain) diff --git a/flyteadmin/pkg/async/schedule/interfaces/event_scheduler.go b/flyteadmin/pkg/async/schedule/interfaces/event_scheduler.go index 8502eae1672..dad2574bd22 100644 --- a/flyteadmin/pkg/async/schedule/interfaces/event_scheduler.go +++ b/flyteadmin/pkg/async/schedule/interfaces/event_scheduler.go @@ -11,9 +11,9 @@ import ( type AddScheduleInput struct { // Defines the unique identifier associated with the schedule - Identifier core.Identifier + Identifier *core.Identifier // Defines the schedule expression. - ScheduleExpression admin.Schedule + ScheduleExpression *admin.Schedule // Message payload encoded as an CloudWatch event rule InputTemplate. Payload *string // Optional: The application-wide prefix to be applied for schedule names. @@ -22,7 +22,7 @@ type AddScheduleInput struct { type RemoveScheduleInput struct { // Defines the unique identifier associated with the schedule - Identifier core.Identifier + Identifier *core.Identifier // Optional: The application-wide prefix to be applied for schedule names. ScheduleNamePrefix string } @@ -32,7 +32,7 @@ type EventScheduler interface { AddSchedule(ctx context.Context, input AddScheduleInput) error // CreateScheduleInput using the scheduler config and launch plan identifier and schedule - CreateScheduleInput(ctx context.Context, appConfig *appInterfaces.SchedulerConfig, identifier core.Identifier, + CreateScheduleInput(ctx context.Context, appConfig *appInterfaces.SchedulerConfig, identifier *core.Identifier, schedule *admin.Schedule) (AddScheduleInput, error) // Removes an existing schedule. diff --git a/flyteadmin/pkg/async/schedule/mocks/mock_event_scheduler.go b/flyteadmin/pkg/async/schedule/mocks/mock_event_scheduler.go index a1bcc9bee73..fb9aebe34e4 100644 --- a/flyteadmin/pkg/async/schedule/mocks/mock_event_scheduler.go +++ b/flyteadmin/pkg/async/schedule/mocks/mock_event_scheduler.go @@ -18,15 +18,15 @@ type MockEventScheduler struct { } func (s *MockEventScheduler) CreateScheduleInput(ctx context.Context, appConfig *runtimeInterfaces.SchedulerConfig, - identifier core.Identifier, schedule *admin.Schedule) (interfaces.AddScheduleInput, error) { + identifier *core.Identifier, schedule *admin.Schedule) (interfaces.AddScheduleInput, error) { payload, _ := aws.SerializeScheduleWorkflowPayload( schedule.GetKickoffTimeInputArg(), - admin.NamedEntityIdentifier{ + &admin.NamedEntityIdentifier{ Project: identifier.Project, Domain: identifier.Domain, Name: identifier.Name, }) - return interfaces.AddScheduleInput{Identifier: identifier, ScheduleExpression: *schedule, Payload: payload}, nil + return interfaces.AddScheduleInput{Identifier: identifier, ScheduleExpression: schedule, Payload: payload}, nil } func (s *MockEventScheduler) AddSchedule(ctx context.Context, input interfaces.AddScheduleInput) error { diff --git a/flyteadmin/pkg/async/schedule/noop/event_scheduler.go b/flyteadmin/pkg/async/schedule/noop/event_scheduler.go index ed058586072..1a6ac3c7d23 100644 --- a/flyteadmin/pkg/async/schedule/noop/event_scheduler.go +++ b/flyteadmin/pkg/async/schedule/noop/event_scheduler.go @@ -13,7 +13,7 @@ import ( type EventScheduler struct{} -func (s *EventScheduler) CreateScheduleInput(ctx context.Context, appConfig *runtimeInterfaces.SchedulerConfig, identifier core.Identifier, schedule *admin.Schedule) (interfaces.AddScheduleInput, error) { +func (s *EventScheduler) CreateScheduleInput(ctx context.Context, appConfig *runtimeInterfaces.SchedulerConfig, identifier *core.Identifier, schedule *admin.Schedule) (interfaces.AddScheduleInput, error) { panic("implement me") } diff --git a/flyteadmin/pkg/common/flyte_url.go b/flyteadmin/pkg/common/flyte_url.go index 49ba984cd58..f5245ac2387 100644 --- a/flyteadmin/pkg/common/flyte_url.go +++ b/flyteadmin/pkg/common/flyte_url.go @@ -125,7 +125,7 @@ func ParseFlyteURLToExecution(flyteURL string) (ParsedExecution, error) { } -func FlyteURLsFromNodeExecutionID(nodeExecutionID core.NodeExecutionIdentifier, deck bool) *admin.FlyteURLs { +func FlyteURLsFromNodeExecutionID(nodeExecutionID *core.NodeExecutionIdentifier, deck bool) *admin.FlyteURLs { base := fmt.Sprintf("flyte://v1/%s/%s/%s/%s", nodeExecutionID.ExecutionId.Project, nodeExecutionID.ExecutionId.Domain, nodeExecutionID.ExecutionId.Name, nodeExecutionID.NodeId) @@ -142,7 +142,7 @@ func FlyteURLsFromNodeExecutionID(nodeExecutionID core.NodeExecutionIdentifier, // FlyteURLKeyFromNodeExecutionID is a modified version of the function above. // This constructs a fully unique prefix, and when post-pended with the output name, forms a fully unique name for // the artifact service (including the project/domain of course, which the artifact service will add). -func FlyteURLKeyFromNodeExecutionID(nodeExecutionID core.NodeExecutionIdentifier) string { +func FlyteURLKeyFromNodeExecutionID(nodeExecutionID *core.NodeExecutionIdentifier) string { res := fmt.Sprintf("%s/%s", nodeExecutionID.ExecutionId.Name, nodeExecutionID.NodeId) return res @@ -150,13 +150,13 @@ func FlyteURLKeyFromNodeExecutionID(nodeExecutionID core.NodeExecutionIdentifier // FlyteURLKeyFromNodeExecutionIDRetry is a modified version of the function above. // See the uniqueness comment above. -func FlyteURLKeyFromNodeExecutionIDRetry(nodeExecutionID core.NodeExecutionIdentifier, retry int) string { +func FlyteURLKeyFromNodeExecutionIDRetry(nodeExecutionID *core.NodeExecutionIdentifier, retry int) string { res := fmt.Sprintf("%s/%s/%s", nodeExecutionID.ExecutionId.Name, nodeExecutionID.NodeId, strconv.Itoa(retry)) return res } -func FlyteURLsFromTaskExecutionID(taskExecutionID core.TaskExecutionIdentifier, deck bool) *admin.FlyteURLs { +func FlyteURLsFromTaskExecutionID(taskExecutionID *core.TaskExecutionIdentifier, deck bool) *admin.FlyteURLs { base := fmt.Sprintf("flyte://v1/%s/%s/%s/%s/%s", taskExecutionID.NodeExecutionId.ExecutionId.Project, taskExecutionID.NodeExecutionId.ExecutionId.Domain, taskExecutionID.NodeExecutionId.ExecutionId.Name, taskExecutionID.NodeExecutionId.NodeId, strconv.Itoa(int(taskExecutionID.RetryAttempt))) diff --git a/flyteadmin/pkg/common/flyte_url_test.go b/flyteadmin/pkg/common/flyte_url_test.go index a3d31414598..a0cbfcda2b4 100644 --- a/flyteadmin/pkg/common/flyte_url_test.go +++ b/flyteadmin/pkg/common/flyte_url_test.go @@ -10,7 +10,7 @@ import ( func TestFlyteURLsFromNodeExecutionID(t *testing.T) { t.Run("with deck", func(t *testing.T) { - ne := core.NodeExecutionIdentifier{ + ne := &core.NodeExecutionIdentifier{ NodeId: "n0-dn0-n1", ExecutionId: &core.WorkflowExecutionIdentifier{ Project: "fs", @@ -25,7 +25,7 @@ func TestFlyteURLsFromNodeExecutionID(t *testing.T) { }) t.Run("without deck", func(t *testing.T) { - ne := core.NodeExecutionIdentifier{ + ne := &core.NodeExecutionIdentifier{ NodeId: "n0-dn0-n1", ExecutionId: &core.WorkflowExecutionIdentifier{ Project: "fs", @@ -42,7 +42,7 @@ func TestFlyteURLsFromNodeExecutionID(t *testing.T) { func TestFlyteURLsFromTaskExecutionID(t *testing.T) { t.Run("with deck", func(t *testing.T) { - te := core.TaskExecutionIdentifier{ + te := &core.TaskExecutionIdentifier{ TaskId: &core.Identifier{ ResourceType: core.ResourceType_TASK, Project: "fs", @@ -67,7 +67,7 @@ func TestFlyteURLsFromTaskExecutionID(t *testing.T) { }) t.Run("without deck", func(t *testing.T) { - te := core.TaskExecutionIdentifier{ + te := &core.TaskExecutionIdentifier{ TaskId: &core.Identifier{ ResourceType: core.ResourceType_TASK, Project: "fs", diff --git a/flyteadmin/pkg/data/implementations/aws_remote_url.go b/flyteadmin/pkg/data/implementations/aws_remote_url.go index aa71309dea0..db8af08d493 100644 --- a/flyteadmin/pkg/data/implementations/aws_remote_url.go +++ b/flyteadmin/pkg/data/implementations/aws_remote_url.go @@ -52,12 +52,12 @@ func (a *AWSRemoteURL) splitURI(ctx context.Context, uri string) (AWSS3Object, e }, nil } -func (a *AWSRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { +func (a *AWSRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { logger.Debugf(ctx, "Getting signed url for - %s", uri) s3URI, err := a.splitURI(ctx, uri) if err != nil { logger.Debugf(ctx, "failed to extract s3 bucket and key from uri: %s", uri) - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid uri: %s", uri) + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid uri: %s", uri) } // First, get the size of the url blob. headResult, err := a.s3Client.HeadObject(&s3.HeadObjectInput{ @@ -66,7 +66,7 @@ func (a *AWSRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, erro }) if err != nil { logger.Debugf(ctx, "failed to get object size for %s with %v", uri, err) - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf( + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf( codes.Internal, "failed to get object size for %s with %v", uri, err) } @@ -79,14 +79,14 @@ func (a *AWSRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, erro if err != nil { logger.Warning(ctx, "failed to presign url for uri [%s] for %v with err %v", uri, a.presignDuration, err) - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to presign url for uri [%s] for %v with err %v", uri, a.presignDuration, err) } var contentLength int64 if headResult.ContentLength != nil { contentLength = *headResult.ContentLength } - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: urlStr, Bytes: contentLength, }, nil diff --git a/flyteadmin/pkg/data/implementations/gcp_remote_url.go b/flyteadmin/pkg/data/implementations/gcp_remote_url.go index 79bd8b29a9c..3a8dc98679f 100644 --- a/flyteadmin/pkg/data/implementations/gcp_remote_url.go +++ b/flyteadmin/pkg/data/implementations/gcp_remote_url.go @@ -118,19 +118,19 @@ func (g *GCPRemoteURL) signURL(ctx context.Context, gcsURI GCPGCSObject) (string return gcs.SignedURL(gcsURI.bucket, gcsURI.object, opts) } -func (g *GCPRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { +func (g *GCPRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { logger.Debugf(ctx, "Getting signed url for - %s", uri) gcsURI, err := g.splitURI(ctx, uri) if err != nil { logger.Debugf(ctx, "failed to extract gcs bucket and object from uri: %s", uri) - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid uri: %s", uri) + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid uri: %s", uri) } // First, get the size of the url blob. attrs, err := g.gcsClient.Bucket(gcsURI.bucket).Object(gcsURI.object).Attrs(ctx) if err != nil { logger.Debugf(ctx, "failed to get object size for %s with %v", uri, err) - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf( + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf( codes.Internal, "failed to get object size for %s with %v", uri, err) } @@ -138,10 +138,10 @@ func (g *GCPRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, erro if err != nil { logger.Warning(ctx, "failed to presign url for uri [%s] for %v with err %v", uri, g.signDuration, err) - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to presign url for uri [%s] for %v with err %v", uri, g.signDuration, err) } - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: urlStr, Bytes: attrs.Size, }, nil diff --git a/flyteadmin/pkg/data/implementations/noop_remote_url.go b/flyteadmin/pkg/data/implementations/noop_remote_url.go index 9e623550dec..520b2adf5e3 100644 --- a/flyteadmin/pkg/data/implementations/noop_remote_url.go +++ b/flyteadmin/pkg/data/implementations/noop_remote_url.go @@ -16,13 +16,13 @@ type NoopRemoteURL struct { remoteDataStoreClient storage.DataStore } -func (n *NoopRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { +func (n *NoopRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { metadata, err := n.remoteDataStoreClient.Head(ctx, storage.DataReference(uri)) if err != nil { - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to get metadata for uri: %s with err: %v", uri, err) } - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: uri, Bytes: metadata.Size(), }, nil diff --git a/flyteadmin/pkg/data/interfaces/remote.go b/flyteadmin/pkg/data/interfaces/remote.go index 049c9a0465e..a027162ad9c 100644 --- a/flyteadmin/pkg/data/interfaces/remote.go +++ b/flyteadmin/pkg/data/interfaces/remote.go @@ -9,5 +9,5 @@ import ( // Defines an interface for fetching pre-signed URLs. type RemoteURLInterface interface { // TODO: Refactor for URI to be of type DataReference. We should package a FromString-like function in flytestdlib - Get(ctx context.Context, uri string) (admin.UrlBlob, error) + Get(ctx context.Context, uri string) (*admin.UrlBlob, error) } diff --git a/flyteadmin/pkg/data/mocks/remote.go b/flyteadmin/pkg/data/mocks/remote.go index 3f7b0f64e03..25e08ab1caf 100644 --- a/flyteadmin/pkg/data/mocks/remote.go +++ b/flyteadmin/pkg/data/mocks/remote.go @@ -9,14 +9,14 @@ import ( // Mock implementation of a RemoteURLInterface type MockRemoteURL struct { - GetCallback func(ctx context.Context, uri string) (admin.UrlBlob, error) + GetCallback func(ctx context.Context, uri string) (*admin.UrlBlob, error) } -func (m *MockRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { +func (m *MockRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { if m.GetCallback != nil { return m.GetCallback(ctx, uri) } - return admin.UrlBlob{}, nil + return &admin.UrlBlob{}, nil } func NewMockRemoteURL() interfaces.RemoteURLInterface { diff --git a/flyteadmin/pkg/manager/impl/description_entity_manager.go b/flyteadmin/pkg/manager/impl/description_entity_manager.go index 4e0e070ad83..a7affd5e885 100644 --- a/flyteadmin/pkg/manager/impl/description_entity_manager.go +++ b/flyteadmin/pkg/manager/impl/description_entity_manager.go @@ -32,17 +32,17 @@ type DescriptionEntityManager struct { metrics DescriptionEntityMetrics } -func (d *DescriptionEntityManager) GetDescriptionEntity(ctx context.Context, request admin.ObjectGetRequest) ( +func (d *DescriptionEntityManager) GetDescriptionEntity(ctx context.Context, request *admin.ObjectGetRequest) ( *admin.DescriptionEntity, error) { if err := validation.ValidateDescriptionEntityGetRequest(request); err != nil { logger.Errorf(ctx, "invalid request [%+v]: %v", request, err) return nil, err } ctx = contextutils.WithProjectDomain(ctx, request.Id.Project, request.Id.Domain) - return util.GetDescriptionEntity(ctx, d.db, *request.Id) + return util.GetDescriptionEntity(ctx, d.db, request.Id) } -func (d *DescriptionEntityManager) ListDescriptionEntity(ctx context.Context, request admin.DescriptionEntityListRequest) (*admin.DescriptionEntityList, error) { +func (d *DescriptionEntityManager) ListDescriptionEntity(ctx context.Context, request *admin.DescriptionEntityListRequest) (*admin.DescriptionEntityList, error) { // Check required fields if err := validation.ValidateDescriptionEntityListRequest(request); err != nil { return nil, err diff --git a/flyteadmin/pkg/manager/impl/description_entity_manager_test.go b/flyteadmin/pkg/manager/impl/description_entity_manager_test.go index 33cab0350de..dbcab8bdb1f 100644 --- a/flyteadmin/pkg/manager/impl/description_entity_manager_test.go +++ b/flyteadmin/pkg/manager/impl/description_entity_manager_test.go @@ -46,13 +46,13 @@ func TestDescriptionEntityManager_Get(t *testing.T) { repository := getMockRepositoryForDETest() manager := NewDescriptionEntityManager(repository, getMockConfigForDETest(), mockScope.NewTestScope()) - response, err := manager.GetDescriptionEntity(context.Background(), admin.ObjectGetRequest{ + response, err := manager.GetDescriptionEntity(context.Background(), &admin.ObjectGetRequest{ Id: &descriptionEntityIdentifier, }) assert.NoError(t, err) assert.NotNil(t, response) - response, err = manager.GetDescriptionEntity(context.Background(), admin.ObjectGetRequest{ + response, err = manager.GetDescriptionEntity(context.Background(), &admin.ObjectGetRequest{ Id: &badDescriptionEntityIdentifier, }) assert.Error(t, err) @@ -64,7 +64,7 @@ func TestDescriptionEntityManager_List(t *testing.T) { manager := NewDescriptionEntityManager(repository, getMockConfigForDETest(), mockScope.NewTestScope()) t.Run("failed to validate a request", func(t *testing.T) { - response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{ + response, err := manager.ListDescriptionEntity(context.Background(), &admin.DescriptionEntityListRequest{ Id: &admin.NamedEntityIdentifier{ Name: "flyte", }, @@ -74,7 +74,7 @@ func TestDescriptionEntityManager_List(t *testing.T) { }) t.Run("failed to sort description entity", func(t *testing.T) { - response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{ + response, err := manager.ListDescriptionEntity(context.Background(), &admin.DescriptionEntityListRequest{ ResourceType: core.ResourceType_TASK, Id: &admin.NamedEntityIdentifier{ Name: "flyte", @@ -89,7 +89,7 @@ func TestDescriptionEntityManager_List(t *testing.T) { }) t.Run("failed to validate token", func(t *testing.T) { - response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{ + response, err := manager.ListDescriptionEntity(context.Background(), &admin.DescriptionEntityListRequest{ ResourceType: core.ResourceType_TASK, Id: &admin.NamedEntityIdentifier{ Name: "flyte", @@ -104,7 +104,7 @@ func TestDescriptionEntityManager_List(t *testing.T) { }) t.Run("list description entities in the task", func(t *testing.T) { - response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{ + response, err := manager.ListDescriptionEntity(context.Background(), &admin.DescriptionEntityListRequest{ ResourceType: core.ResourceType_TASK, Id: &admin.NamedEntityIdentifier{ Name: "flyte", @@ -118,7 +118,7 @@ func TestDescriptionEntityManager_List(t *testing.T) { }) t.Run("list description entities in the workflow", func(t *testing.T) { - response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{ + response, err := manager.ListDescriptionEntity(context.Background(), &admin.DescriptionEntityListRequest{ ResourceType: core.ResourceType_WORKFLOW, Id: &admin.NamedEntityIdentifier{ Name: "flyte", @@ -132,7 +132,7 @@ func TestDescriptionEntityManager_List(t *testing.T) { }) t.Run("failed to get filter", func(t *testing.T) { - response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{ + response, err := manager.ListDescriptionEntity(context.Background(), &admin.DescriptionEntityListRequest{ ResourceType: core.ResourceType_WORKFLOW, Id: &admin.NamedEntityIdentifier{ Name: "flyte", diff --git a/flyteadmin/pkg/manager/impl/execution_manager.go b/flyteadmin/pkg/manager/impl/execution_manager.go index 13521cedbb9..6ae9a61a521 100644 --- a/flyteadmin/pkg/manager/impl/execution_manager.go +++ b/flyteadmin/pkg/manager/impl/execution_manager.go @@ -105,7 +105,7 @@ func getUser(ctx context.Context) string { } func (m *ExecutionManager) populateExecutionQueue( - ctx context.Context, identifier core.Identifier, compiledWorkflow *core.CompiledWorkflowClosure) { + ctx context.Context, identifier *core.Identifier, compiledWorkflow *core.CompiledWorkflowClosure) { queueConfig := m.queueAllocator.GetQueue(ctx, identifier) for _, task := range compiledWorkflow.Tasks { container := task.Template.GetContainer() @@ -287,7 +287,7 @@ func (m *ExecutionManager) getInheritedExecMetadata(ctx context.Context, request parentNodeExecutionID = parentNodeExecutionModel.ID - sourceExecutionModel, err := util.GetExecutionModel(ctx, m.db, *requestSpec.Metadata.ParentNodeExecution.ExecutionId) + sourceExecutionModel, err := util.GetExecutionModel(ctx, m.db, requestSpec.Metadata.ParentNodeExecution.ExecutionId) if err != nil { logger.Errorf(ctx, "Failed to get workflow execution [%+v] that launched this execution [%+v] with error %v", requestSpec.Metadata.ParentNodeExecution, workflowExecutionID, err) @@ -321,7 +321,7 @@ func (m *ExecutionManager) getInheritedExecMetadata(ctx context.Context, request func (m *ExecutionManager) getExecutionConfig(ctx context.Context, request *admin.ExecutionCreateRequest, launchPlan *admin.LaunchPlan) (*admin.WorkflowExecutionConfig, error) { - workflowExecConfig := admin.WorkflowExecutionConfig{} + workflowExecConfig := &admin.WorkflowExecutionConfig{} // Merge the request spec into workflowExecConfig workflowExecConfig = util.MergeIntoExecConfig(workflowExecConfig, request.Spec) @@ -399,7 +399,7 @@ func (m *ExecutionManager) getExecutionConfig(ctx context.Context, request *admi logger.Infof(ctx, "getting the workflow execution config from application configuration") // Defaults to one from the application config - return &workflowExecConfig, nil + return workflowExecConfig, nil } func (m *ExecutionManager) getClusterAssignment(ctx context.Context, request *admin.ExecutionCreateRequest) ( @@ -428,7 +428,7 @@ func (m *ExecutionManager) getClusterAssignment(ctx context.Context, request *ad } func (m *ExecutionManager) launchSingleTaskExecution( - ctx context.Context, request admin.ExecutionCreateRequest, requestedAt time.Time) ( + ctx context.Context, request *admin.ExecutionCreateRequest, requestedAt time.Time) ( context.Context, *models.Execution, error) { taskModel, err := m.db.TaskRepo().Get(ctx, repositoryInterfaces.Identifier{ @@ -483,12 +483,12 @@ func (m *ExecutionManager) launchSingleTaskExecution( } name := util.GetExecutionName(request) - workflowExecutionID := core.WorkflowExecutionIdentifier{ + workflowExecutionID := &core.WorkflowExecutionIdentifier{ Project: request.Project, Domain: request.Domain, Name: name, } - ctx = getExecutionContext(ctx, &workflowExecutionID) + ctx = getExecutionContext(ctx, workflowExecutionID) namespace := common.GetNamespaceName( m.config.NamespaceMappingConfiguration().GetNamespaceTemplate(), workflowExecutionID.Project, workflowExecutionID.Domain) @@ -501,7 +501,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( // Get the node execution (if any) that launched this execution var parentNodeExecutionID uint var sourceExecutionID uint - parentNodeExecutionID, sourceExecutionID, err = m.getInheritedExecMetadata(ctx, requestSpec, &workflowExecutionID) + parentNodeExecutionID, sourceExecutionID, err = m.getInheritedExecMetadata(ctx, requestSpec, workflowExecutionID) if err != nil { return nil, nil, err } @@ -513,7 +513,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( } // Dynamically assign execution queues. - m.populateExecutionQueue(ctx, *workflow.Id, workflow.Closure.CompiledWorkflow) + m.populateExecutionQueue(ctx, workflow.Id, workflow.Closure.CompiledWorkflow) inputsURI, err := common.OffloadLiteralMap(ctx, m.storageClient, request.Inputs, workflowExecutionID.Project, workflowExecutionID.Domain, workflowExecutionID.Name, shared.Inputs) if err != nil { @@ -523,7 +523,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( if err != nil { return nil, nil, err } - executionConfig, err := m.getExecutionConfig(ctx, &request, nil) + executionConfig, err := m.getExecutionConfig(ctx, request, nil) if err != nil { return nil, nil, err } @@ -548,7 +548,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( rawOutputDataConfig = executionConfig.RawOutputDataConfig } - clusterAssignment, err := m.getClusterAssignment(ctx, &request) + clusterAssignment, err := m.getClusterAssignment(ctx, request) if err != nil { return nil, nil, err } @@ -571,7 +571,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( ExecutionClusterLabel: executionClusterLabel, } - overrides, err := m.addPluginOverrides(ctx, &workflowExecutionID, workflowExecutionID.Name, "") + overrides, err := m.addPluginOverrides(ctx, workflowExecutionID, workflowExecutionID.Name, "") if err != nil { return nil, nil, err } @@ -586,7 +586,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( workflowExecutor := plugins.Get[workflowengineInterfaces.WorkflowExecutor](m.pluginRegistry, plugins.PluginIDWorkflowExecutor) execInfo, err := workflowExecutor.Execute(ctx, workflowengineInterfaces.ExecutionData{ Namespace: namespace, - ExecutionID: &workflowExecutionID, + ExecutionID: workflowExecutionID, ReferenceWorkflowName: workflow.Id.Name, ReferenceLaunchPlanName: launchPlan.Id.Name, WorkflowClosure: workflow.Closure.CompiledWorkflow, @@ -598,7 +598,7 @@ func (m *ExecutionManager) launchSingleTaskExecution( if err != nil { m.systemMetrics.PropellerFailures.Inc() logger.Infof(ctx, "Failed to execute workflow %+v with execution id %+v and inputs %+v with err %v", - request, workflowExecutionID, request.Inputs, err) + request, &workflowExecutionID, request.Inputs, err) return nil, nil, err } executionCreatedAt := time.Now() @@ -692,7 +692,7 @@ func resolveSecurityCtx(ctx context.Context, executionConfigSecurityCtx *core.Se // getStringFromInput should be called when a tag or partition value is a binding to an input. the input is looked up // from the input map and the binding, and an error is returned if the input key is not in the map. -func (m *ExecutionManager) getStringFromInput(ctx context.Context, inputBinding core.InputBindingData, inputs map[string]*core.Literal) (string, error) { +func (m *ExecutionManager) getStringFromInput(ctx context.Context, inputBinding *core.InputBindingData, inputs map[string]*core.Literal) (string, error) { inputName := inputBinding.GetVar() if inputName == "" { @@ -732,7 +732,7 @@ func (m *ExecutionManager) getLabelValue(ctx context.Context, l *core.LabelValue return "", errors.NewFlyteAdminErrorf(codes.InvalidArgument, "label value is nil") } if l.GetInputBinding() != nil { - return m.getStringFromInput(ctx, *l.GetInputBinding(), inputs) + return m.getStringFromInput(ctx, l.GetInputBinding(), inputs) } if l.GetStaticValue() != "" { return l.GetStaticValue(), nil @@ -740,7 +740,7 @@ func (m *ExecutionManager) getLabelValue(ctx context.Context, l *core.LabelValue return "", errors.NewFlyteAdminErrorf(codes.InvalidArgument, "label value is empty") } -func (m *ExecutionManager) fillInTemplateArgs(ctx context.Context, query core.ArtifactQuery, inputs map[string]*core.Literal) (core.ArtifactQuery, error) { +func (m *ExecutionManager) fillInTemplateArgs(ctx context.Context, query *core.ArtifactQuery, inputs map[string]*core.Literal) (*core.ArtifactQuery, error) { if query.GetUri() != "" { // If a query string, then just pass it through, nothing to fill in. return query, nil @@ -805,7 +805,7 @@ func (m *ExecutionManager) fillInTemplateArgs(ctx context.Context, query core.Ar } } - return core.ArtifactQuery{ + return &core.ArtifactQuery{ Identifier: &core.ArtifactQuery_ArtifactId{ ArtifactId: &core.ArtifactID{ ArtifactKey: &core.ArtifactKey{ @@ -825,7 +825,7 @@ func (m *ExecutionManager) fillInTemplateArgs(ctx context.Context, query core.Ar } func (m *ExecutionManager) launchExecutionAndPrepareModel( - ctx context.Context, request admin.ExecutionCreateRequest, requestedAt time.Time) ( + ctx context.Context, request *admin.ExecutionCreateRequest, requestedAt time.Time) ( context.Context, *models.Execution, []*models.ExecutionTag, error) { err := validation.ValidateExecutionRequest(ctx, request, m.db, m.config.ApplicationConfiguration()) @@ -840,7 +840,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( return ctx, model, nil, err } - launchPlanModel, err := util.GetLaunchPlanModel(ctx, m.db, *request.Spec.LaunchPlan) + launchPlanModel, err := util.GetLaunchPlanModel(ctx, m.db, request.Spec.LaunchPlan) if err != nil { logger.Debugf(ctx, "Failed to get launch plan model for ExecutionCreateRequest %+v with err %v", request, err) return nil, nil, nil, err @@ -870,7 +870,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( return nil, nil, nil, err } - workflowModel, err := util.GetWorkflowModel(ctx, m.db, *launchPlan.Spec.WorkflowId) + workflowModel, err := util.GetWorkflowModel(ctx, m.db, launchPlan.Spec.WorkflowId) if err != nil { logger.Debugf(ctx, "Failed to get workflow with id %+v with err %v", launchPlan.Spec.WorkflowId, err) return nil, nil, nil, err @@ -889,12 +889,12 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( workflow.Closure = closure name := util.GetExecutionName(request) - workflowExecutionID := core.WorkflowExecutionIdentifier{ + workflowExecutionID := &core.WorkflowExecutionIdentifier{ Project: request.Project, Domain: request.Domain, Name: name, } - ctx = getExecutionContext(ctx, &workflowExecutionID) + ctx = getExecutionContext(ctx, workflowExecutionID) var requestSpec = request.Spec if requestSpec.Metadata == nil { requestSpec.Metadata = &admin.ExecutionMetadata{} @@ -905,7 +905,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( // Get the node and parent execution (if any) that launched this execution var parentNodeExecutionID uint var sourceExecutionID uint - parentNodeExecutionID, sourceExecutionID, err = m.getInheritedExecMetadata(ctx, requestSpec, &workflowExecutionID) + parentNodeExecutionID, sourceExecutionID, err = m.getInheritedExecMetadata(ctx, requestSpec, workflowExecutionID) if err != nil { return nil, nil, nil, err } @@ -917,7 +917,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( } // Dynamically assign execution queues. - m.populateExecutionQueue(ctx, *workflow.Id, workflow.Closure.CompiledWorkflow) + m.populateExecutionQueue(ctx, workflow.Id, workflow.Closure.CompiledWorkflow) inputsURI, err := common.OffloadLiteralMap(ctx, m.storageClient, executionInputs, workflowExecutionID.Project, workflowExecutionID.Domain, workflowExecutionID.Name, shared.Inputs) if err != nil { @@ -928,7 +928,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( return nil, nil, nil, err } - executionConfig, err := m.getExecutionConfig(ctx, &request, launchPlan) + executionConfig, err := m.getExecutionConfig(ctx, request, launchPlan) if err != nil { return nil, nil, nil, err } @@ -953,7 +953,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( rawOutputDataConfig = executionConfig.RawOutputDataConfig } - clusterAssignment, err := m.getClusterAssignment(ctx, &request) + clusterAssignment, err := m.getClusterAssignment(ctx, request) if err != nil { return nil, nil, nil, err } @@ -977,7 +977,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( ExecutionClusterLabel: executionClusterLabel, } - overrides, err := m.addPluginOverrides(ctx, &workflowExecutionID, launchPlan.GetSpec().WorkflowId.Name, launchPlan.Id.Name) + overrides, err := m.addPluginOverrides(ctx, workflowExecutionID, launchPlan.GetSpec().WorkflowId.Name, launchPlan.Id.Name) if err != nil { return nil, nil, nil, err } @@ -1027,7 +1027,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( workflowExecutor := plugins.Get[workflowengineInterfaces.WorkflowExecutor](m.pluginRegistry, plugins.PluginIDWorkflowExecutor) execInfo, execErr := workflowExecutor.Execute(ctx, workflowengineInterfaces.ExecutionData{ Namespace: namespace, - ExecutionID: &workflowExecutionID, + ExecutionID: workflowExecutionID, ReferenceWorkflowName: workflow.Id.Name, ReferenceLaunchPlanName: launchPlan.Id.Name, WorkflowClosure: workflow.Closure.CompiledWorkflow, @@ -1065,7 +1065,7 @@ func (m *ExecutionManager) launchExecutionAndPrepareModel( // Inserts an execution model into the database store and emits platform metrics. func (m *ExecutionManager) createExecutionModel( ctx context.Context, executionModel *models.Execution, executionTagModel []*models.ExecutionTag) (*core.WorkflowExecutionIdentifier, error) { - workflowExecutionIdentifier := core.WorkflowExecutionIdentifier{ + workflowExecutionIdentifier := &core.WorkflowExecutionIdentifier{ Project: executionModel.ExecutionKey.Project, Domain: executionModel.ExecutionKey.Domain, Name: executionModel.ExecutionKey.Name, @@ -1080,11 +1080,11 @@ func (m *ExecutionManager) createExecutionModel( m.systemMetrics.ExecutionsCreated.Inc() m.systemMetrics.SpecSizeBytes.Observe(float64(len(executionModel.Spec))) m.systemMetrics.ClosureSizeBytes.Observe(float64(len(executionModel.Closure))) - return &workflowExecutionIdentifier, nil + return workflowExecutionIdentifier, nil } func (m *ExecutionManager) CreateExecution( - ctx context.Context, request admin.ExecutionCreateRequest, requestedAt time.Time) ( + ctx context.Context, request *admin.ExecutionCreateRequest, requestedAt time.Time) ( *admin.ExecutionCreateResponse, error) { // Prior to flyteidl v0.15.0, Inputs was held in ExecutionSpec. Ensure older clients continue to work. @@ -1108,9 +1108,9 @@ func (m *ExecutionManager) CreateExecution( } func (m *ExecutionManager) RelaunchExecution( - ctx context.Context, request admin.ExecutionRelaunchRequest, requestedAt time.Time) ( + ctx context.Context, request *admin.ExecutionRelaunchRequest, requestedAt time.Time) ( *admin.ExecutionCreateResponse, error) { - existingExecutionModel, err := util.GetExecutionModel(ctx, m.db, *request.Id) + existingExecutionModel, err := util.GetExecutionModel(ctx, m.db, request.Id) if err != nil { logger.Debugf(ctx, "Failed to get execution model for request [%+v] with err %v", request, err) return nil, err @@ -1144,7 +1144,7 @@ func (m *ExecutionManager) RelaunchExecution( executionSpec.OverwriteCache = request.GetOverwriteCache() var executionModel *models.Execution var executionTagModel []*models.ExecutionTag - ctx, executionModel, executionTagModel, err = m.launchExecutionAndPrepareModel(ctx, admin.ExecutionCreateRequest{ + ctx, executionModel, executionTagModel, err = m.launchExecutionAndPrepareModel(ctx, &admin.ExecutionCreateRequest{ Project: request.Id.Project, Domain: request.Id.Domain, Name: request.Name, @@ -1166,9 +1166,9 @@ func (m *ExecutionManager) RelaunchExecution( } func (m *ExecutionManager) RecoverExecution( - ctx context.Context, request admin.ExecutionRecoverRequest, requestedAt time.Time) ( + ctx context.Context, request *admin.ExecutionRecoverRequest, requestedAt time.Time) ( *admin.ExecutionCreateResponse, error) { - existingExecutionModel, err := util.GetExecutionModel(ctx, m.db, *request.Id) + existingExecutionModel, err := util.GetExecutionModel(ctx, m.db, request.Id) if err != nil { logger.Debugf(ctx, "Failed to get execution model for request [%+v] with err %v", request, err) return nil, err @@ -1196,7 +1196,7 @@ func (m *ExecutionManager) RecoverExecution( executionSpec.Metadata.ReferenceExecution = existingExecution.Id var executionModel *models.Execution var executionTagModel []*models.ExecutionTag - ctx, executionModel, executionTagModel, err = m.launchExecutionAndPrepareModel(ctx, admin.ExecutionCreateRequest{ + ctx, executionModel, executionTagModel, err = m.launchExecutionAndPrepareModel(ctx, &admin.ExecutionCreateRequest{ Project: request.Id.Project, Domain: request.Id.Domain, Name: request.Name, @@ -1232,7 +1232,7 @@ func (m *ExecutionManager) emitScheduledWorkflowMetrics( "[%s/%s/%s]", executionModel.Project, executionModel.Domain, executionModel.Name) return } - launchPlan, err := util.GetLaunchPlan(context.Background(), m.db, *execution.Spec.LaunchPlan) + launchPlan, err := util.GetLaunchPlan(context.Background(), m.db, execution.Spec.LaunchPlan) if err != nil { logger.Warningf(context.Background(), "failed to find launch plan when emitting scheduled workflow execution stats with for "+ @@ -1345,7 +1345,7 @@ func (m *ExecutionManager) emitOverallWorkflowExecutionTime( watch.Observe(*executionModel.ExecutionCreatedAt, terminalEventTime) } -func (m *ExecutionManager) CreateWorkflowEvent(ctx context.Context, request admin.WorkflowExecutionEventRequest) ( +func (m *ExecutionManager) CreateWorkflowEvent(ctx context.Context, request *admin.WorkflowExecutionEventRequest) ( *admin.WorkflowExecutionEventResponse, error) { err := validation.ValidateCreateWorkflowEventRequest(request, m.config.ApplicationConfiguration().GetRemoteDataConfig().MaxSizeInBytes) if err != nil { @@ -1356,7 +1356,7 @@ func (m *ExecutionManager) CreateWorkflowEvent(ctx context.Context, request admi logger.Debugf(ctx, "Received workflow execution event for [%+v] transitioning to phase [%v]", request.Event.ExecutionId, request.Event.Phase) - executionModel, err := util.GetExecutionModel(ctx, m.db, *request.Event.ExecutionId) + executionModel, err := util.GetExecutionModel(ctx, m.db, request.Event.ExecutionId) if err != nil { logger.Debugf(ctx, "failed to find execution [%+v] for recorded event [%s]: %v", request.Event.ExecutionId, request.RequestId, err) @@ -1441,14 +1441,14 @@ func (m *ExecutionManager) CreateWorkflowEvent(ctx context.Context, request admi } } - if err := m.eventPublisher.Publish(ctx, proto.MessageName(&request), &request); err != nil { + if err := m.eventPublisher.Publish(ctx, proto.MessageName(request), request); err != nil { m.systemMetrics.PublishEventError.Inc() logger.Infof(ctx, "error publishing event [%+v] with err: [%v]", request.RequestId, err) } go func() { ceCtx := context.TODO() - if err := m.cloudEventPublisher.Publish(ceCtx, proto.MessageName(&request), &request); err != nil { + if err := m.cloudEventPublisher.Publish(ceCtx, proto.MessageName(request), request); err != nil { m.systemMetrics.PublishEventError.Inc() logger.Infof(ctx, "error publishing cloud event [%+v] with err: [%v]", request.RequestId, err) } @@ -1458,13 +1458,13 @@ func (m *ExecutionManager) CreateWorkflowEvent(ctx context.Context, request admi } func (m *ExecutionManager) GetExecution( - ctx context.Context, request admin.WorkflowExecutionGetRequest) (*admin.Execution, error) { + ctx context.Context, request *admin.WorkflowExecutionGetRequest) (*admin.Execution, error) { if err := validation.ValidateWorkflowExecutionIdentifier(request.Id); err != nil { logger.Debugf(ctx, "GetExecution request [%+v] failed validation with err: %v", request, err) return nil, err } ctx = getExecutionContext(ctx, request.Id) - executionModel, err := util.GetExecutionModel(ctx, m.db, *request.Id) + executionModel, err := util.GetExecutionModel(ctx, m.db, request.Id) if err != nil { logger.Debugf(ctx, "Failed to get execution model for request [%+v] with err: %v", request, err) return nil, err @@ -1483,14 +1483,14 @@ func (m *ExecutionManager) GetExecution( return execution, nil } -func (m *ExecutionManager) UpdateExecution(ctx context.Context, request admin.ExecutionUpdateRequest, +func (m *ExecutionManager) UpdateExecution(ctx context.Context, request *admin.ExecutionUpdateRequest, requestedAt time.Time) (*admin.ExecutionUpdateResponse, error) { if err := validation.ValidateWorkflowExecutionIdentifier(request.Id); err != nil { logger.Debugf(ctx, "UpdateExecution request [%+v] failed validation with err: %v", request, err) return nil, err } ctx = getExecutionContext(ctx, request.Id) - executionModel, err := util.GetExecutionModel(ctx, m.db, *request.Id) + executionModel, err := util.GetExecutionModel(ctx, m.db, request.Id) if err != nil { logger.Debugf(ctx, "Failed to get execution model for request [%+v] with err: %v", request, err) return nil, err @@ -1509,9 +1509,9 @@ func (m *ExecutionManager) UpdateExecution(ctx context.Context, request admin.Ex } func (m *ExecutionManager) GetExecutionData( - ctx context.Context, request admin.WorkflowExecutionGetDataRequest) (*admin.WorkflowExecutionGetDataResponse, error) { + ctx context.Context, request *admin.WorkflowExecutionGetDataRequest) (*admin.WorkflowExecutionGetDataResponse, error) { ctx = getExecutionContext(ctx, request.Id) - executionModel, err := util.GetExecutionModel(ctx, m.db, *request.Id) + executionModel, err := util.GetExecutionModel(ctx, m.db, request.Id) if err != nil { logger.Debugf(ctx, "Failed to get execution model for request [%+v] with err: %v", request, err) return nil, err @@ -1565,7 +1565,7 @@ func (m *ExecutionManager) GetExecutionData( } func (m *ExecutionManager) ListExecutions( - ctx context.Context, request admin.ResourceListRequest) (*admin.ExecutionList, error) { + ctx context.Context, request *admin.ResourceListRequest) (*admin.ExecutionList, error) { // Check required fields if err := validation.ValidateResourceListRequest(request); err != nil { logger.Debugf(ctx, "ListExecutions request [%+v] failed validation with err: %v", request, err) @@ -1642,7 +1642,7 @@ func (m *ExecutionManager) ListExecutions( // publishNotifications will only forward major errors because the assumption made is all of the objects // that are being manipulated have already been validated/manipulated by Flyte itself. // Note: This method should be refactored somewhere else once the interaction with pushing to SNS. -func (m *ExecutionManager) publishNotifications(ctx context.Context, request admin.WorkflowExecutionEventRequest, +func (m *ExecutionManager) publishNotifications(ctx context.Context, request *admin.WorkflowExecutionEventRequest, execution models.Execution) error { // Notifications are stored in the Spec object of an admin.Execution object. adminExecution, err := transformers.FromExecutionModel(ctx, execution, transformers.DefaultExecutionTransformerOptions) @@ -1670,7 +1670,7 @@ func (m *ExecutionManager) publishNotifications(ctx context.Context, request adm // Currently all three supported notifications use email underneath to send the notification. // Convert Slack and PagerDuty into an EmailNotification type. - var emailNotification admin.EmailNotification + emailNotification := &admin.EmailNotification{} if notification.GetEmail() != nil { emailNotification.RecipientsEmail = notification.GetEmail().GetRecipientsEmail() } else if notification.GetPagerDuty() != nil { @@ -1692,7 +1692,7 @@ func (m *ExecutionManager) publishNotifications(ctx context.Context, request adm *m.config.ApplicationConfiguration().GetNotificationsConfig(), emailNotification, request, adminExecution) // Errors seen while publishing a message are considered non-fatal to the method and will not result // in the method returning an error. - if err = m.notificationClient.Publish(ctx, proto.MessageName(&emailNotification), email); err != nil { + if err = m.notificationClient.Publish(ctx, proto.MessageName(emailNotification), email); err != nil { m.systemMetrics.PublishNotificationError.Inc() logger.Infof(ctx, "error publishing email notification [%+v] with err: [%v]", notification, err) } @@ -1701,7 +1701,7 @@ func (m *ExecutionManager) publishNotifications(ctx context.Context, request adm } func (m *ExecutionManager) TerminateExecution( - ctx context.Context, request admin.ExecutionTerminateRequest) (*admin.ExecutionTerminateResponse, error) { + ctx context.Context, request *admin.ExecutionTerminateRequest) (*admin.ExecutionTerminateResponse, error) { if err := validation.ValidateWorkflowExecutionIdentifier(request.Id); err != nil { logger.Debugf(ctx, "received terminate execution request: %v with invalid identifier: %v", request, err) return nil, err diff --git a/flyteadmin/pkg/manager/impl/execution_manager_test.go b/flyteadmin/pkg/manager/impl/execution_manager_test.go index 58e50c04447..1cf27130830 100644 --- a/flyteadmin/pkg/manager/impl/execution_manager_test.go +++ b/flyteadmin/pkg/manager/impl/execution_manager_test.go @@ -154,7 +154,7 @@ func getLegacyExecutionRequest() *admin.ExecutionCreateRequest { r := testutils.GetExecutionRequest() r.Spec.Inputs = r.Inputs r.Inputs = nil - return &r + return r } func getMockNamespaceMappingConfig() runtimeInterfaces.NamespaceMappingConfiguration { @@ -190,7 +190,7 @@ func setDefaultLpCallbackForExecTest(repository interfaces.Repository) { }, } - lpSpecBytes, _ := proto.Marshal(&lpSpec) + lpSpecBytes, _ := proto.Marshal(lpSpec) lpClosure := admin.LaunchPlanClosure{ ExpectedInputs: lpSpec.DefaultInputs, } @@ -358,7 +358,7 @@ func TestCreateExecution(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &mockExecutor) qosProvider := &runtimeIFaceMocks.QualityOfServiceConfiguration{} - qosProvider.OnGetTierExecutionValues().Return(map[core.QualityOfService_Tier]core.QualityOfServiceSpec{ + qosProvider.OnGetTierExecutionValues().Return(map[core.QualityOfService_Tier]*core.QualityOfServiceSpec{ core.QualityOfService_HIGH: { QueueingBudget: ptypes.DurationProto(10 * time.Minute), }, @@ -405,7 +405,7 @@ func TestCreateExecutionFromWorkflowNode(t *testing.T) { repository := getMockRepositoryForExecTest() setDefaultLpCallbackForExecTest(repository) - parentNodeExecutionID := core.NodeExecutionIdentifier{ + parentNodeExecutionID := &core.NodeExecutionIdentifier{ ExecutionId: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -459,7 +459,7 @@ func TestCreateExecutionFromWorkflowNode(t *testing.T) { err := proto.Unmarshal(input.Spec, &spec) assert.NoError(t, err) assert.Equal(t, admin.ExecutionMetadata_CHILD_WORKFLOW, spec.Metadata.Mode) - assert.True(t, proto.Equal(&parentNodeExecutionID, spec.Metadata.ParentNodeExecution)) + assert.True(t, proto.Equal(parentNodeExecutionID, spec.Metadata.ParentNodeExecution)) assert.EqualValues(t, input.ParentNodeExecutionID, 1) assert.EqualValues(t, input.SourceExecutionID, 2) assert.Equal(t, 2, int(spec.Metadata.Nesting)) @@ -482,7 +482,7 @@ func TestCreateExecutionFromWorkflowNode(t *testing.T) { request := testutils.GetExecutionRequest() request.Spec.Metadata = &admin.ExecutionMetadata{ Mode: admin.ExecutionMetadata_CHILD_WORKFLOW, - ParentNodeExecution: &parentNodeExecutionID, + ParentNodeExecution: parentNodeExecutionID, } response, err := execManager.CreateExecution(context.Background(), request, requestedAt) assert.Nil(t, err) @@ -636,7 +636,7 @@ func TestCreateExecutionPropellerFailure(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &mockExecutor) qosProvider := &runtimeIFaceMocks.QualityOfServiceConfiguration{} - qosProvider.OnGetTierExecutionValues().Return(map[core.QualityOfService_Tier]core.QualityOfServiceSpec{ + qosProvider.OnGetTierExecutionValues().Return(map[core.QualityOfService_Tier]*core.QualityOfServiceSpec{ core.QualityOfService_HIGH: { QueueingBudget: ptypes.DurationProto(10 * time.Minute), }, @@ -863,7 +863,7 @@ func TestCreateExecutionNoNotifications(t *testing.T) { // CreateExecutionRequest. lpSpec := testutils.GetSampleLpSpecForTest() lpSpec.EntityMetadata.Notifications = nil - lpSpecBytes, _ := proto.Marshal(&lpSpec) + lpSpecBytes, _ := proto.Marshal(lpSpec) lpClosure := admin.LaunchPlanClosure{ ExpectedInputs: lpSpec.DefaultInputs, } @@ -1464,7 +1464,7 @@ func TestRelaunchExecution(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) // Issue request. - response, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + response, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1512,7 +1512,7 @@ func TestRelaunchExecution_GetExistingFailure(t *testing.T) { }) // Issue request. - _, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + _, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1553,7 +1553,7 @@ func TestRelaunchExecution_CreateFailure(t *testing.T) { }) // Issue request. - _, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + _, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1605,7 +1605,7 @@ func TestRelaunchExecutionInterruptibleOverride(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - _, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + _, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1656,7 +1656,7 @@ func TestRelaunchExecutionOverwriteCacheOverride(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - asd, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + asd, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1691,7 +1691,7 @@ func TestRelaunchExecutionOverwriteCacheOverride(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - asd, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + asd, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1726,7 +1726,7 @@ func TestRelaunchExecutionOverwriteCacheOverride(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - asd, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + asd, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1780,7 +1780,7 @@ func TestRelaunchExecutionEnvsOverride(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) - _, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + _, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1829,7 +1829,7 @@ func TestRecoverExecution(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) // Issue request. - response, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + response, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1919,7 +1919,7 @@ func TestRecoverExecution_RecoveredChildNode(t *testing.T) { NodeId: "parent", } repository.NodeExecutionRepo().(*repositoryMocks.MockNodeExecutionRepo).SetGetCallback(func(ctx context.Context, input interfaces.NodeExecutionResource) (models.NodeExecution, error) { - assert.True(t, proto.Equal(&parentNodeExecution, &input.NodeExecutionIdentifier)) + assert.True(t, proto.Equal(&parentNodeExecution, input.NodeExecutionIdentifier)) return models.NodeExecution{ BaseModel: models.BaseModel{ @@ -1929,7 +1929,7 @@ func TestRecoverExecution_RecoveredChildNode(t *testing.T) { }) // Issue request. - response, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + response, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -1977,7 +1977,7 @@ func TestRecoverExecution_GetExistingFailure(t *testing.T) { }) // Issue request. - _, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + _, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -2016,7 +2016,7 @@ func TestRecoverExecution_GetExistingInputsFailure(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) // Issue request. - _, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + _, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -2069,7 +2069,7 @@ func TestRecoverExecutionInterruptibleOverride(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) // Issue request. - response, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + response, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -2130,7 +2130,7 @@ func TestRecoverExecutionOverwriteCacheOverride(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) // Issue request. - response, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + response, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -2192,7 +2192,7 @@ func TestRecoverExecutionEnvsOverride(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) // Issue request. - response, err := execManager.RecoverExecution(context.Background(), admin.ExecutionRecoverRequest{ + response, err := execManager.RecoverExecution(context.Background(), &admin.ExecutionRecoverRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -2259,7 +2259,7 @@ func TestCreateWorkflowEvent(t *testing.T) { return nil } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2307,7 +2307,7 @@ func TestCreateWorkflowEvent_TerminalState(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2347,7 +2347,7 @@ func TestCreateWorkflowEvent_NoRunningToQueued(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2380,7 +2380,7 @@ func TestCreateWorkflowEvent_CurrentlyAborting(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) - req := admin.WorkflowExecutionEventRequest{ + req := &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2421,7 +2421,7 @@ func TestCreateWorkflowEvent_StartedRunning(t *testing.T) { executionGetFunc := makeExecutionGetFunc(t, closureBytes, nil) repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) - closure := admin.ExecutionClosure{ + closure := &admin.ExecutionClosure{ Phase: core.WorkflowExecution_RUNNING, StartedAt: occurredAtProto, UpdatedAt: occurredAtProto, @@ -2430,7 +2430,7 @@ func TestCreateWorkflowEvent_StartedRunning(t *testing.T) { OccurredAt: testutils.MockCreatedAtProto, }, } - closureBytes, _ := proto.Marshal(&closure) + closureBytes, _ := proto.Marshal(closure) updateExecutionFunc := func( context context.Context, execution models.Execution) error { assert.Equal(t, "project", execution.Project) @@ -2448,7 +2448,7 @@ func TestCreateWorkflowEvent_StartedRunning(t *testing.T) { } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetUpdateCallback(updateExecutionFunc) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2496,7 +2496,7 @@ func TestCreateWorkflowEvent_DuplicateRunning(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2539,7 +2539,7 @@ func TestCreateWorkflowEvent_InvalidPhaseChange(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2592,7 +2592,7 @@ func TestCreateWorkflowEvent_ClusterReassignmentOnQueued(t *testing.T) { occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) mockDbEventWriter := &eventWriterMocks.WorkflowExecutionEventWriter{} - request := admin.WorkflowExecutionEventRequest{ + request := &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2628,7 +2628,7 @@ func TestCreateWorkflowEvent_InvalidEvent(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2658,7 +2658,7 @@ func TestCreateWorkflowEvent_UpdateModelError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2693,7 +2693,7 @@ func TestCreateWorkflowEvent_DatabaseGetError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2729,7 +2729,7 @@ func TestCreateWorkflowEvent_DatabaseUpdateError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2775,7 +2775,7 @@ func TestCreateWorkflowEvent_IncompatibleCluster(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) occurredAtTimestamp, _ := ptypes.TimestampProto(occurredAt) - resp, err := execManager.CreateWorkflowEvent(context.Background(), admin.WorkflowExecutionEventRequest{ + resp, err := execManager.CreateWorkflowEvent(context.Background(), &admin.WorkflowExecutionEventRequest{ RequestId: "1", Event: &event.WorkflowExecutionEvent{ ExecutionId: &executionIdentifier, @@ -2832,7 +2832,7 @@ func TestGetExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ + execution, err := execManager.GetExecution(context.Background(), &admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) assert.NoError(t, err) @@ -2855,7 +2855,7 @@ func TestGetExecution_DatabaseError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ + execution, err := execManager.GetExecution(context.Background(), &admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) assert.Nil(t, execution) @@ -2887,7 +2887,7 @@ func TestGetExecution_TransformerError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ + execution, err := execManager.GetExecution(context.Background(), &admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) assert.Nil(t, execution) @@ -2900,7 +2900,7 @@ func TestUpdateExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - _, err := execManager.UpdateExecution(context.Background(), admin.ExecutionUpdateRequest{ + _, err := execManager.UpdateExecution(context.Background(), &admin.ExecutionUpdateRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -2922,7 +2922,7 @@ func TestUpdateExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - updateResponse, err := execManager.UpdateExecution(context.Background(), admin.ExecutionUpdateRequest{ + updateResponse, err := execManager.UpdateExecution(context.Background(), &admin.ExecutionUpdateRequest{ Id: &executionIdentifier, }, time.Now()) assert.NoError(t, err) @@ -2943,7 +2943,7 @@ func TestUpdateExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - updateResponse, err := execManager.UpdateExecution(context.Background(), admin.ExecutionUpdateRequest{ + updateResponse, err := execManager.UpdateExecution(context.Background(), &admin.ExecutionUpdateRequest{ Id: &executionIdentifier, State: admin.ExecutionState_EXECUTION_ARCHIVED, }, time.Now()) @@ -2961,7 +2961,7 @@ func TestUpdateExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - _, err := execManager.UpdateExecution(context.Background(), admin.ExecutionUpdateRequest{ + _, err := execManager.UpdateExecution(context.Background(), &admin.ExecutionUpdateRequest{ Id: &executionIdentifier, State: admin.ExecutionState_EXECUTION_ARCHIVED, }, time.Now()) @@ -2978,7 +2978,7 @@ func TestUpdateExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - _, err := execManager.UpdateExecution(context.Background(), admin.ExecutionUpdateRequest{ + _, err := execManager.UpdateExecution(context.Background(), &admin.ExecutionUpdateRequest{ Id: &executionIdentifier, State: admin.ExecutionState_EXECUTION_ARCHIVED, }, time.Now()) @@ -3049,7 +3049,7 @@ func TestListExecutions(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - executionList, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + executionList, err := execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, Domain: domainValue, @@ -3081,7 +3081,7 @@ func TestListExecutions_MissingParameters(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repositoryMocks.NewMockRepository(), r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - _, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + _, err := execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Domain: domainValue, }, @@ -3090,7 +3090,7 @@ func TestListExecutions_MissingParameters(t *testing.T) { assert.Error(t, err) assert.Equal(t, codes.InvalidArgument, err.(flyteAdminErrors.FlyteAdminError).Code()) - _, err = execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + _, err = execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, }, @@ -3099,7 +3099,7 @@ func TestListExecutions_MissingParameters(t *testing.T) { assert.Error(t, err) assert.Equal(t, codes.InvalidArgument, err.(flyteAdminErrors.FlyteAdminError).Code()) - _, err = execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + _, err = execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, Domain: domainValue, @@ -3120,7 +3120,7 @@ func TestListExecutions_DatabaseError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - _, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + _, err := execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, Domain: domainValue, @@ -3154,7 +3154,7 @@ func TestListExecutions_TransformerError(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - executionList, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + executionList, err := execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, Domain: domainValue, @@ -3192,7 +3192,7 @@ func TestExecutionManager_PublishNotifications(t *testing.T) { } // Currently this doesn't do anything special as the code to invoke pushing to SNS isn't enabled yet. // This sets up the skeleton for it and appeases the go lint overlords. - workflowRequest := admin.WorkflowExecutionEventRequest{ + workflowRequest := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_FAILED, OutputResult: &event.WorkflowExecutionEvent_Error{ @@ -3204,7 +3204,7 @@ func TestExecutionManager_PublishNotifications(t *testing.T) { ExecutionId: &executionIdentifier, }, } - var execClosure = admin.ExecutionClosure{ + var execClosure = &admin.ExecutionClosure{ Notifications: testutils.GetExecutionRequest().Spec.GetNotifications().Notifications, WorkflowId: &core.Identifier{ ResourceType: core.ResourceType_WORKFLOW, @@ -3244,7 +3244,7 @@ func TestExecutionManager_PublishNotifications(t *testing.T) { execClosure.Notifications = append(execClosure.Notifications, extraNotifications[0]) execClosure.Notifications = append(execClosure.Notifications, extraNotifications[1]) - execClosureBytes, _ := proto.Marshal(&execClosure) + execClosureBytes, _ := proto.Marshal(execClosure) executionModel := models.Execution{ ExecutionKey: models.ExecutionKey{ Project: "project", @@ -3273,7 +3273,7 @@ func TestExecutionManager_PublishNotificationsTransformError(t *testing.T) { notificationClient: &mockPublisher, } - workflowRequest := admin.WorkflowExecutionEventRequest{ + workflowRequest := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_FAILED, OutputResult: &event.WorkflowExecutionEvent_Error{ @@ -3331,7 +3331,7 @@ func TestExecutionManager_TestExecutionManager_PublishNotificationsTransformErro } // Currently this doesn't do anything special as the code to invoke pushing to SNS isn't enabled yet. // This sets up the skeleton for it and appeases the go lint overlords. - workflowRequest := admin.WorkflowExecutionEventRequest{ + workflowRequest := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_FAILED, OutputResult: &event.WorkflowExecutionEvent_Error{ @@ -3343,7 +3343,7 @@ func TestExecutionManager_TestExecutionManager_PublishNotificationsTransformErro ExecutionId: &executionIdentifier, }, } - var execClosure = admin.ExecutionClosure{ + var execClosure = &admin.ExecutionClosure{ Notifications: testutils.GetExecutionRequest().Spec.GetNotifications().Notifications, WorkflowId: &core.Identifier{ ResourceType: core.ResourceType_WORKFLOW, @@ -3353,7 +3353,7 @@ func TestExecutionManager_TestExecutionManager_PublishNotificationsTransformErro Version: "wf_version", }, } - execClosureBytes, _ := proto.Marshal(&execClosure) + execClosureBytes, _ := proto.Marshal(execClosure) executionModel := models.Execution{ ExecutionKey: models.ExecutionKey{ Project: "project", @@ -3385,7 +3385,7 @@ func TestExecutionManager_PublishNotificationsNoPhaseMatch(t *testing.T) { } // Currently this doesn't do anything special as the code to invoke pushing to SNS isn't enabled yet. // This sets up the skeleton for it and appeases the go lint overlords. - workflowRequest := admin.WorkflowExecutionEventRequest{ + workflowRequest := &admin.WorkflowExecutionEventRequest{ Event: &event.WorkflowExecutionEvent{ Phase: core.WorkflowExecution_SUCCEEDED, OutputResult: &event.WorkflowExecutionEvent_OutputUri{ @@ -3394,10 +3394,10 @@ func TestExecutionManager_PublishNotificationsNoPhaseMatch(t *testing.T) { ExecutionId: &executionIdentifier, }, } - var execClosure = admin.ExecutionClosure{ + var execClosure = &admin.ExecutionClosure{ Notifications: testutils.GetExecutionRequest().Spec.GetNotifications().Notifications, } - execClosureBytes, _ := proto.Marshal(&execClosure) + execClosureBytes, _ := proto.Marshal(execClosure) executionModel := models.Execution{ ExecutionKey: models.ExecutionKey{ Project: "project", @@ -3460,7 +3460,7 @@ func TestTerminateExecution(t *testing.T) { identity, err := auth.NewIdentityContext("", principal, "", time.Now(), sets.NewString(), nil, nil) assert.NoError(t, err) ctx := identity.WithContext(context.Background()) - resp, err := execManager.TerminateExecution(ctx, admin.ExecutionTerminateRequest{ + resp, err := execManager.TerminateExecution(ctx, &admin.ExecutionTerminateRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -3492,7 +3492,7 @@ func TestTerminateExecution_PropellerError(t *testing.T) { }) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.TerminateExecution(context.Background(), admin.ExecutionTerminateRequest{ + resp, err := execManager.TerminateExecution(context.Background(), &admin.ExecutionTerminateRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -3523,7 +3523,7 @@ func TestTerminateExecution_DatabaseError(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &mockExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.TerminateExecution(context.Background(), admin.ExecutionTerminateRequest{ + resp, err := execManager.TerminateExecution(context.Background(), &admin.ExecutionTerminateRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -3553,7 +3553,7 @@ func TestTerminateExecution_AlreadyTerminated(t *testing.T) { }, nil }) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - resp, err := execManager.TerminateExecution(context.Background(), admin.ExecutionTerminateRequest{ + resp, err := execManager.TerminateExecution(context.Background(), &admin.ExecutionTerminateRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -3601,20 +3601,20 @@ func TestGetExecutionData(t *testing.T) { } mockExecutionRemoteURL := dataMocks.NewMockRemoteURL() mockExecutionRemoteURL.(*dataMocks.MockRemoteURL).GetCallback = func( - ctx context.Context, uri string) (admin.UrlBlob, error) { + ctx context.Context, uri string) (*admin.UrlBlob, error) { if uri == outputURI { - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: "outputs", Bytes: 200, }, nil } else if strings.HasSuffix(uri, shared.Inputs) { - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: "inputs", Bytes: 200, }, nil } - return admin.UrlBlob{}, errors.New("unexpected input") + return &admin.UrlBlob{}, errors.New("unexpected input") } mockStorage := commonMocks.GetMockStorageClient() fullInputs := &core.LiteralMap{ @@ -3645,7 +3645,7 @@ func TestGetExecutionData(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), mockStorage, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - dataResponse, err := execManager.GetExecutionData(context.Background(), admin.WorkflowExecutionGetDataRequest{ + dataResponse, err := execManager.GetExecutionData(context.Background(), &admin.WorkflowExecutionGetDataRequest{ Id: &executionIdentifier, }) assert.Nil(t, err) @@ -3777,7 +3777,7 @@ func TestGetExecution_Legacy(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - execution, err := execManager.GetExecution(context.Background(), admin.WorkflowExecutionGetRequest{ + execution, err := execManager.GetExecution(context.Background(), &admin.WorkflowExecutionGetRequest{ Id: &executionIdentifier, }) assert.NoError(t, err) @@ -3819,20 +3819,20 @@ func TestGetExecutionData_LegacyModel(t *testing.T) { } mockExecutionRemoteURL := dataMocks.NewMockRemoteURL() mockExecutionRemoteURL.(*dataMocks.MockRemoteURL).GetCallback = func( - ctx context.Context, uri string) (admin.UrlBlob, error) { + ctx context.Context, uri string) (*admin.UrlBlob, error) { if uri == outputURI { - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: "outputs", Bytes: 200, }, nil } else if strings.HasSuffix(uri, shared.Inputs) { - return admin.UrlBlob{ + return &admin.UrlBlob{ Url: "inputs", Bytes: 200, }, nil } - return admin.UrlBlob{}, errors.New("unexpected input") + return &admin.UrlBlob{}, errors.New("unexpected input") } repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetGetCallback(executionGetFunc) @@ -3840,7 +3840,7 @@ func TestGetExecutionData_LegacyModel(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), storageClient, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - dataResponse, err := execManager.GetExecutionData(context.Background(), admin.WorkflowExecutionGetDataRequest{ + dataResponse, err := execManager.GetExecutionData(context.Background(), &admin.WorkflowExecutionGetDataRequest{ Id: &executionIdentifier, }) assert.Nil(t, err) @@ -3888,7 +3888,7 @@ func TestCreateExecution_LegacyClient(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &mockExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - response, err := execManager.CreateExecution(context.Background(), *getLegacyExecutionRequest(), requestedAt) + response, err := execManager.CreateExecution(context.Background(), getLegacyExecutionRequest(), requestedAt) assert.Nil(t, err) expectedResponse := &admin.ExecutionCreateResponse{ @@ -3939,7 +3939,7 @@ func TestRelaunchExecution_LegacyModel(t *testing.T) { repository.ExecutionRepo().(*repositoryMocks.MockExecutionRepo).SetCreateCallback(exCreateFunc) // Issue request. - response, err := execManager.RelaunchExecution(context.Background(), admin.ExecutionRelaunchRequest{ + response, err := execManager.RelaunchExecution(context.Background(), &admin.ExecutionRelaunchRequest{ Id: &core.WorkflowExecutionIdentifier{ Project: "project", Domain: "domain", @@ -4031,7 +4031,7 @@ func TestListExecutions_LegacyModel(t *testing.T) { r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &defaultTestExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, nil, nil, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - executionList, err := execManager.ListExecutions(context.Background(), admin.ResourceListRequest{ + executionList, err := execManager.ListExecutions(context.Background(), &admin.ResourceListRequest{ Id: &admin.NamedEntityIdentifier{ Project: projectValue, Domain: domainValue, @@ -4494,7 +4494,7 @@ func TestCreateSingleTaskExecution(t *testing.T) { r := plugins.NewRegistry() r.RegisterDefault(plugins.PluginIDWorkflowExecutor, &mockExecutor) execManager := NewExecutionManager(repository, r, getMockExecutionsConfigProvider(), mockStorage, mockScope.NewTestScope(), mockScope.NewTestScope(), &mockPublisher, mockExecutionRemoteURL, workflowManager, namedEntityManager, nil, nil, &eventWriterMocks.WorkflowExecutionEventWriter{}) - request := admin.ExecutionCreateRequest{ + request := &admin.ExecutionCreateRequest{ Project: "flytekit", Domain: "production", Name: "singletaskexec", @@ -4521,7 +4521,7 @@ func TestCreateSingleTaskExecution(t *testing.T) { } marshaller := jsonpb.Marshaler{} - _, ferr := marshaller.MarshalToString(&request) + _, ferr := marshaller.MarshalToString(request) assert.NoError(t, ferr) // test once to create an initial launchplan @@ -5757,7 +5757,7 @@ func TestQueryTemplate(t *testing.T) { } p := &core.Partitions{Value: pMap} - q := core.ArtifactQuery{ + q := &core.ArtifactQuery{ Identifier: &core.ArtifactQuery_ArtifactId{ ArtifactId: &core.ArtifactID{ ArtifactKey: ak, @@ -5769,7 +5769,7 @@ func TestQueryTemplate(t *testing.T) { filledQuery, err := m.fillInTemplateArgs(ctx, q, otherInputs.Literals) assert.NoError(t, err) - assert.True(t, proto.Equal(&q, &filledQuery)) + assert.True(t, proto.Equal(q, filledQuery)) }) t.Run("template date-times, both in explicit tp and not", func(t *testing.T) { @@ -5779,7 +5779,7 @@ func TestQueryTemplate(t *testing.T) { } p := &core.Partitions{Value: pMap} - q := core.ArtifactQuery{ + q := &core.ArtifactQuery{ Identifier: &core.ArtifactQuery_ArtifactId{ ArtifactId: &core.ArtifactID{ ArtifactKey: ak, @@ -5803,7 +5803,7 @@ func TestQueryTemplate(t *testing.T) { } p := &core.Partitions{Value: pMap} - q := core.ArtifactQuery{ + q := &core.ArtifactQuery{ Identifier: &core.ArtifactQuery_ArtifactId{ ArtifactId: &core.ArtifactID{ ArtifactKey: ak, diff --git a/flyteadmin/pkg/manager/impl/executions/quality_of_service_test.go b/flyteadmin/pkg/manager/impl/executions/quality_of_service_test.go index 86763a672d4..41a04ec2bc1 100644 --- a/flyteadmin/pkg/manager/impl/executions/quality_of_service_test.go +++ b/flyteadmin/pkg/manager/impl/executions/quality_of_service_test.go @@ -38,7 +38,7 @@ func getQualityOfServiceWithDuration(duration time.Duration) *core.QualityOfServ func getMockConfig() runtimeInterfaces.Configuration { mockConfig := mocks.NewMockConfigurationProvider(nil, nil, nil, nil, nil, nil) provider := &runtimeIFaceMocks.QualityOfServiceConfiguration{} - provider.OnGetTierExecutionValues().Return(map[core.QualityOfService_Tier]core.QualityOfServiceSpec{ + provider.OnGetTierExecutionValues().Return(map[core.QualityOfService_Tier]*core.QualityOfServiceSpec{ core.QualityOfService_HIGH: { QueueingBudget: ptypes.DurationProto(10 * time.Minute), }, diff --git a/flyteadmin/pkg/manager/impl/executions/queues.go b/flyteadmin/pkg/manager/impl/executions/queues.go index 5e4706700c0..90a5951a33b 100644 --- a/flyteadmin/pkg/manager/impl/executions/queues.go +++ b/flyteadmin/pkg/manager/impl/executions/queues.go @@ -25,7 +25,7 @@ type queues = []singleQueueConfiguration type queueConfig = map[tag]queues type QueueAllocator interface { - GetQueue(ctx context.Context, identifier core.Identifier) singleQueueConfiguration + GetQueue(ctx context.Context, identifier *core.Identifier) singleQueueConfiguration } type queueAllocatorImpl struct { @@ -52,7 +52,7 @@ func (q *queueAllocatorImpl) refreshExecutionQueues(executionQueues []runtimeInt q.queueConfigMap = queueConfigMap } -func (q *queueAllocatorImpl) GetQueue(ctx context.Context, identifier core.Identifier) singleQueueConfiguration { +func (q *queueAllocatorImpl) GetQueue(ctx context.Context, identifier *core.Identifier) singleQueueConfiguration { // NOTE: If refreshing the execution queues & workflow configs on every call to GetQueue becomes too slow we should // investigate caching the computed queue assignments. executionQueues := q.config.QueueConfiguration().GetExecutionQueues() diff --git a/flyteadmin/pkg/manager/impl/executions/queues_test.go b/flyteadmin/pkg/manager/impl/executions/queues_test.go index 808a482fd35..baa1bef9b96 100644 --- a/flyteadmin/pkg/manager/impl/executions/queues_test.go +++ b/flyteadmin/pkg/manager/impl/executions/queues_test.go @@ -67,22 +67,22 @@ func TestGetQueue(t *testing.T) { queueConfig := singleQueueConfiguration{ DynamicQueue: "queue dynamic", } - assert.Equal(t, queueConfig, queueAllocator.GetQueue(context.Background(), core.Identifier{ + assert.Equal(t, queueConfig, queueAllocator.GetQueue(context.Background(), &core.Identifier{ Project: "project", Domain: "domain", Name: "name", })) - assert.EqualValues(t, singleQueueConfiguration{}, queueAllocator.GetQueue(context.Background(), core.Identifier{ + assert.EqualValues(t, singleQueueConfiguration{}, queueAllocator.GetQueue(context.Background(), &core.Identifier{ Project: "project", Domain: "domain", Name: "name2", })) - assert.EqualValues(t, singleQueueConfiguration{}, queueAllocator.GetQueue(context.Background(), core.Identifier{ + assert.EqualValues(t, singleQueueConfiguration{}, queueAllocator.GetQueue(context.Background(), &core.Identifier{ Project: "project", Domain: "domain2", Name: "name", })) - assert.EqualValues(t, singleQueueConfiguration{}, queueAllocator.GetQueue(context.Background(), core.Identifier{ + assert.EqualValues(t, singleQueueConfiguration{}, queueAllocator.GetQueue(context.Background(), &core.Identifier{ Project: "project2", Domain: "domain", Name: "name", @@ -174,7 +174,7 @@ func TestGetQueueDefaults(t *testing.T) { assert.Equal(t, singleQueueConfiguration{ DynamicQueue: "default dynamic", }, queueAllocator.GetQueue( - context.Background(), core.Identifier{ + context.Background(), &core.Identifier{ Project: "unmatched", Domain: "domain", Name: "workflow", @@ -182,7 +182,7 @@ func TestGetQueueDefaults(t *testing.T) { assert.EqualValues(t, singleQueueConfiguration{ DynamicQueue: "queue1 dynamic", }, queueAllocator.GetQueue( - context.Background(), core.Identifier{ + context.Background(), &core.Identifier{ Project: "project", Domain: "UNMATCHED", Name: "workflow", @@ -190,7 +190,7 @@ func TestGetQueueDefaults(t *testing.T) { assert.EqualValues(t, singleQueueConfiguration{ DynamicQueue: "queue2 dynamic", }, queueAllocator.GetQueue( - context.Background(), core.Identifier{ + context.Background(), &core.Identifier{ Project: "project", Domain: "domain", Name: "UNMATCHED", @@ -198,7 +198,7 @@ func TestGetQueueDefaults(t *testing.T) { assert.Equal(t, singleQueueConfiguration{ DynamicQueue: "queue3 dynamic", }, queueAllocator.GetQueue( - context.Background(), core.Identifier{ + context.Background(), &core.Identifier{ Project: "project", Domain: "domain", Name: "workflow", diff --git a/flyteadmin/pkg/manager/impl/launch_plan_manager.go b/flyteadmin/pkg/manager/impl/launch_plan_manager.go index 57936313e51..74f0571f86b 100644 --- a/flyteadmin/pkg/manager/impl/launch_plan_manager.go +++ b/flyteadmin/pkg/manager/impl/launch_plan_manager.go @@ -52,15 +52,15 @@ func (m *LaunchPlanManager) getNamedEntityContext(ctx context.Context, identifie func (m *LaunchPlanManager) CreateLaunchPlan( ctx context.Context, - request admin.LaunchPlanCreateRequest) (*admin.LaunchPlanCreateResponse, error) { + request *admin.LaunchPlanCreateRequest) (*admin.LaunchPlanCreateResponse, error) { if err := validation.ValidateIdentifier(request.GetSpec().GetWorkflowId(), common.Workflow); err != nil { logger.Debugf(ctx, "Failed to validate provided workflow ID for CreateLaunchPlan with err: %v", err) return nil, err } - workflowModel, err := util.GetWorkflowModel(ctx, m.db, *request.Spec.WorkflowId) + workflowModel, err := util.GetWorkflowModel(ctx, m.db, request.Spec.WorkflowId) if err != nil { logger.Debugf(ctx, "Failed to get workflow with id [%+v] for CreateLaunchPlan with id [%+v] with err %v", - *request.Spec.WorkflowId, request.Id) + request.Spec.WorkflowId, request.Id) return nil, err } var workflowInterface core.TypedInterface @@ -69,7 +69,7 @@ func (m *LaunchPlanManager) CreateLaunchPlan( if err != nil { logger.Errorf(ctx, "Failed to unmarshal TypedInterface for workflow [%+v] with err: %v", - *request.Spec.WorkflowId, err) + request.Spec.WorkflowId, err) return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to unmarshal workflow inputs") } } @@ -79,16 +79,16 @@ func (m *LaunchPlanManager) CreateLaunchPlan( } ctx = getLaunchPlanContext(ctx, request.Id) launchPlan := transformers.CreateLaunchPlan(request, workflowInterface.Outputs) - launchPlanDigest, err := util.GetLaunchPlanDigest(ctx, &launchPlan) + launchPlanDigest, err := util.GetLaunchPlanDigest(ctx, launchPlan) if err != nil { logger.Errorf(ctx, "failed to compute launch plan digest for [%+v] with err: %v", launchPlan.Id, err) return nil, err } - existingLaunchPlanModel, err := util.GetLaunchPlanModel(ctx, m.db, *request.Id) + existingLaunchPlanModel, err := util.GetLaunchPlanModel(ctx, m.db, request.Id) if err == nil { if bytes.Equal(existingLaunchPlanModel.Digest, launchPlanDigest) { - return nil, errors.NewLaunchPlanExistsIdenticalStructureError(ctx, &request) + return nil, errors.NewLaunchPlanExistsIdenticalStructureError(ctx, request) } existingLaunchPlan, transformerErr := transformers.FromLaunchPlanModel(existingLaunchPlanModel) if transformerErr != nil { @@ -96,7 +96,7 @@ func (m *LaunchPlanManager) CreateLaunchPlan( return nil, transformerErr } // A launch plan exists with different structure - return nil, errors.NewLaunchPlanExistsDifferentStructureError(ctx, &request, existingLaunchPlan.Spec, launchPlan.Spec) + return nil, errors.NewLaunchPlanExistsDifferentStructureError(ctx, request, existingLaunchPlan.Spec, launchPlan.Spec) } launchPlanModel, err := @@ -138,7 +138,7 @@ func (m *LaunchPlanManager) updateLaunchPlanModelState(launchPlan *models.Launch return nil } -func isScheduleEmpty(launchPlanSpec admin.LaunchPlanSpec) bool { +func isScheduleEmpty(launchPlanSpec *admin.LaunchPlanSpec) bool { schedule := launchPlanSpec.GetEntityMetadata().GetSchedule() if schedule == nil { return true @@ -155,8 +155,8 @@ func isScheduleEmpty(launchPlanSpec admin.LaunchPlanSpec) bool { return true } -func (m *LaunchPlanManager) enableSchedule(ctx context.Context, launchPlanIdentifier core.Identifier, - launchPlanSpec admin.LaunchPlanSpec) error { +func (m *LaunchPlanManager) enableSchedule(ctx context.Context, launchPlanIdentifier *core.Identifier, + launchPlanSpec *admin.LaunchPlanSpec) error { addScheduleInput, err := m.scheduler.CreateScheduleInput(ctx, m.config.ApplicationConfiguration().GetSchedulerConfig(), launchPlanIdentifier, @@ -169,7 +169,7 @@ func (m *LaunchPlanManager) enableSchedule(ctx context.Context, launchPlanIdenti } func (m *LaunchPlanManager) disableSchedule( - ctx context.Context, launchPlanIdentifier core.Identifier) error { + ctx context.Context, launchPlanIdentifier *core.Identifier) error { return m.scheduler.RemoveSchedule(ctx, scheduleInterfaces.RemoveScheduleInput{ Identifier: launchPlanIdentifier, ScheduleNamePrefix: m.config.ApplicationConfiguration().GetSchedulerConfig().EventSchedulerConfig.ScheduleNamePrefix, @@ -178,21 +178,21 @@ func (m *LaunchPlanManager) disableSchedule( func (m *LaunchPlanManager) updateSchedules( ctx context.Context, newlyActiveLaunchPlan models.LaunchPlan, formerlyActiveLaunchPlan *models.LaunchPlan) error { - var newlyActiveLaunchPlanSpec admin.LaunchPlanSpec - err := proto.Unmarshal(newlyActiveLaunchPlan.Spec, &newlyActiveLaunchPlanSpec) + newlyActiveLaunchPlanSpec := &admin.LaunchPlanSpec{} + err := proto.Unmarshal(newlyActiveLaunchPlan.Spec, newlyActiveLaunchPlanSpec) if err != nil { logger.Errorf(ctx, "failed to unmarshal newly enabled launch plan spec") return errors.NewFlyteAdminErrorf(codes.Internal, "failed to unmarshal newly enabled launch plan spec") } - launchPlanIdentifier := core.Identifier{ + launchPlanIdentifier := &core.Identifier{ Project: newlyActiveLaunchPlan.Project, Domain: newlyActiveLaunchPlan.Domain, Name: newlyActiveLaunchPlan.Name, Version: newlyActiveLaunchPlan.Version, } - var formerlyActiveLaunchPlanSpec admin.LaunchPlanSpec + formerlyActiveLaunchPlanSpec := &admin.LaunchPlanSpec{} if formerlyActiveLaunchPlan != nil { - err = proto.Unmarshal(formerlyActiveLaunchPlan.Spec, &formerlyActiveLaunchPlanSpec) + err = proto.Unmarshal(formerlyActiveLaunchPlan.Spec, formerlyActiveLaunchPlanSpec) if err != nil { return errors.NewFlyteAdminErrorf(codes.Internal, "failed to unmarshal formerly enabled launch plan spec") } @@ -200,7 +200,7 @@ func (m *LaunchPlanManager) updateSchedules( if !isScheduleEmpty(formerlyActiveLaunchPlanSpec) { // Disable previous schedule - formerlyActiveLaunchPlanIdentifier := core.Identifier{ + formerlyActiveLaunchPlanIdentifier := &core.Identifier{ Project: formerlyActiveLaunchPlan.Project, Domain: formerlyActiveLaunchPlan.Domain, Name: formerlyActiveLaunchPlan.Name, @@ -221,13 +221,13 @@ func (m *LaunchPlanManager) updateSchedules( return nil } -func (m *LaunchPlanManager) disableLaunchPlan(ctx context.Context, request admin.LaunchPlanUpdateRequest) ( +func (m *LaunchPlanManager) disableLaunchPlan(ctx context.Context, request *admin.LaunchPlanUpdateRequest) ( *admin.LaunchPlanUpdateResponse, error) { if err := validation.ValidateIdentifier(request.Id, common.LaunchPlan); err != nil { logger.Debugf(ctx, "can't disable launch plan [%+v] with invalid identifier: %v", request.Id, err) return nil, err } - launchPlanModel, err := util.GetLaunchPlanModel(ctx, m.db, *request.Id) + launchPlanModel, err := util.GetLaunchPlanModel(ctx, m.db, request.Id) if err != nil { logger.Debugf(ctx, "couldn't find launch plan [%+v] to disable with err: %v", request.Id, err) return nil, err @@ -247,7 +247,7 @@ func (m *LaunchPlanManager) disableLaunchPlan(ctx context.Context, request admin "failed to unmarshal launch plan spec when disabling schedule for %+v", request.Id) } if launchPlanSpec.EntityMetadata != nil && launchPlanSpec.EntityMetadata.Schedule != nil { - err = m.disableSchedule(ctx, core.Identifier{ + err = m.disableSchedule(ctx, &core.Identifier{ Project: launchPlanModel.Project, Domain: launchPlanModel.Domain, Name: launchPlanModel.Name, @@ -266,7 +266,7 @@ func (m *LaunchPlanManager) disableLaunchPlan(ctx context.Context, request admin return &admin.LaunchPlanUpdateResponse{}, nil } -func (m *LaunchPlanManager) enableLaunchPlan(ctx context.Context, request admin.LaunchPlanUpdateRequest) ( +func (m *LaunchPlanManager) enableLaunchPlan(ctx context.Context, request *admin.LaunchPlanUpdateRequest) ( *admin.LaunchPlanUpdateResponse, error) { newlyActiveLaunchPlanModel, err := m.db.LaunchPlanRepo().Get(ctx, repoInterfaces.Identifier{ Project: request.Id.Project, @@ -329,7 +329,7 @@ func (m *LaunchPlanManager) enableLaunchPlan(ctx context.Context, request admin. } -func (m *LaunchPlanManager) UpdateLaunchPlan(ctx context.Context, request admin.LaunchPlanUpdateRequest) ( +func (m *LaunchPlanManager) UpdateLaunchPlan(ctx context.Context, request *admin.LaunchPlanUpdateRequest) ( *admin.LaunchPlanUpdateResponse, error) { if err := validation.ValidateIdentifier(request.Id, common.LaunchPlan); err != nil { logger.Debugf(ctx, "can't update launch plan [%+v] state, invalid identifier: %v", request.Id, err) @@ -347,17 +347,17 @@ func (m *LaunchPlanManager) UpdateLaunchPlan(ctx context.Context, request admin. } } -func (m *LaunchPlanManager) GetLaunchPlan(ctx context.Context, request admin.ObjectGetRequest) ( +func (m *LaunchPlanManager) GetLaunchPlan(ctx context.Context, request *admin.ObjectGetRequest) ( *admin.LaunchPlan, error) { if err := validation.ValidateIdentifier(request.Id, common.LaunchPlan); err != nil { logger.Debugf(ctx, "can't get launch plan [%+v] with invalid identifier: %v", request.Id, err) return nil, err } ctx = getLaunchPlanContext(ctx, request.Id) - return util.GetLaunchPlan(ctx, m.db, *request.Id) + return util.GetLaunchPlan(ctx, m.db, request.Id) } -func (m *LaunchPlanManager) GetActiveLaunchPlan(ctx context.Context, request admin.ActiveLaunchPlanRequest) ( +func (m *LaunchPlanManager) GetActiveLaunchPlan(ctx context.Context, request *admin.ActiveLaunchPlanRequest) ( *admin.LaunchPlan, error) { if err := validation.ValidateActiveLaunchPlanRequest(request); err != nil { logger.Debugf(ctx, "can't get active launch plan [%+v] with invalid request: %v", request.Id, err) @@ -389,7 +389,7 @@ func (m *LaunchPlanManager) GetActiveLaunchPlan(ctx context.Context, request adm return transformers.FromLaunchPlanModel(output.LaunchPlans[0]) } -func (m *LaunchPlanManager) ListLaunchPlans(ctx context.Context, request admin.ResourceListRequest) ( +func (m *LaunchPlanManager) ListLaunchPlans(ctx context.Context, request *admin.ResourceListRequest) ( *admin.LaunchPlanList, error) { // Check required fields @@ -447,7 +447,7 @@ func (m *LaunchPlanManager) ListLaunchPlans(ctx context.Context, request admin.R }, nil } -func (m *LaunchPlanManager) ListActiveLaunchPlans(ctx context.Context, request admin.ActiveLaunchPlanListRequest) ( +func (m *LaunchPlanManager) ListActiveLaunchPlans(ctx context.Context, request *admin.ActiveLaunchPlanListRequest) ( *admin.LaunchPlanList, error) { // Check required fields @@ -501,7 +501,7 @@ func (m *LaunchPlanManager) ListActiveLaunchPlans(ctx context.Context, request a } // At least project name and domain must be specified along with limit. -func (m *LaunchPlanManager) ListLaunchPlanIds(ctx context.Context, request admin.NamedEntityIdentifierListRequest) ( +func (m *LaunchPlanManager) ListLaunchPlanIds(ctx context.Context, request *admin.NamedEntityIdentifierListRequest) ( *admin.NamedEntityIdentifierList, error) { ctx = contextutils.WithProjectDomain(ctx, request.Project, request.Domain) filters, err := util.GetDbFilters(util.FilterSpec{ diff --git a/flyteadmin/pkg/manager/impl/launch_plan_manager_test.go b/flyteadmin/pkg/manager/impl/launch_plan_manager_test.go index 3e5b36793ef..d40d7c5e1f9 100644 --- a/flyteadmin/pkg/manager/impl/launch_plan_manager_test.go +++ b/flyteadmin/pkg/manager/impl/launch_plan_manager_test.go @@ -32,7 +32,7 @@ import ( var active = int32(admin.LaunchPlanState_ACTIVE) var inactive = int32(admin.LaunchPlanState_INACTIVE) var mockScheduler = mocks.NewMockEventScheduler() -var launchPlanIdentifier = core.Identifier{ +var launchPlanIdentifier = &core.Identifier{ ResourceType: core.ResourceType_LAUNCH_PLAN, Project: project, Domain: domain, @@ -40,7 +40,7 @@ var launchPlanIdentifier = core.Identifier{ Version: version, } -var launchPlanNamedIdentifier = core.Identifier{ +var launchPlanNamedIdentifier = &core.Identifier{ Project: project, Domain: domain, Name: name, @@ -128,8 +128,8 @@ func TestLaunchPlanManager_GetLaunchPlan(t *testing.T) { }, nil } repository.LaunchPlanRepo().(*repositoryMocks.MockLaunchPlanRepo).SetGetCallback(launchPlanGetFunc) - response, err := lpManager.GetLaunchPlan(context.Background(), admin.ObjectGetRequest{ - Id: &launchPlanIdentifier, + response, err := lpManager.GetLaunchPlan(context.Background(), &admin.ObjectGetRequest{ + Id: launchPlanIdentifier, }) assert.NoError(t, err) assert.NotNil(t, response) @@ -183,7 +183,7 @@ func TestLaunchPlanManager_GetActiveLaunchPlan(t *testing.T) { }, nil } repository.LaunchPlanRepo().(*repositoryMocks.MockLaunchPlanRepo).SetListCallback(launchPlanListFunc) - response, err := lpManager.GetActiveLaunchPlan(context.Background(), admin.ActiveLaunchPlanRequest{ + response, err := lpManager.GetActiveLaunchPlan(context.Background(), &admin.ActiveLaunchPlanRequest{ Id: &admin.NamedEntityIdentifier{ Project: lpRequest.Id.Project, Domain: lpRequest.Id.Domain, @@ -203,7 +203,7 @@ func TestLaunchPlanManager_GetActiveLaunchPlan_NoneActive(t *testing.T) { return interfaces.LaunchPlanCollectionOutput{}, nil } repository.LaunchPlanRepo().(*repositoryMocks.MockLaunchPlanRepo).SetListCallback(launchPlanListFunc) - response, err := lpManager.GetActiveLaunchPlan(context.Background(), admin.ActiveLaunchPlanRequest{ + response, err := lpManager.GetActiveLaunchPlan(context.Background(), &admin.ActiveLaunchPlanRequest{ Id: &admin.NamedEntityIdentifier{ Project: lpRequest.Id.Project, Domain: lpRequest.Id.Domain, @@ -217,7 +217,7 @@ func TestLaunchPlanManager_GetActiveLaunchPlan_NoneActive(t *testing.T) { func TestLaunchPlanManager_GetActiveLaunchPlan_InvalidRequest(t *testing.T) { repository := getMockRepositoryForLpTest() lpManager := NewLaunchPlanManager(repository, getMockConfigForLpTest(), mockScheduler, mockScope.NewTestScope()) - response, err := lpManager.GetActiveLaunchPlan(context.Background(), admin.ActiveLaunchPlanRequest{ + response, err := lpManager.GetActiveLaunchPlan(context.Background(), &admin.ActiveLaunchPlanRequest{ Id: &admin.NamedEntityIdentifier{ Domain: domain, Name: name, @@ -394,7 +394,7 @@ func makeLaunchPlanRepoGetCallback(t *testing.T) repositoryMocks.GetLaunchPlanFu func TestEnableSchedule(t *testing.T) { repository := getMockRepositoryForLpTest() mockScheduler := mocks.NewMockEventScheduler() - scheduleExpression := admin.Schedule{ + scheduleExpression := &admin.Schedule{ ScheduleExpression: &admin.Schedule_Rate{ Rate: &admin.FixedRate{ Value: 2, @@ -404,8 +404,8 @@ func TestEnableSchedule(t *testing.T) { } mockScheduler.(*mocks.MockEventScheduler).SetAddScheduleFunc( func(ctx context.Context, input scheduleInterfaces.AddScheduleInput) error { - assert.True(t, proto.Equal(&launchPlanNamedIdentifier, &input.Identifier)) - assert.True(t, proto.Equal(&scheduleExpression, &input.ScheduleExpression)) + assert.True(t, proto.Equal(launchPlanNamedIdentifier, input.Identifier)) + assert.True(t, proto.Equal(scheduleExpression, input.ScheduleExpression)) assert.Equal(t, "{\"time\":