Skip to content

Commit

Permalink
Added support for EnvironmentVariables on ExecutionConfig (flyteorg#558)
Browse files Browse the repository at this point in the history
* added environment variables to ExecutionConfig and propagated to taskExecutionMetadata

Signed-off-by: Daniel Rammer <[email protected]>

* fixed unit tests

Signed-off-by: Daniel Rammer <[email protected]>

* updated flyteplugins

Signed-off-by: Daniel Rammer <[email protected]>

---------

Signed-off-by: Daniel Rammer <[email protected]>
  • Loading branch information
hamersaw authored May 3, 2023
1 parent 0025cbe commit 5b50d88
Show file tree
Hide file tree
Showing 48 changed files with 62 additions and 11 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ require (
github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295
github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
github.com/fatih/color v1.13.0
github.com/flyteorg/flyteidl v1.3.14
github.com/flyteorg/flyteplugins v1.0.52
github.com/flyteorg/flyteidl v1.3.19
github.com/flyteorg/flyteplugins v1.0.56
github.com/flyteorg/flytestdlib v1.0.15
github.com/ghodss/yaml v1.0.0
github.com/go-redis/redis v6.15.7+incompatible
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -260,10 +260,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/flyteorg/flyteidl v1.3.14 h1:o5M0g/r6pXTPu5PEurbYxbQmuOu3hqqsaI2M6uvK0N8=
github.com/flyteorg/flyteidl v1.3.14/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM=
github.com/flyteorg/flyteplugins v1.0.52 h1:AWNrRYgm0bCzOws+bIfJDfPBZqBmTdABxW78r8q3kP4=
github.com/flyteorg/flyteplugins v1.0.52/go.mod h1:ztsonku5fKwyxcIg1k69PTiBVjRI6d3nK5DnC+iwx08=
github.com/flyteorg/flyteidl v1.3.19 h1:i79Dh7UoP8Z4LEJ2ox6jlfZVJtFZ+r4g84CJj1gh22Y=
github.com/flyteorg/flyteidl v1.3.19/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM=
github.com/flyteorg/flyteplugins v1.0.56 h1:kBTDgTpdSi7wcptk4cMwz5vfh1MU82VaUMMboe1InXw=
github.com/flyteorg/flyteplugins v1.0.56/go.mod h1:aFCKSn8TPzxSAILIiogHtUnHlUCN9+y6Vf+r9R4KZDU=
github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0=
github.com/flyteorg/flytestdlib v1.0.15/go.mod h1:ghw/cjY0sEWIIbyCtcJnL/Gt7ZS7gf9SUi0CCPhbz3s=
github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk=
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/flyteworkflow/v1alpha1/execution_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ type ExecutionConfig struct {
Interruptible *bool
// Defines whether a workflow should skip all its cached results and re-compute its output, overwriting any already stored data.
OverwriteCache bool
// Defines a map of environment varable name / value pairs that are applied to all tasks.
EnvironmentVariables map[string]string
}

type TaskPluginOverride struct {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apiVersion: flyte.lyft.com/v1alpha1
executionConfig:
EnvironmentVariables: null
Interruptible: null
MaxParallelism: 0
OverwriteCache: false
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/nodes/dynamic/dynamic_workflow_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t
composedPBStore.OnWriteRawMatch(
mock.MatchedBy(func(ctx context.Context) bool { return true }),
storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb"),
int64(1473),
int64(1501),
storage.Options{},
mock.MatchedBy(func(rdr *bytes.Reader) bool { return true })).Return(errors.New("foo"))

Expand Down
14 changes: 10 additions & 4 deletions pkg/controller/nodes/task/taskexec_context.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ func (te taskExecutionID) GetGeneratedNameWith(minLength, maxLength int) (string

type taskExecutionMetadata struct {
handler.NodeExecutionMetadata
taskExecID taskExecutionID
o pluginCore.TaskOverrides
maxAttempts uint32
platformResources *v1.ResourceRequirements
taskExecID taskExecutionID
o pluginCore.TaskOverrides
maxAttempts uint32
platformResources *v1.ResourceRequirements
environmentVariables map[string]string
}

func (t taskExecutionMetadata) GetTaskExecutionID() pluginCore.TaskExecutionID {
Expand All @@ -81,6 +82,10 @@ func (t taskExecutionMetadata) GetPlatformResources() *v1.ResourceRequirements {
return t.platformResources
}

func (t taskExecutionMetadata) GetEnvironmentVariables() map[string]string {
return t.environmentVariables
}

type taskExecutionContext struct {
handler.NodeExecutionContext
tm taskExecutionMetadata
Expand Down Expand Up @@ -289,6 +294,7 @@ func (t *Handler) newTaskExecutionContext(ctx context.Context, nCtx handler.Node
o: nCtx.Node(),
maxAttempts: maxAttempts,
platformResources: convertTaskResourcesToRequirements(nCtx.ExecutionContext().GetExecutionConfig().TaskResources),
environmentVariables: nCtx.ExecutionContext().GetExecutionConfig().EnvironmentVariables,
},
rm: resourcemanager.GetTaskResourceManager(
t.resourceManager, resourceNamespacePrefix, id),
Expand Down

0 comments on commit 5b50d88

Please sign in to comment.