Moving from flytepropeller - Adding flags for ignore-retry-cause and …

…default-max-attempts (#4153) * added flags for ignore-retry-cause and default-max-attempts Signed-off-by: Daniel Rammer <[email protected]> * Switch non interruptible @ unified retry behavior (#610) Signed-off-by: Dennis Keck <[email protected]> * fixed lint issues Signed-off-by: Daniel Rammer <[email protected]> * began unit tests Signed-off-by: Daniel Rammer <[email protected]> * enabling negative interruptibleFailureThreshold Signed-off-by: Daniel Rammer <[email protected]> * fixed unit tests Signed-off-by: Daniel Rammer <[email protected]> * Finish simplify retry behavior PR (#623) * Cleanup retry behavior Signed-off-by: Dennis Keck <[email protected]> * Fix interruptible retry threshold for odl behavior Signed-off-by: Dennis Keck <[email protected]> * Add tests for BuildNodeExecutionContext Signed-off-by: Dennis Keck <[email protected]> * Fix IsElgibileForRetries Tests Signed-off-by: Dennis Keck <[email protected]> --------- Signed-off-by: Dennis Keck <[email protected]> * updated flyteplugins Signed-off-by: Daniel Rammer <[email protected]> * fixed lint issue Signed-off-by: Daniel Rammer <[email protected]> * fixed monorepo transition Signed-off-by: Daniel Rammer <[email protected]> --------- Signed-off-by: Daniel Rammer <[email protected]> Signed-off-by: Dennis Keck <[email protected]> Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Daniel Rammer <[email protected]> Co-authored-by: Dennis Keck <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Oct 6, 2023 · a903dc0 · a903dc0
1 parent def8eb7
commit a903dc0
Show file tree

Hide file tree

Showing 57 changed files with 228 additions and 245 deletions.
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/exec_metadata.go b/flyteplugins/go/tasks/pluginmachinery/core/exec_metadata.go
@@ -44,6 +44,6 @@ type TaskExecutionMetadata interface {
 	GetSecurityContext() core.SecurityContext
 	IsInterruptible() bool
 	GetPlatformResources() *v1.ResourceRequirements
-	GetInterruptibleFailureThreshold() uint32
+	GetInterruptibleFailureThreshold() int32
 	GetEnvironmentVariables() map[string]string
 }
diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_metadata.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_execution_metadata.go
diff --git a/flyteplugins/go/tasks/plugins/array/k8s/subtask_exec_context.go b/flyteplugins/go/tasks/plugins/array/k8s/subtask_exec_context.go
@@ -268,7 +268,7 @@ func NewSubTaskExecutionMetadata(taskExecutionMetadata pluginsCore.TaskExecution
 	}
 
 	subTaskExecutionID := NewSubTaskExecutionID(taskExecutionMetadata.GetTaskExecutionID(), executionIndex, retryAttempt)
-	interruptible := taskExecutionMetadata.IsInterruptible() && uint32(systemFailures) < taskExecutionMetadata.GetInterruptibleFailureThreshold()
+	interruptible := taskExecutionMetadata.IsInterruptible() && int32(systemFailures) < taskExecutionMetadata.GetInterruptibleFailureThreshold()
 	return SubTaskExecutionMetadata{
 		taskExecutionMetadata,
 		utils.UnionMaps(taskExecutionMetadata.GetAnnotations(), secretsMap),

diff --git a/...est/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml b/...est/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml
@@ -75,8 +75,6 @@ spec:
       kind: task
       name: svm_trainer
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.containerization.multi_images.svm_trainer" '
     n1:
       id: n1
@@ -104,8 +102,6 @@ spec:
       kind: task
       name: svm_predictor
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.containerization.multi_images.svm_predictor" '
     start-node:
       id: start-node

diff --git a/...mpiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml b/...mpiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml
@@ -125,8 +125,6 @@ spec:
       kind: task
       name: ellipse-area-metadata-shell
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"ellipse-area-metadata-shell" '
     n1:
       id: n1
@@ -144,8 +142,6 @@ spec:
       kind: task
       name: ellipse-area-metadata-python
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"ellipse-area-metadata-python" '
     n2:
       id: n2
@@ -163,8 +159,6 @@ spec:
       kind: task
       name: ellipse-area-metadata-r
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"ellipse-area-metadata-r" '
     n3:
       id: n3
@@ -182,8 +176,6 @@ spec:
       kind: task
       name: ellipse-area-metadata-haskell
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"ellipse-area-metadata-haskell" '
     n4:
       id: n4
@@ -201,8 +193,6 @@ spec:
       kind: task
       name: ellipse-area-metadata-julia
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"ellipse-area-metadata-julia" '
     n5:
       id: n5
@@ -260,8 +250,6 @@ spec:
       kind: task
       name: report_all_calculated_areas
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.containerization.raw_container.report_all_calculated_areas" '
     start-node:
       id: start-node

diff --git a/...stdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml b/...stdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml
@@ -107,24 +107,18 @@ spec:
       kind: task
       name: secret_task
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.containerization.use_secrets.secret_task" '
     n1:
       id: n1
       kind: task
       name: user_info_task
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.containerization.use_secrets.user_info_task" '
     n2:
       id: n2
       kind: task
       name: secret_file_task
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.containerization.use_secrets.secret_file_task" '
     start-node:
       id: start-node

diff --git a/.../test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml b/.../test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml
@@ -75,16 +75,12 @@ spec:
       kind: task
       name: write
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.chain_tasks.write" '
     n1:
       id: n1
       kind: task
       name: read
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.chain_tasks.read" '
     start-node:
       id: start-node

diff --git a/...piler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml b/...piler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml
@@ -106,8 +106,6 @@ spec:
       kind: branch
       name: fractions
       resources: {}
-      retry:
-        minAttempts: 1
     n0-n0:
       id: n0-n0
       inputBindings:
@@ -119,8 +117,6 @@ spec:
       kind: task
       name: double
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.double" '
     n0-n1:
       id: n0-n1
@@ -133,8 +129,6 @@ spec:
       kind: task
       name: square
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.square" '
     start-node:
       id: start-node

diff --git a/...ler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml b/...ler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml
@@ -128,8 +128,6 @@ spec:
       kind: branch
       name: fractions
       resources: {}
-      retry:
-        minAttempts: 1
     n0-n0:
       id: n0-n0
       inputBindings:
@@ -141,8 +139,6 @@ spec:
       kind: task
       name: double
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.double" '
     n0-n1:
       id: n0-n1
@@ -155,8 +151,6 @@ spec:
       kind: task
       name: square
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.square" '
     start-node:
       id: start-node

diff --git a/...ler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml b/...ler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml
@@ -134,8 +134,6 @@ spec:
       kind: branch
       name: fractions
       resources: {}
-      retry:
-        minAttempts: 1
     n0-n0:
       id: n0-n0
       inputBindings:
@@ -147,8 +145,6 @@ spec:
       kind: task
       name: double
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.double" '
     n0-n1:
       id: n0-n1
@@ -161,8 +157,6 @@ spec:
       kind: task
       name: square
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.square" '
     n1:
       id: n1
@@ -175,8 +169,6 @@ spec:
       kind: task
       name: double
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.double" '
     start-node:
       id: start-node

diff --git a/...test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml b/...test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml
@@ -86,8 +86,6 @@ spec:
       kind: task
       name: coin_toss
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.coin_toss" '
     n1:
       branch:
@@ -111,23 +109,17 @@ spec:
       kind: branch
       name: test
       resources: {}
-      retry:
-        minAttempts: 1
     n1-n0:
       id: n1-n0
       kind: task
       name: success
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.success" '
     n1-n1:
       id: n1-n1
       kind: task
       name: failed
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.failed" '
     start-node:
       id: start-node

diff --git a/...er/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml b/...er/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml
@@ -91,23 +91,17 @@ spec:
       kind: branch
       name: test
       resources: {}
-      retry:
-        minAttempts: 1
     n0-n0:
       id: n0-n0
       kind: task
       name: success
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.success" '
     n0-n1:
       id: n0-n1
       kind: task
       name: failed
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.failed" '
     start-node:
       id: start-node

diff --git a/...est/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml b/...est/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml
@@ -132,8 +132,6 @@ spec:
       kind: branch
       name: fractions
       resources: {}
-      retry:
-        minAttempts: 1
     n0-n0:
       branch:
         elseFail:
@@ -179,8 +177,6 @@ spec:
       kind: branch
       name: inner_fractions
       resources: {}
-      retry:
-        minAttempts: 1
     n0-n0-n0:
       id: n0-n0-n0
       inputBindings:
@@ -192,8 +188,6 @@ spec:
       kind: task
       name: double
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.double" '
     n0-n0-n1:
       id: n0-n0-n1
@@ -206,8 +200,6 @@ spec:
       kind: task
       name: square
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.square" '
     n0-n1:
       id: n0-n1
@@ -220,8 +212,6 @@ spec:
       kind: task
       name: square
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.square" '
     n0-n2:
       id: n0-n2
@@ -234,8 +224,6 @@ spec:
       kind: task
       name: double
       resources: {}
-      retry:
-        minAttempts: 1
       task: 'resource_type:TASK name:"core.control_flow.conditions.double" '
     start-node:
       id: start-node