Implementation for node timeout (flyteorg#42)

* Implementation for node timeout

* .

* adding some tests

* bogus change to retrigger travis

* cr feedback

Creating a separate state in the CRD for failure type so we can distinguish between user and system error. For now, we will use it for timeout failures.

* removing failure type and adding TimingOut as a separate phase

* updated mockery

* finalize to abort on timeout

* fixing NodePhaseTimedout

cmd/kubectl-flyte/cmd/printers/node.go

@@ -26,6 +26,8 @@ func ColorizeNodePhase(p v1alpha1.NodePhase) string {
 		return color.YellowString("%s", p.String())
 	case v1alpha1.NodePhaseSucceeded:
 		return color.HiGreenString("%s", p.String())
+	case v1alpha1.NodePhaseTimedOut:
+		return color.HiRedString("%s", p.String())
 	case v1alpha1.NodePhaseFailed:
 		return color.HiRedString("%s", p.String())
 	}

pkg/apis/flyteworkflow/v1alpha1/iface.go

@@ -3,6 +3,8 @@ package v1alpha1
 import (
 	"context"
 
+	"time"
+
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -54,6 +56,8 @@ const (
 	NodePhaseFailed
 	NodePhaseSkipped
 	NodePhaseRetryableFailure
+	NodePhaseTimingOut
+	NodePhaseTimedOut
 )
 
 func (p NodePhase) String() string {
@@ -64,6 +68,10 @@ func (p NodePhase) String() string {
 		return "Queued"
 	case NodePhaseRunning:
 		return "Running"
+	case NodePhaseTimingOut:
+		return "NodePhaseTimingOut"
+	case NodePhaseTimedOut:
+		return "NodePhaseTimedOut"
 	case NodePhaseSucceeding:
 		return "Succeeding"
 	case NodePhaseSucceeded:
@@ -216,6 +224,7 @@ type MutableNodeStatus interface {
 	GetOrCreateDynamicNodeStatus() MutableDynamicNodeStatus
 	GetDynamicNodeStatus() MutableDynamicNodeStatus
 	ClearDynamicNodeStatus()
+	ClearLastAttemptStartedAt()
 }
 
 // Interface for a Node p. This provides a mutable API.
@@ -228,6 +237,7 @@ type ExecutableNodeStatus interface {
 	GetStoppedAt() *metav1.Time
 	GetStartedAt() *metav1.Time
 	GetLastUpdatedAt() *metav1.Time
+	GetLastAttemptStartedAt() *metav1.Time
 	GetParentNodeID() *NodeID
 	GetParentTaskID() *core.TaskExecutionIdentifier
 	GetDataDir() DataReference
@@ -290,6 +300,8 @@ type ExecutableNode interface {
 	GetResources() *v1.ResourceRequirements
 	GetConfig() *v1.ConfigMap
 	GetRetryStrategy() *RetryStrategy
+	GetExecutionDeadline() *time.Duration
+	GetActiveDeadline() *time.Duration
 }
 
 // Interface for the Workflow p. This is the mutable portion for a Workflow