From 01df846a3b933ccd0d59b7019c7b864734b87837 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Wed, 14 Dec 2022 14:09:56 -0600 Subject: [PATCH 1/6] corrected timestamps for pod plugin Signed-off-by: Dan Rammer --- go/tasks/pluginmachinery/flytek8s/pod_helper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 82f482285..5af867b65 100755 --- a/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -489,13 +489,13 @@ func GetLastTransitionOccurredAt(pod *v1.Pod) v12.Time { var lastTransitionTime v12.Time containerStatuses := append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) for _, containerStatus := range containerStatuses { - if r := containerStatus.LastTerminationState.Running; r != nil { + if r := containerStatus.State.Running; r != nil { if r.StartedAt.Unix() > lastTransitionTime.Unix() { lastTransitionTime = r.StartedAt } } else if r := containerStatus.LastTerminationState.Terminated; r != nil { if r.FinishedAt.Unix() > lastTransitionTime.Unix() { - lastTransitionTime = r.StartedAt + lastTransitionTime = r.FinishedAt } } } From a47017ea2cf60b5dd84b6ee0c00f49d198c7649e Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Thu, 15 Dec 2022 09:46:46 -0600 Subject: [PATCH 2/6] but actually this time Signed-off-by: Dan Rammer --- go/tasks/pluginmachinery/flytek8s/pod_helper.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 5af867b65..48c0a8e3c 100755 --- a/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -487,13 +487,15 @@ func DemystifyFailure(status v1.PodStatus, info pluginsCore.TaskInfo) (pluginsCo func GetLastTransitionOccurredAt(pod *v1.Pod) v12.Time { var lastTransitionTime v12.Time + // TODO @hamersaw - do we want to look at InitContainerStatuses? containerStatuses := append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) for _, containerStatus := range containerStatuses { + // TODO @hamersaw - is ths right? if r := containerStatus.State.Running; r != nil { if r.StartedAt.Unix() > lastTransitionTime.Unix() { lastTransitionTime = r.StartedAt } - } else if r := containerStatus.LastTerminationState.Terminated; r != nil { + } else if r := containerStatus.State.Terminated; r != nil { if r.FinishedAt.Unix() > lastTransitionTime.Unix() { lastTransitionTime = r.FinishedAt } From 0ff3ee922f9a221a6b9b3dc888d7bea093560d37 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 4 Jan 2023 02:38:20 -0600 Subject: [PATCH 3/6] added reported at support Signed-off-by: Daniel Rammer --- go/tasks/pluginmachinery/core/phase.go | 7 +++++-- go/tasks/pluginmachinery/flytek8s/pod_helper.go | 15 +++++++++++++-- go/tasks/plugins/k8s/pod/plugin.go | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/go/tasks/pluginmachinery/core/phase.go b/go/tasks/pluginmachinery/core/phase.go index 9cfdbe2ba..fd8128f3b 100644 --- a/go/tasks/pluginmachinery/core/phase.go +++ b/go/tasks/pluginmachinery/core/phase.go @@ -86,9 +86,12 @@ type ExternalResource struct { type TaskInfo struct { // log information for the task execution Logs []*core.TaskLog - // Set this value to the intended time when the status occurred at. If not provided, will be defaulted to the current - // time at the time of publishing the event. + // This value represents the time the status occurred at. If not provided, it will be defaulted to the time Flyte + // checked the task status. OccurredAt *time.Time + // This value represents the time the status was reported at. If not provided, will be defaulted to the current time + // when Flyte published the event. + ReportedAt *time.Time // Custom Event information that the plugin would like to expose to the front-end CustomInfo *structpb.Struct // A collection of information about external resources launched by this task diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/go/tasks/pluginmachinery/flytek8s/pod_helper.go index b70d003d4..77287e773 100755 --- a/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -508,10 +508,8 @@ func DemystifyFailure(status v1.PodStatus, info pluginsCore.TaskInfo) (pluginsCo func GetLastTransitionOccurredAt(pod *v1.Pod) v12.Time { var lastTransitionTime v12.Time - // TODO @hamersaw - do we want to look at InitContainerStatuses? containerStatuses := append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) for _, containerStatus := range containerStatuses { - // TODO @hamersaw - is ths right? if r := containerStatus.State.Running; r != nil { if r.StartedAt.Unix() > lastTransitionTime.Unix() { lastTransitionTime = r.StartedAt @@ -529,3 +527,16 @@ func GetLastTransitionOccurredAt(pod *v1.Pod) v12.Time { return lastTransitionTime } + +func GetReportedAt(pod *v1.Pod) v12.Time { + var reportedAt v12.Time + for _, condition := range pod.Status.Conditions { + if condition.Reason == "PodCompleted" && condition.Type == v1.PodReady && condition.Status == v1.ConditionFalse { + if condition.LastTransitionTime.Unix() > reportedAt.Unix() { + reportedAt = condition.LastTransitionTime + } + } + } + + return reportedAt +} diff --git a/go/tasks/plugins/k8s/pod/plugin.go b/go/tasks/plugins/k8s/pod/plugin.go index 2b492dc5c..5e475707a 100644 --- a/go/tasks/plugins/k8s/pod/plugin.go +++ b/go/tasks/plugins/k8s/pod/plugin.go @@ -101,8 +101,14 @@ func (plugin) GetTaskPhaseWithLogs(ctx context.Context, pluginContext k8s.Plugin pod := r.(*v1.Pod) transitionOccurredAt := flytek8s.GetLastTransitionOccurredAt(pod).Time + reportedAt := flytek8s.GetReportedAt(pod).Time + if reportedAt.IsZero() { + reportedAt = transitionOccurredAt + } + info := pluginsCore.TaskInfo{ OccurredAt: &transitionOccurredAt, + ReportedAt: &reportedAt, } if pod.Status.Phase != v1.PodPending && pod.Status.Phase != v1.PodUnknown { From 3c9d9d6477eeaf8152b0231418d71a60c645a3b0 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Fri, 24 Feb 2023 09:03:21 -0600 Subject: [PATCH 4/6] fixed merge Signed-off-by: Daniel Rammer --- go/tasks/pluginmachinery/flytek8s/pod_helper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 8c442b591..b63c13728 100755 --- a/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -674,8 +674,8 @@ func GetLastTransitionOccurredAt(pod *v1.Pod) metav1.Time { return lastTransitionTime } -func GetReportedAt(pod *v1.Pod) v12.Time { - var reportedAt v12.Time +func GetReportedAt(pod *v1.Pod) metav1.Time { + var reportedAt metav1.Time for _, condition := range pod.Status.Conditions { if condition.Reason == "PodCompleted" && condition.Type == v1.PodReady && condition.Status == v1.ConditionFalse { if condition.LastTransitionTime.Unix() > reportedAt.Unix() { From 5645009ca10240ad3b6396cb3313787cd6e935cb Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Fri, 17 Mar 2023 11:05:12 -0500 Subject: [PATCH 5/6] updated flyteidl Signed-off-by: Daniel Rammer --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 48e906f32..43c774da5 100644 --- a/go.mod +++ b/go.mod @@ -134,3 +134,5 @@ require ( ) replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d + +replace github.com/flyteorg/flyteidl => github.com/flyteorg/flyteidl v1.3.13-0.20230317155955-7a6b342f2ff7 diff --git a/go.sum b/go.sum index 559ec0ef5..e9ab8866c 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/flyteidl v1.3.6 h1:PI846AdnrQZ84pxRVAzA3WGihv+xXmjQHO91nj/kV9g= -github.com/flyteorg/flyteidl v1.3.6/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= +github.com/flyteorg/flyteidl v1.3.13-0.20230317155955-7a6b342f2ff7 h1:DCU870glnEwVocHOmOlD20xyz3NlKKI5s0Hu+iW5WYA= +github.com/flyteorg/flyteidl v1.3.13-0.20230317155955-7a6b342f2ff7/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0= github.com/flyteorg/flytestdlib v1.0.15/go.mod h1:ghw/cjY0sEWIIbyCtcJnL/Gt7ZS7gf9SUi0CCPhbz3s= github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk= From f5ff949b7414e655bb1c65b22082632acef55050 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 22 Mar 2023 18:30:28 -0500 Subject: [PATCH 6/6] updated flyteidl deps Signed-off-by: Daniel Rammer --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d48dd5352..4d54cbdea 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/athena v1.0.0 github.com/bstadlbauer/dask-k8s-operator-go-client v0.1.0 github.com/coocood/freecache v1.1.1 - github.com/flyteorg/flyteidl v1.3.12 + github.com/flyteorg/flyteidl v1.3.14 github.com/flyteorg/flytestdlib v1.0.15 github.com/go-test/deep v1.0.7 github.com/golang/protobuf v1.5.2 @@ -134,5 +134,3 @@ require ( ) replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d - -replace github.com/flyteorg/flyteidl => github.com/flyteorg/flyteidl v1.3.13-0.20230317155955-7a6b342f2ff7 diff --git a/go.sum b/go.sum index e9ab8866c..179728dec 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/flyteidl v1.3.13-0.20230317155955-7a6b342f2ff7 h1:DCU870glnEwVocHOmOlD20xyz3NlKKI5s0Hu+iW5WYA= -github.com/flyteorg/flyteidl v1.3.13-0.20230317155955-7a6b342f2ff7/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= +github.com/flyteorg/flyteidl v1.3.14 h1:o5M0g/r6pXTPu5PEurbYxbQmuOu3hqqsaI2M6uvK0N8= +github.com/flyteorg/flyteidl v1.3.14/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0= github.com/flyteorg/flytestdlib v1.0.15/go.mod h1:ghw/cjY0sEWIIbyCtcJnL/Gt7ZS7gf9SUi0CCPhbz3s= github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk=