Skip to content

Commit

Permalink
Merge branch 'master' into code_conduct
Browse files Browse the repository at this point in the history
  • Loading branch information
@vishnuchalla
vishnuchalla authored Jul 28, 2023
2 parents 48685de + 898e485 commit 03d1e08
Showing 1 changed file with 66 additions and 28 deletions.
94 changes: 66 additions & 28 deletions pkg/measurements/pod_latency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,23 +38,28 @@ const (
)

type podMetric struct {
Timestamp time.Time `json:"timestamp"`
scheduled time.Time
SchedulingLatency int `json:"schedulingLatency"`
initialized time.Time
InitializedLatency int `json:"initializedLatency"`
containersReady time.Time
ContainersReadyLatency int `json:"containersReadyLatency"`
podReady time.Time
PodReadyLatency int `json:"podReadyLatency"`
MetricName string `json:"metricName"`
JobName string `json:"jobName"`
JobConfig config.Job `json:"jobConfig"`
UUID string `json:"uuid"`
Namespace string `json:"namespace"`
Name string `json:"podName"`
NodeName string `json:"nodeName"`
Metadata interface{} `json:"metadata,omitempty"`
Timestamp time.Time `json:"timestamp"`
CreationTimestampV2 time.Time `json:"creationTimestampV2"`
scheduled time.Time
SchedulingLatency int `json:"schedulingLatency"`
SchedulingLatencyV2 int `json:"schedulingLatency_v2"`
initialized time.Time
InitializedLatency int `json:"initializedLatency"`
InitializedLatencyV2 int `json:"initializedLatency_v2"`
containersReady time.Time
ContainersReadyLatency int `json:"containersReadyLatency"`
ContainersReadyLatencyV2 int `json:"containersReadyLatency_v2"`
podReady time.Time
PodReadyLatency int `json:"podReadyLatency"`
PodReadyLatencyV2 int `json:"podReadyLatency_v2"`
MetricName string `json:"metricName"`
JobName string `json:"jobName"`
JobConfig config.Job `json:"jobConfig"`
UUID string `json:"uuid"`
Namespace string `json:"namespace"`
Name string `json:"podName"`
NodeName string `json:"nodeName"`
Metadata interface{} `json:"metadata,omitempty"`
}

type podLatency struct {
Expand All @@ -76,14 +81,15 @@ func (p *podLatency) handleCreatePod(obj interface{}) {
if _, exists := p.metrics[string(pod.UID)]; !exists {
if strings.Contains(pod.Namespace, factory.jobConfig.Namespace) {
p.metrics[string(pod.UID)] = podMetric{
Timestamp: now,
Namespace: pod.Namespace,
Name: pod.Name,
MetricName: podLatencyMeasurement,
UUID: globalCfg.UUID,
JobConfig: jobConfig,
JobName: factory.jobConfig.Name,
Metadata: factory.metadata,
Timestamp: now,
CreationTimestampV2: pod.CreationTimestamp.Time.UTC(),
Namespace: pod.Namespace,
Name: pod.Name,
MetricName: podLatencyMeasurement,
UUID: globalCfg.UUID,
JobConfig: jobConfig,
JobName: factory.jobConfig.Name,
Metadata: factory.metadata,
}
}
}
Expand Down Expand Up @@ -202,25 +208,53 @@ func (p *podLatency) normalizeMetrics() {
continue
}
// latencyTime should be always larger than zero, however, in some cases, it might be a
// negative value due to the precision of timestamp can only get to the level of second,
// the microsecond and nanosecond have been discarded purposely in kubelet, this is
// because apiserver does not support RFC339NANO.
// negative value due to the precision of timestamp can only get to the level of second
// and also the creation timestamp we capture using time.Now().UTC() might even have a
// delay over 1s in some cases. The microsecond and nanosecond have been discarded purposely
// in kubelet, this is because apiserver does not support RFC339NANO. The newly introduced
// v2 latencies are currently under AB testing which blindly trust kubernetes as source of
// truth and will prevent us from those over 1s delays as well as <0 cases.

creationTimeStampsDiff := int(m.Timestamp.Sub(m.CreationTimestampV2).Milliseconds())
if creationTimeStampsDiff >= 1000 {
log.Tracef("Difference between V1 and V2 creation timestamps for pod %+v is >=1s and is %+v seconds", m.Name, float64(creationTimeStampsDiff)/1000)
}

m.ContainersReadyLatency = int(m.containersReady.Sub(m.Timestamp).Milliseconds())
m.ContainersReadyLatencyV2 = int(m.containersReady.Sub(m.CreationTimestampV2).Milliseconds())
if m.ContainersReadyLatency < 0 {
log.Tracef("ContainersReadyLatency for pod %+v falling under negative case. So explicitly setting it to 0", m.Name)
m.ContainersReadyLatency = 0
}
log.Tracef("ContainersReadyLatency: %+v for pod %+v", m.ContainersReadyLatency, m.Name)
log.Tracef("ContainersReadyLatencyV2: %+v for pod %+v", m.ContainersReadyLatencyV2, m.Name)

m.SchedulingLatency = int(m.scheduled.Sub(m.Timestamp).Milliseconds())
m.SchedulingLatencyV2 = int(m.scheduled.Sub(m.CreationTimestampV2).Milliseconds())
if m.SchedulingLatency < 0 {
log.Tracef("SchedulingLatency for pod %+v falling under negative case. So explicitly setting it to 0", m.Name)
m.SchedulingLatency = 0
}
log.Tracef("SchedulingLatency: %+v for pod %+v", m.SchedulingLatency, m.Name)
log.Tracef("SchedulingLatencyV2: %+v for pod %+v", m.SchedulingLatencyV2, m.Name)

m.InitializedLatency = int(m.initialized.Sub(m.Timestamp).Milliseconds())
m.InitializedLatencyV2 = int(m.initialized.Sub(m.CreationTimestampV2).Milliseconds())
if m.InitializedLatency < 0 {
log.Tracef("InitializedLatency for pod %+v falling under negative case. So explicitly setting it to 0", m.Name)
m.InitializedLatency = 0
}
log.Tracef("InitializedLatency: %+v for pod %+v", m.InitializedLatency, m.Name)
log.Tracef("InitializedLatencyV2: %+v for pod %+v", m.InitializedLatencyV2, m.Name)

m.PodReadyLatency = int(m.podReady.Sub(m.Timestamp).Milliseconds())
m.PodReadyLatencyV2 = int(m.podReady.Sub(m.CreationTimestampV2).Milliseconds())
if m.PodReadyLatency < 0 {
log.Tracef("PodReadyLatency for pod %+v falling under negative case. So explicitly setting it to 0", m.Name)
m.PodReadyLatency = 0
}
log.Tracef("PodReadyLatency: %+v for pod %+v", m.PodReadyLatency, m.Name)
log.Tracef("PodReadyLatencyV2: %+v for pod %+v", m.PodReadyLatencyV2, m.Name)
p.normLatencies = append(p.normLatencies, m)
}
}
Expand All @@ -232,9 +266,13 @@ func (p *podLatency) calcQuantiles() {
jc.Objects = nil
for _, normLatency := range p.normLatencies {
quantileMap[v1.PodScheduled] = append(quantileMap[v1.PodScheduled], normLatency.(podMetric).SchedulingLatency)
quantileMap["PodScheduledV2"] = append(quantileMap["PodScheduledV2"], normLatency.(podMetric).SchedulingLatencyV2)
quantileMap[v1.ContainersReady] = append(quantileMap[v1.ContainersReady], normLatency.(podMetric).ContainersReadyLatency)
quantileMap["ContainersReadyV2"] = append(quantileMap["ContainersReadyV2"], normLatency.(podMetric).ContainersReadyLatencyV2)
quantileMap[v1.PodInitialized] = append(quantileMap[v1.PodInitialized], normLatency.(podMetric).InitializedLatency)
quantileMap["InitializedV2"] = append(quantileMap["InitializedV2"], normLatency.(podMetric).InitializedLatencyV2)
quantileMap[v1.PodReady] = append(quantileMap[v1.PodReady], normLatency.(podMetric).PodReadyLatency)
quantileMap["ReadyV2"] = append(quantileMap["ReadyV2"], normLatency.(podMetric).PodReadyLatencyV2)
}
for quantileName, v := range quantileMap {
podQ := metrics.LatencyQuantiles{
Expand Down

0 comments on commit 03d1e08

Please sign in to comment.