Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ws-manager] add metrics to track initialize and finalize of workspaces #9355

Merged
merged 1 commit into from
Apr 18, 2022
Merged

[ws-manager] add metrics to track initialize and finalize of workspaces #9355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions components/ws-manager/pkg/manager/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ type metrics struct {
manager *Manager

startupTimeHistVec *prometheus.HistogramVec
initializeTimeHistVec *prometheus.HistogramVec
finalizeTimeHistVec *prometheus.HistogramVec
totalStartsCounterVec *prometheus.CounterVec
totalStopsCounterVec *prometheus.CounterVec
totalOpenPortGauge prometheus.GaugeFunc
Expand All @@ -58,6 +60,20 @@ func newMetrics(m *Manager) *metrics {
// same as components/ws-manager-bridge/src/prometheus-metrics-exporter.ts#L15
Buckets: prometheus.ExponentialBuckets(2, 2, 10),
}, []string{"type"}),
initializeTimeHistVec: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsWorkspaceSubsystem,
Name: "workspace_initialize_seconds",
Help: "time it took to initialize workspace",
Buckets: prometheus.ExponentialBuckets(2, 2, 10),
}, []string{"type"}),
finalizeTimeHistVec: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsWorkspaceSubsystem,
Name: "workspace_finalize_seconds",
Help: "time it took to finalize workspace",
Buckets: prometheus.ExponentialBuckets(2, 2, 10),
}, []string{"type"}),
totalStartsCounterVec: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsWorkspaceSubsystem,
Expand Down Expand Up @@ -118,6 +134,8 @@ func newTotalOpenPortGaugeHandler(m *Manager) func() float64 {
func (m *metrics) Register(reg prometheus.Registerer) error {
collectors := []prometheus.Collector{
m.startupTimeHistVec,
m.initializeTimeHistVec,
m.finalizeTimeHistVec,
newPhaseTotalVec(m.manager),
newWorkspaceActivityVec(m.manager),
newTimeoutSettingsVec(m.manager),
Expand Down
15 changes: 14 additions & 1 deletion components/ws-manager/pkg/manager/monitor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -734,7 +734,7 @@ func (m *Monitor) initializeWorkspaceContent(ctx context.Context, pod *corev1.Po
// we are already initialising
return nil
}

t := time.Now()
err = retryIfUnavailable(ctx, func(ctx context.Context) error {
_, err = snc.InitWorkspace(ctx, &wsdaemon.InitWorkspaceRequest{
Id: workspaceID,
Expand All @@ -755,6 +755,12 @@ func (m *Monitor) initializeWorkspaceContent(ctx context.Context, pod *corev1.Po
} else {
err = handleGRPCError(ctx, err)
}
wsType := pod.Labels[wsk8s.TypeLabel]
hist, errHist := m.manager.metrics.initializeTimeHistVec.GetMetricWithLabelValues(wsType)
if errHist != nil {
log.WithError(errHist).WithField("type", wsType).Warn("cannot get initialize time histogram metric")
}
hist.Observe(time.Since(t).Seconds())
if err != nil {
return xerrors.Errorf("cannot initialize workspace: %w", err)
}
Expand Down Expand Up @@ -916,6 +922,7 @@ func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceOb
backupError error
gitStatus *csapi.GitStatus
)
t := time.Now()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be better that calculate the start time at finalizeWorkspaceContent and have a defer function to calculate the end time? For example:

func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceObjects) {
    t := time.Now()
    ...
    tpe, err := wso.WorkspaceType()
    if err != nil {
        tracing.LogError(span, err)
        log.WithError(err).Warn("cannot determine workspace type - assuming this is a regular")
        tpe = api.WorkspaceType_REGULAR
    }
    defer func() {
        wsType := api.WorkspaceType_name[int32(tpe)]
	hist, errHist := m.manager.metrics.finalizeTimeHistVec.GetMetricWithLabelValues(wsType)
	if errHist != nil {
		log.WithError(errHist).WithField("type", wsType).Warn("cannot get finalize time histogram metric")
	}
	hist.Observe(time.Since(t).Seconds())
    }()
    ...
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I thought of that, but that would catch incorrect metrics in case of this return statement:

gitpod/components/ws-manager/pkg/manager/monitor.go

Lines 929 to 933 in 054bd0b

if !didSometing {
// someone else is managing finalization process ... we don't have to bother
return
}

I only want to catch timing for workspaces where we actually did some work.

for i := 0; i < wsdaemonMaxAttempts; i++ {
span.LogKV("attempt", i)
didSometing, gs, err := doFinalize()
Expand Down Expand Up @@ -959,6 +966,12 @@ func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceOb
}
break
}
wsType := api.WorkspaceType_name[int32(tpe)]
hist, err := m.manager.metrics.finalizeTimeHistVec.GetMetricWithLabelValues(wsType)
if err != nil {
log.WithError(err).WithField("type", wsType).Warn("cannot get finalize time histogram metric")
}
hist.Observe(time.Since(t).Seconds())

disposalStatus = &workspaceDisposalStatus{
BackupComplete: true,
Expand Down