From b477b15c0aa0fe59d79f0c298aab5ec43b397ee9 Mon Sep 17 00:00:00 2001 From: Milan Pavlik Date: Mon, 4 Apr 2022 21:12:12 +0000 Subject: [PATCH] [ws-manager-bridge] Skip stale prebuild events --- components/ws-manager-bridge/ee/src/bridge.ts | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/components/ws-manager-bridge/ee/src/bridge.ts b/components/ws-manager-bridge/ee/src/bridge.ts index f5940f7ff3d88b..b2576ef3ca5174 100644 --- a/components/ws-manager-bridge/ee/src/bridge.ts +++ b/components/ws-manager-bridge/ee/src/bridge.ts @@ -14,7 +14,6 @@ import { log, LogContext } from "@gitpod/gitpod-protocol/lib/util/logging"; @injectable() export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { - protected async cleanupProbeWorkspace(ctx: TraceContext, status: WorkspaceStatus.AsObject | undefined) { if (!status) { return; @@ -29,16 +28,21 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { const span = TraceContext.startSpan("cleanupProbeWorkspace", ctx); try { const workspaceId = status.metadata!.metaId!; - await this.workspaceDB.trace({span}).hardDeleteWorkspace(workspaceId); + await this.workspaceDB.trace({ span }).hardDeleteWorkspace(workspaceId); } catch (e) { - TraceContext.setError({span}, e); + TraceContext.setError({ span }, e); throw e; } finally { span.finish(); } } - protected async updatePrebuiltWorkspace(ctx: TraceContext, userId: string, status: WorkspaceStatus.AsObject, writeToDB: boolean) { + protected async updatePrebuiltWorkspace( + ctx: TraceContext, + userId: string, + status: WorkspaceStatus.AsObject, + writeToDB: boolean, + ) { if (status.spec && status.spec.type != WorkspaceType.PREBUILD) { return; } @@ -49,11 +53,11 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { const span = TraceContext.startSpan("updatePrebuiltWorkspace", ctx); try { - const prebuild = await this.workspaceDB.trace({span}).findPrebuildByWorkspaceID(status.metadata!.metaId!); + const prebuild = await this.workspaceDB.trace({ span }).findPrebuildByWorkspaceID(status.metadata!.metaId!); if (!prebuild) { log.warn(logCtx, "Headless workspace without prebuild"); - TraceContext.setError({span}, new Error("headless workspace without prebuild")); - return + TraceContext.setError({ span }, new Error("headless workspace without prebuild")); + return; } span.setTag("updatePrebuiltWorkspace.prebuildId", prebuild.id); span.setTag("updatePrebuiltWorkspace.workspaceInstance.statusVersion", status.statusVersion); @@ -61,19 +65,23 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { if (prebuild.statusVersion <= status.statusVersion) { // prebuild.statusVersion = 0 is the default value in the DB, these shouldn't be counted as stale in our metrics if (prebuild.statusVersion > 0) { + // We've gotten an event which is younger than one we've already processed. We shouldn't process the stale one. + span.setTag("updatePrebuiltWorkspace.staleEvent", true); this.prometheusExporter.recordStalePrebuildEvent(); + log.info(logCtx, "Stale prebuild event received, skipping."); + return; } } - prebuild.statusVersion = status.statusVersion + prebuild.statusVersion = status.statusVersion; - if (prebuild.state === 'queued') { + if (prebuild.state === "queued") { // We've received an update from ws-man for this workspace, hence it must be running. prebuild.state = "building"; if (writeToDB) { - await this.workspaceDB.trace({span}).storePrebuiltWorkspace(prebuild); + await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); } - await this.messagebus.notifyHeadlessUpdate({span}, userId, workspaceId, { + await this.messagebus.notifyHeadlessUpdate({ span }, userId, workspaceId, { type: HeadlessWorkspaceEventType.Started, workspaceID: workspaceId, }); @@ -95,8 +103,7 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { headlessUpdateType = HeadlessWorkspaceEventType.Aborted; } else if (!!status.conditions!.headlessTaskFailed) { prebuild.state = "available"; - if (status.conditions!.headlessTaskFailed) - prebuild.error = status.conditions!.headlessTaskFailed; + if (status.conditions!.headlessTaskFailed) prebuild.error = status.conditions!.headlessTaskFailed; prebuild.snapshot = status.conditions!.snapshot; headlessUpdateType = HeadlessWorkspaceEventType.FinishedButFailed; } else if (!!status.conditions!.snapshot) { @@ -108,28 +115,28 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { return; } - span.setTag("updatePrebuildWorkspace.prebuild.state", prebuild.state) - span.setTag("updatePrebuildWorkspace.prebuild.error", prebuild.error) + span.setTag("updatePrebuildWorkspace.prebuild.state", prebuild.state); + span.setTag("updatePrebuildWorkspace.prebuild.error", prebuild.error); if (writeToDB) { - await this.workspaceDB.trace({span}).storePrebuiltWorkspace(prebuild); + await this.workspaceDB.trace({ span }).storePrebuiltWorkspace(prebuild); } // notify updates // headless update - await this.messagebus.notifyHeadlessUpdate({span}, userId, workspaceId, { + await this.messagebus.notifyHeadlessUpdate({ span }, userId, workspaceId, { type: headlessUpdateType, workspaceID: workspaceId, }); // prebuild info - const info = (await this.workspaceDB.trace({span}).findPrebuildInfos([prebuild.id]))[0]; + const info = (await this.workspaceDB.trace({ span }).findPrebuildInfos([prebuild.id]))[0]; if (info) { this.messagebus.notifyOnPrebuildUpdate({ info, status: prebuild.state }); } } } catch (e) { - TraceContext.setError({span}, e); + TraceContext.setError({ span }, e); throw e; } finally { span.finish(); @@ -142,16 +149,16 @@ export class WorkspaceManagerBridgeEE extends WorkspaceManagerBridge { const prebuild = await this.workspaceDB.trace({}).findPrebuildByWorkspaceID(instance.workspaceId); if (prebuild) { // this is a prebuild - set it to aborted - prebuild.state = 'aborted'; + prebuild.state = "aborted"; await this.workspaceDB.trace({}).storePrebuiltWorkspace(prebuild); - { // notify about prebuild updated - const info = (await this.workspaceDB.trace({span}).findPrebuildInfos([prebuild.id]))[0]; + { + // notify about prebuild updated + const info = (await this.workspaceDB.trace({ span }).findPrebuildInfos([prebuild.id]))[0]; if (info) { this.messagebus.notifyOnPrebuildUpdate({ info, status: prebuild.state }); } } } } - }