[ws-manager-bridge] Use WS update status version

gitpod-io · Jul 8, 2022 · 48aff21 · 48aff21
1 parent 5bb6e2a
commit 48aff21
Show file tree

Hide file tree

Showing 8 changed files with 366 additions and 271 deletions.
diff --git a/components/ee/payment-endpoint/src/accounting/account-service.spec.db.ts b/components/ee/payment-endpoint/src/accounting/account-service.spec.db.ts
diff --git a/components/gitpod-db/src/user-db.spec.db.ts b/components/gitpod-db/src/user-db.spec.db.ts
@@ -218,6 +218,7 @@ namespace TestData {
         deployedTime: undefined,
         stoppedTime: undefined,
         status: {
+            version: 1,
             phase: "preparing",
             conditions: {},
         },

diff --git a/components/gitpod-db/src/workspace-db.spec.db.ts b/components/gitpod-db/src/workspace-db.spec.db.ts
@@ -58,6 +58,7 @@ class WorkspaceDBSpec {
         stoppingTime: undefined,
         stoppedTime: undefined,
         status: {
+            version: 1,
             phase: "preparing",
             conditions: {},
         },
@@ -81,6 +82,7 @@ class WorkspaceDBSpec {
         stoppingTime: undefined,
         stoppedTime: undefined,
         status: {
+            version: 1,
             phase: "running",
             conditions: {},
         },
@@ -119,6 +121,7 @@ class WorkspaceDBSpec {
         stoppingTime: undefined,
         stoppedTime: undefined,
         status: {
+            version: 1,
             phase: "preparing",
             conditions: {},
         },
@@ -157,6 +160,7 @@ class WorkspaceDBSpec {
         stoppingTime: undefined,
         stoppedTime: undefined,
         status: {
+            version: 1,
             phase: "preparing",
             conditions: {},
         },

diff --git a/components/gitpod-protocol/src/workspace-instance.ts b/components/gitpod-protocol/src/workspace-instance.ts
@@ -72,6 +72,11 @@ export interface WorkspaceInstance {
 
 // WorkspaceInstanceStatus describes the current state of a workspace instance
 export interface WorkspaceInstanceStatus {
+    // version is the current version of the workspace instance status
+    // Note: consider this value opague. The only guarantee given is that it imposes
+    //       a partial order on status updates, i.e. a.version > b.version -> a newer than b.
+    version?: number;
+
     // phase describes a high-level state of the workspace instance
     phase: WorkspaceInstancePhase;
 

diff --git a/components/server/src/auth/resource-access.spec.ts b/components/server/src/auth/resource-access.spec.ts
@@ -585,6 +585,7 @@ class TestResourceAccess {
                 creationTime: new Date(2000, 1, 2).toISOString(),
                 region: "local",
                 status: {
+                    version: 1,
                     conditions: {},
                     phase: "running",
                 },

diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts
@@ -780,6 +780,7 @@ export class WorkspaceStarter {
             region: this.config.installationShortname, // Shortname set to bridge can cleanup workspaces stuck preparing
             workspaceImage: "", // Initially empty, filled during starting process
             status: {
+                version: 0,
                 conditions: {},
                 phase: "preparing",
             },

diff --git a/components/ws-manager-bridge/src/bridge.ts b/components/ws-manager-bridge/src/bridge.ts
@@ -227,6 +227,7 @@ export class WorkspaceManagerBridge implements Disposable {
         const span = TraceContext.startSpan("handleStatusUpdate", ctx);
         span.setTag("status", JSON.stringify(filterStatus(status)));
         span.setTag("writeToDB", writeToDB);
+        span.setTag("statusVersion", status.statusVersion);
         try {
             // Beware of the ID mapping here: What's a workspace to the ws-manager is a workspace instance to the rest of the system.
             //                                The workspace ID of ws-manager is the workspace instance ID in the database.
@@ -252,6 +253,14 @@ export class WorkspaceManagerBridge implements Disposable {
                 return;
             }
 
+            const currentStatusVersion = instance.status.version || 0;
+            if (currentStatusVersion > 0 && currentStatusVersion >= status.statusVersion) {
+                // We've gotten an event which is older than one we've already processed. We shouldn't process the stale one.
+                span.setTag("statusUpdate.staleEvent", true);
+                this.prometheusExporter.recordStaleStatusUpdate();
+                log.debug(ctx, "Stale status update received, skipping.");
+            }
+
             if (!!status.spec.exposedPortsList) {
                 instance.status.exposedPorts = status.spec.exposedPortsList.map((p) => {
                     return <WorkspaceInstancePort>{
@@ -269,6 +278,7 @@ export class WorkspaceManagerBridge implements Disposable {
             }
 
             instance.ideUrl = status.spec.url!;
+            instance.status.version = status.statusVersion;
             instance.status.timeout = status.spec.timeout;
             if (!!instance.status.conditions.failed && !status.conditions.failed) {
                 // We already have a "failed" condition, and received an empty one: This is a bug, "failed" conditions are terminal per definition.

diff --git a/components/ws-manager-bridge/src/prometheus-metrics-exporter.ts b/components/ws-manager-bridge/src/prometheus-metrics-exporter.ts
@@ -17,6 +17,7 @@ export class PrometheusMetricsExporter {
     protected readonly clusterScore: prom.Gauge<string>;
     protected readonly clusterCordoned: prom.Gauge<string>;
     protected readonly statusUpdatesTotal: prom.Counter<string>;
+    protected readonly staleStatusUpdatesTotal: prom.Counter<string>;
     protected readonly stalePrebuildEventsTotal: prom.Counter<string>;
     protected readonly prebuildsCompletedTotal: prom.Counter<string>;
 
@@ -53,6 +54,10 @@ export class PrometheusMetricsExporter {
             help: "Total workspace status updates received",
             labelNames: ["workspace_cluster", "known_instance"],
         });
+        this.staleStatusUpdatesTotal = new prom.Counter({
+            name: "gitpod_ws_manager_bridge_stale_status_updates_total",
+            help: "Total count of stale status updates received by workspace manager bridge",
+        });
         this.stalePrebuildEventsTotal = new prom.Counter({
             name: "gitpod_ws_manager_bridge_stale_prebuild_events_total",
             help: "Total count of stale prebuild events received by workspace manager bridge",
@@ -129,6 +134,10 @@ export class PrometheusMetricsExporter {
         this.statusUpdatesTotal.labels(installation, knownInstance ? "true" : "false").inc();
     }
 
+    recordStaleStatusUpdate(): void {
+        this.staleStatusUpdatesTotal.inc();
+    }
+
     recordStalePrebuildEvent(): void {
         this.stalePrebuildEventsTotal.inc();
     }