[workspace]: add force-stop check on stopping workspaces

Since #4910 stopped counting "stopping" workspaces for billing purposes, any workspace caught in a "stopping" phase would never be force-stopped. This adds a conditional "includeStopping" boolean (defaulting to `false`) to the DB implementation and the meta-instance-controller simply includes that phase in the search. It was discovered that ~200 workspaces were caught in this phase (90% prebuilds) so this phase is necessary to force-stop.
gitpod-io · Aug 13, 2021 · f35e762 · f35e762
1 parent df20a0a
commit f35e762
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 5 deletions.
diff --git a/chart/templates/ws-manager-bridge-configmap.yaml b/chart/templates/ws-manager-bridge-configmap.yaml
@@ -28,6 +28,7 @@ data:
         "timeouts": {
           "metaInstanceCheckIntervalSeconds": 60,
           "preparingPhaseSeconds": 7200,
+          "stoppingPhaseSeconds": 7200,
           "unknownPhaseSeconds": 600
         },
         "staticBridges": {{ index (include "ws-manager-list" (dict "root" . "gp" $.Values "comp" .Values.components.server) | fromYaml) "manager" | default list | toJson }}

diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts
@@ -327,9 +327,13 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB {
         ).map(wsinfo => wsinfo.latestInstance);
     }
 
-    public async findRunningInstancesWithWorkspaces(installation?: string, userId?: string): Promise<RunningWorkspaceInfo[]> {
+    public async findRunningInstancesWithWorkspaces(installation?: string, userId?: string, includeStopping: boolean = false): Promise<RunningWorkspaceInfo[]> {
         const params: any = {};
-        const conditions = ["wsi.phasePersisted != 'stopped'", "wsi.phasePersisted != 'stopping'", "wsi.deleted != TRUE"];
+        const conditions = ["wsi.phasePersisted != 'stopped'", "wsi.deleted != TRUE"];
+        if (!includeStopping) {
+            // This excludes instances in a 'stopping' phase
+            conditions.push("wsi.phasePersisted != 'stopping'");
+        }
         if (installation) {
             params.region = installation;
             conditions.push("wsi.region = :region");

diff --git a/components/gitpod-db/src/workspace-db.ts b/components/gitpod-db/src/workspace-db.ts
@@ -86,7 +86,7 @@ export interface WorkspaceDB {
     findAllWorkspaceInstances(offset: number, limit: number, orderBy: keyof WorkspaceInstance, orderDir: "ASC" | "DESC", ownerId?: string, minCreationTime?: Date, maxCreationTime?: Date, onlyRunning?: boolean, type?: WorkspaceType): Promise<{ total: number, rows: WorkspaceInstance[] }>;
 
     findRegularRunningInstances(userId?: string): Promise<WorkspaceInstance[]>;
-    findRunningInstancesWithWorkspaces(installation?: string, userId?: string): Promise<RunningWorkspaceInfo[]>;
+    findRunningInstancesWithWorkspaces(installation?: string, userId?: string, includeStopping?: boolean): Promise<RunningWorkspaceInfo[]>;
 
     isWhitelisted(repositoryUrl : string): Promise<boolean>;
     getFeaturedRepositories(): Promise<Partial<WhitelistedRepository>[]>;

diff --git a/components/ws-manager-bridge/src/config.ts b/components/ws-manager-bridge/src/config.ts
@@ -33,6 +33,7 @@ export interface Configuration {
     timeouts: {
         metaInstanceCheckIntervalSeconds: number;
         preparingPhaseSeconds: number;
+        stoppingPhaseSeconds: number;
         unknownPhaseSeconds: number;
     }
 }
diff --git a/components/ws-manager-bridge/src/meta-instance-controller.ts b/components/ws-manager-bridge/src/meta-instance-controller.ts
@@ -23,7 +23,7 @@ export class MetaInstanceController {
     protected readonly workspaceDB: WorkspaceDB;
 
     protected async checkAndStopWorkspaces() {
-        const instances = await this.workspaceDB.findRunningInstancesWithWorkspaces(this.config.installation);
+        const instances = await this.workspaceDB.findRunningInstancesWithWorkspaces(this.config.installation, undefined, true);
 
         await Promise.all(instances.map(async (instance: RunningWorkspaceInfo) => {
             const logContext = { instanceId: instance.latestInstance.id };
@@ -33,11 +33,17 @@ export class MetaInstanceController {
 
                 const now = Date.now();
                 const creationTime = new Date(instance.latestInstance.creationTime).getTime();
+                const stoppingTime = new Date(instance.latestInstance.stoppingTime ?? now).getTime(); // stoppingTime only set if entered stopping state
                 const timedOutInPreparing = now >= creationTime + (this.config.timeouts.preparingPhaseSeconds * 1000);
+                const timedOutInStopping  = now >= stoppingTime + (this.config.timeouts.stoppingPhaseSeconds * 1000);
                 const timedOutInUnknown   = now >= creationTime + (this.config.timeouts.unknownPhaseSeconds * 1000);
 
                 const currentPhase = instance.latestInstance.status.phase;
-                if ((currentPhase === 'preparing' && timedOutInPreparing) || (currentPhase === 'unknown' && timedOutInUnknown)) {
+                if (
+                    (currentPhase === 'preparing' && timedOutInPreparing) ||
+                    (currentPhase === 'stopping' && timedOutInStopping) ||
+                    (currentPhase === 'unknown' && timedOutInUnknown)
+                ) {
                     log.info(logContext, 'MetaInstanceController: Setting workspace instance to stopped', {
                         creationTime,
                         currentPhase