From 9affb83a1c81c35cdef9fe096ae467b8bd231d7c Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Fri, 13 Aug 2021 08:20:08 +0000 Subject: [PATCH] [workspace]: add force-stop check on stopping workspaces Since #4910 stopped counting "stopping" workspaces for billing purposes, any workspace caught in a "stopping" phase would never be force-stopped. This adds a conditional "includeStopping" boolean (defaulting to `false`) to the DB implementation and the meta-instance-controller simply includes that phase in the search. It was discovered that ~200 workspaces were caught in this phase (90% prebuilds) so this phase is necessary to force-stop. --- chart/templates/ws-manager-bridge-configmap.yaml | 1 + components/gitpod-db/src/typeorm/workspace-db-impl.ts | 8 ++++++-- components/gitpod-db/src/workspace-db.ts | 2 +- components/ws-manager-bridge/src/config.ts | 1 + .../ws-manager-bridge/src/meta-instance-controller.ts | 10 ++++++++-- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/chart/templates/ws-manager-bridge-configmap.yaml b/chart/templates/ws-manager-bridge-configmap.yaml index ab537429dfce7b..e52cdf82b05598 100644 --- a/chart/templates/ws-manager-bridge-configmap.yaml +++ b/chart/templates/ws-manager-bridge-configmap.yaml @@ -28,6 +28,7 @@ data: "timeouts": { "metaInstanceCheckIntervalSeconds": 60, "preparingPhaseSeconds": 7200, + "stoppingPhaseSeconds": 7200, "unknownPhaseSeconds": 600 }, "staticBridges": {{ index (include "ws-manager-list" (dict "root" . "gp" $.Values "comp" .Values.components.server) | fromYaml) "manager" | default list | toJson }} diff --git a/components/gitpod-db/src/typeorm/workspace-db-impl.ts b/components/gitpod-db/src/typeorm/workspace-db-impl.ts index 3a53525e491ae4..5f3022ef1f3458 100644 --- a/components/gitpod-db/src/typeorm/workspace-db-impl.ts +++ b/components/gitpod-db/src/typeorm/workspace-db-impl.ts @@ -327,9 +327,13 @@ export abstract class AbstractTypeORMWorkspaceDBImpl implements WorkspaceDB { ).map(wsinfo => wsinfo.latestInstance); } - public async findRunningInstancesWithWorkspaces(installation?: string, userId?: string): Promise { + public async findRunningInstancesWithWorkspaces(installation?: string, userId?: string, includeStopping: boolean = false): Promise { const params: any = {}; - const conditions = ["wsi.phasePersisted != 'stopped'", "wsi.phasePersisted != 'stopping'", "wsi.deleted != TRUE"]; + const conditions = ["wsi.phasePersisted != 'stopped'", "wsi.deleted != TRUE"]; + if (!includeStopping) { + // This excludes instances in a 'stopping' phase + conditions.push("wsi.phasePersisted != 'stopping'"); + } if (installation) { params.region = installation; conditions.push("wsi.region = :region"); diff --git a/components/gitpod-db/src/workspace-db.ts b/components/gitpod-db/src/workspace-db.ts index 2026b104e47168..b78affab79c5ee 100644 --- a/components/gitpod-db/src/workspace-db.ts +++ b/components/gitpod-db/src/workspace-db.ts @@ -86,7 +86,7 @@ export interface WorkspaceDB { findAllWorkspaceInstances(offset: number, limit: number, orderBy: keyof WorkspaceInstance, orderDir: "ASC" | "DESC", ownerId?: string, minCreationTime?: Date, maxCreationTime?: Date, onlyRunning?: boolean, type?: WorkspaceType): Promise<{ total: number, rows: WorkspaceInstance[] }>; findRegularRunningInstances(userId?: string): Promise; - findRunningInstancesWithWorkspaces(installation?: string, userId?: string): Promise; + findRunningInstancesWithWorkspaces(installation?: string, userId?: string, includeStopping?: boolean): Promise; isWhitelisted(repositoryUrl : string): Promise; getFeaturedRepositories(): Promise[]>; diff --git a/components/ws-manager-bridge/src/config.ts b/components/ws-manager-bridge/src/config.ts index 07bdfe113fdba5..ccd40f9cb2289f 100644 --- a/components/ws-manager-bridge/src/config.ts +++ b/components/ws-manager-bridge/src/config.ts @@ -33,6 +33,7 @@ export interface Configuration { timeouts: { metaInstanceCheckIntervalSeconds: number; preparingPhaseSeconds: number; + stoppingPhaseSeconds: number; unknownPhaseSeconds: number; } } diff --git a/components/ws-manager-bridge/src/meta-instance-controller.ts b/components/ws-manager-bridge/src/meta-instance-controller.ts index 2450ce0716e969..062ee61a287ff4 100644 --- a/components/ws-manager-bridge/src/meta-instance-controller.ts +++ b/components/ws-manager-bridge/src/meta-instance-controller.ts @@ -23,7 +23,7 @@ export class MetaInstanceController { protected readonly workspaceDB: WorkspaceDB; protected async checkAndStopWorkspaces() { - const instances = await this.workspaceDB.findRunningInstancesWithWorkspaces(this.config.installation); + const instances = await this.workspaceDB.findRunningInstancesWithWorkspaces(this.config.installation, undefined, true); await Promise.all(instances.map(async (instance: RunningWorkspaceInfo) => { const logContext = { instanceId: instance.latestInstance.id }; @@ -33,11 +33,17 @@ export class MetaInstanceController { const now = Date.now(); const creationTime = new Date(instance.latestInstance.creationTime).getTime(); + const stoppingTime = new Date(instance.latestInstance.stoppingTime ?? now).getTime(); // stoppingTime only set if entered stopping state const timedOutInPreparing = now >= creationTime + (this.config.timeouts.preparingPhaseSeconds * 1000); + const timedOutInStopping = now >= stoppingTime + (this.config.timeouts.stoppingPhaseSeconds * 1000); const timedOutInUnknown = now >= creationTime + (this.config.timeouts.unknownPhaseSeconds * 1000); const currentPhase = instance.latestInstance.status.phase; - if ((currentPhase === 'preparing' && timedOutInPreparing) || (currentPhase === 'unknown' && timedOutInUnknown)) { + if ( + (currentPhase === 'preparing' && timedOutInPreparing) || + (currentPhase === 'stopping' && timedOutInStopping) || + (currentPhase === 'unknown' && timedOutInUnknown) + ) { log.info(logContext, 'MetaInstanceController: Setting workspace instance to stopped', { creationTime, currentPhase