[server, dashboard] Do basic rate limiting on startWorkspace

gitpod-io · Feb 7, 2022 · d955ce1 · d955ce1
1 parent 29c3a7d
commit d955ce1
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 69 deletions.
diff --git a/components/dashboard/src/start/StartWorkspace.tsx b/components/dashboard/src/start/StartWorkspace.tsx
@@ -4,7 +4,7 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-import { ContextURL, DisposableCollection, WithPrebuild, Workspace, WorkspaceImageBuild, WorkspaceInstance } from "@gitpod/gitpod-protocol";
+import { ContextURL, DisposableCollection, GitpodServer, RateLimiterError, StartWorkspaceResult, WithPrebuild, Workspace, WorkspaceImageBuild, WorkspaceInstance } from "@gitpod/gitpod-protocol";
 import { IDEOptions } from "@gitpod/gitpod-protocol/lib/ide-protocol";
 import { ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error";
 import EventEmitter from "events";
@@ -124,7 +124,7 @@ export default class StartWorkspace extends React.Component<StartWorkspaceProps,
 
     const { workspaceId } = this.props;
     try {
-      const result = await getGitpodService().server.startWorkspace(workspaceId, { forceDefaultImage });
+      const result = await this.startWorkspaceRateLimited(workspaceId, { forceDefaultImage });
       if (!result) {
         throw new Error("No result!");
       }
@@ -151,6 +151,35 @@ export default class StartWorkspace extends React.Component<StartWorkspaceProps,
     }
   }
 
+  /**
+   * TODO(gpl) Ideally this can be pushed into the GitpodService implementation. But to get started we hand-roll it here.
+   * @param workspaceId
+   * @param options
+   * @returns
+   */
+  protected async startWorkspaceRateLimited(workspaceId: string, options: GitpodServer.StartWorkspaceOptions): Promise<StartWorkspaceResult> {
+    let retries = 0;
+    while (true) {
+      try {
+        return await getGitpodService().server.startWorkspace(workspaceId, options);
+      } catch (err) {
+        if (err?.code !== ErrorCodes.TOO_MANY_REQUESTS) {
+          throw err;
+        }
+
+        if (retries >= 10) {
+          throw err;
+        }
+        retries++;
+
+        const data = err?.data as RateLimiterError | undefined;
+        const timeoutSeconds = data?.retryAfter || 5;
+        console.log(`startWorkspace was rate-limited: waiting for ${timeoutSeconds}s before doing ${retries}nd retry...`)
+        await new Promise(resolve => setTimeout(resolve, timeoutSeconds * 1000));
+      }
+    }
+  }
+
   async fetchWorkspaceInfo() {
     const { workspaceId } = this.props;
     try {

diff --git a/components/gitpod-protocol/src/gitpod-service.ts b/components/gitpod-protocol/src/gitpod-service.ts
@@ -252,6 +252,16 @@ export interface GitpodServer extends JsonRpcServer<GitpodClient>, AdminServer,
     identifyUser(event: RemoteIdentifyMessage): Promise<void>;
 }
 
+export interface RateLimiterError {
+    method?: string,
+
+    /**
+     * Retry after this many seconds, earliest.
+     * cmp.: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+     */
+    retryAfter: number,
+}
+
 export interface CreateProjectParams {
     name: string;
     slug?: string;

diff --git a/components/gitpod-protocol/src/messaging/connection-error-handler.ts b/components/gitpod-protocol/src/messaging/connection-error-handler.ts
diff --git a/components/server/src/auth/rate-limiter.ts b/components/server/src/auth/rate-limiter.ts
@@ -12,6 +12,7 @@ import { RateLimiterMemory, RateLimiterRes } from "rate-limiter-flexible";
 export const accessCodeSyncStorage = 'accessCodeSyncStorage';
 export const accessHeadlessLogs = 'accessHeadlessLogs';
 type GitpodServerMethodType = keyof Omit<GitpodServer, "dispose" | "setClient"> | typeof accessCodeSyncStorage | typeof accessHeadlessLogs;
+type GroupKey = "default" | "startWorkspace";
 type GroupsConfig = {
     [key: string]: {
         points: number,
@@ -20,7 +21,7 @@ type GroupsConfig = {
 }
 type FunctionsConfig = {
     [K in GitpodServerMethodType]: {
-        group: string,
+        group: GroupKey,
         points: number,
     }
 }
@@ -34,7 +35,11 @@ function getConfig(config: RateLimiterConfig): RateLimiterConfig {
         default: {
             points: 60000, // 1,000 calls per user per second
             durationsSec: 60,
-        }
+        },
+        startWorkspace: {
+            points: 1,  // 1 workspace start per user per 10s
+            durationsSec: 10
+        },
     }
     const defaultFunctions: FunctionsConfig = {
         "getLoggedInUser": { group: "default", points: 1 },
@@ -59,7 +64,7 @@ function getConfig(config: RateLimiterConfig): RateLimiterConfig {
         "isWorkspaceOwner": { group: "default", points: 1 },
         "getOwnerToken": { group: "default", points: 1 },
         "createWorkspace": { group: "default", points: 1 },
-        "startWorkspace": { group: "default", points: 1 },
+        "startWorkspace": { group: "startWorkspace", points: 1 },
         "stopWorkspace": { group: "default", points: 1 },
         "deleteWorkspace": { group: "default", points: 1 },
         "setWorkspaceDescription": { group: "default", points: 1 },

diff --git a/components/server/src/websocket/websocket-connection-manager.ts b/components/server/src/websocket/websocket-connection-manager.ts
@@ -4,7 +4,7 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-import { ClientHeaderFields, Disposable, GitpodClient as GitpodApiClient, GitpodServerPath, User } from "@gitpod/gitpod-protocol";
+import { ClientHeaderFields, Disposable, GitpodClient as GitpodApiClient, GitpodServerPath, RateLimiterError, User } from "@gitpod/gitpod-protocol";
 import { ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error";
 import { ConnectionHandler } from "@gitpod/gitpod-protocol/lib/messaging/handler";
 import { JsonRpcConnectionHandler, JsonRpcProxy, JsonRpcProxyFactory } from "@gitpod/gitpod-protocol/lib/messaging/proxy-factory";
@@ -356,7 +356,7 @@ class GitpodJsonRpcProxyFactory<T extends object> extends JsonRpcProxyFactory<T>
                     throw rlRejected;
                 }
                 log.warn({ userId }, "Rate limiter prevents accessing method due to too many requests.", rlRejected, { method });
-                throw new ResponseError(ErrorCodes.TOO_MANY_REQUESTS, "too many requests", { "Retry-After": String(Math.round(rlRejected.msBeforeNext / 1000)) || 1 });
+                throw new ResponseError<RateLimiterError>(ErrorCodes.TOO_MANY_REQUESTS, "too many requests", { method, retryAfter: Math.round(rlRejected.msBeforeNext / 1000) || 1 });
             }
 
             // access guard