diff --git a/components/dashboard/src/start/StartWorkspace.tsx b/components/dashboard/src/start/StartWorkspace.tsx index 384eff1434fbcb..ba94c7866024c0 100644 --- a/components/dashboard/src/start/StartWorkspace.tsx +++ b/components/dashboard/src/start/StartWorkspace.tsx @@ -4,7 +4,7 @@ * See License-AGPL.txt in the project root for license information. */ -import { ContextURL, DisposableCollection, WithPrebuild, Workspace, WorkspaceImageBuild, WorkspaceInstance } from "@gitpod/gitpod-protocol"; +import { ContextURL, DisposableCollection, GitpodServer, RateLimiterError, StartWorkspaceResult, WithPrebuild, Workspace, WorkspaceImageBuild, WorkspaceInstance } from "@gitpod/gitpod-protocol"; import { IDEOptions } from "@gitpod/gitpod-protocol/lib/ide-protocol"; import { ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; import EventEmitter from "events"; @@ -124,7 +124,7 @@ export default class StartWorkspace extends React.Component { + let retries = 0; + while (true) { + try { + return await getGitpodService().server.startWorkspace(workspaceId, options); + } catch (err) { + if (err?.code !== ErrorCodes.TOO_MANY_REQUESTS) { + throw err; + } + + if (retries >= 10) { + throw err; + } + retries++; + + const data = err?.data as RateLimiterError | undefined; + const timeoutSeconds = data?.retryAfter || 5; + console.log(`startWorkspace was rate-limited: waiting for ${timeoutSeconds}s before doing ${retries}nd retry...`) + await new Promise(resolve => setTimeout(resolve, timeoutSeconds * 1000)); + } + } + } + async fetchWorkspaceInfo() { const { workspaceId } = this.props; try { diff --git a/components/gitpod-protocol/src/gitpod-service.ts b/components/gitpod-protocol/src/gitpod-service.ts index bd2c7d42563d57..ae7491038e48c0 100644 --- a/components/gitpod-protocol/src/gitpod-service.ts +++ b/components/gitpod-protocol/src/gitpod-service.ts @@ -252,6 +252,16 @@ export interface GitpodServer extends JsonRpcServer, AdminServer, identifyUser(event: RemoteIdentifyMessage): Promise; } +export interface RateLimiterError { + method?: string, + + /** + * Retry after this many seconds, earliest. + * cmp.: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After + */ + retryAfter: number, +} + export interface CreateProjectParams { name: string; slug?: string; diff --git a/components/gitpod-protocol/src/messaging/connection-error-handler.ts b/components/gitpod-protocol/src/messaging/connection-error-handler.ts deleted file mode 100644 index dc409962ebbd64..00000000000000 --- a/components/gitpod-protocol/src/messaging/connection-error-handler.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2017 TypeFox and others. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - */ - -import { Message } from "vscode-jsonrpc"; -import { log } from '../util/logging'; - -export interface ResolvedConnectionErrorHandlerOptions { - readonly serverName: string - /** - * The maximum amount of errors allowed before stopping the server. - */ - readonly maxErrors: number - /** - * The maimum amount of restarts allowed in the restart interval. - */ - readonly maxRestarts: number - /** - * In minutes. - */ - readonly restartInterval: number -} - -export type ConnectionErrorHandlerOptions = Partial & { - readonly serverName: string -}; - -export class ConnectionErrorHandler { - - protected readonly options: ResolvedConnectionErrorHandlerOptions; - constructor(options: ConnectionErrorHandlerOptions) { - this.options = { - maxErrors: 3, - maxRestarts: 5, - restartInterval: 3, - ...options - }; - } - - shouldStop(error: Error, message?: Message, count?: number): boolean { - return !count || count > this.options.maxErrors; - } - - protected readonly restarts: number[] = []; - shouldRestart(): boolean { - this.restarts.push(Date.now()); - if (this.restarts.length <= this.options.maxRestarts) { - return true; - } - const diff = this.restarts[this.restarts.length - 1] - this.restarts[0]; - if (diff <= this.options.restartInterval * 60 * 1000) { - log.error(`Server ${this.options.serverName} crashed ${this.options.maxRestarts} times in the last ${this.options.restartInterval} minutes. Will not restart`); - return false; - } - this.restarts.shift(); - return true; - } - -} diff --git a/components/server/src/auth/rate-limiter.ts b/components/server/src/auth/rate-limiter.ts index 99356702f24d46..60dce27003a58b 100644 --- a/components/server/src/auth/rate-limiter.ts +++ b/components/server/src/auth/rate-limiter.ts @@ -12,6 +12,7 @@ import { RateLimiterMemory, RateLimiterRes } from "rate-limiter-flexible"; export const accessCodeSyncStorage = 'accessCodeSyncStorage'; export const accessHeadlessLogs = 'accessHeadlessLogs'; type GitpodServerMethodType = keyof Omit | typeof accessCodeSyncStorage | typeof accessHeadlessLogs; +type GroupKey = "default" | "startWorkspace"; type GroupsConfig = { [key: string]: { points: number, @@ -20,7 +21,7 @@ type GroupsConfig = { } type FunctionsConfig = { [K in GitpodServerMethodType]: { - group: string, + group: GroupKey, points: number, } } @@ -34,7 +35,11 @@ function getConfig(config: RateLimiterConfig): RateLimiterConfig { default: { points: 60000, // 1,000 calls per user per second durationsSec: 60, - } + }, + startWorkspace: { + points: 1, // 1 workspace start per user per 10s + durationsSec: 10 + }, } const defaultFunctions: FunctionsConfig = { "getLoggedInUser": { group: "default", points: 1 }, @@ -59,7 +64,7 @@ function getConfig(config: RateLimiterConfig): RateLimiterConfig { "isWorkspaceOwner": { group: "default", points: 1 }, "getOwnerToken": { group: "default", points: 1 }, "createWorkspace": { group: "default", points: 1 }, - "startWorkspace": { group: "default", points: 1 }, + "startWorkspace": { group: "startWorkspace", points: 1 }, "stopWorkspace": { group: "default", points: 1 }, "deleteWorkspace": { group: "default", points: 1 }, "setWorkspaceDescription": { group: "default", points: 1 }, diff --git a/components/server/src/websocket/websocket-connection-manager.ts b/components/server/src/websocket/websocket-connection-manager.ts index d697aeeec648de..183e5dab83f1bd 100644 --- a/components/server/src/websocket/websocket-connection-manager.ts +++ b/components/server/src/websocket/websocket-connection-manager.ts @@ -4,7 +4,7 @@ * See License-AGPL.txt in the project root for license information. */ -import { ClientHeaderFields, Disposable, GitpodClient as GitpodApiClient, GitpodServerPath, User } from "@gitpod/gitpod-protocol"; +import { ClientHeaderFields, Disposable, GitpodClient as GitpodApiClient, GitpodServerPath, RateLimiterError, User } from "@gitpod/gitpod-protocol"; import { ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error"; import { ConnectionHandler } from "@gitpod/gitpod-protocol/lib/messaging/handler"; import { JsonRpcConnectionHandler, JsonRpcProxy, JsonRpcProxyFactory } from "@gitpod/gitpod-protocol/lib/messaging/proxy-factory"; @@ -356,7 +356,7 @@ class GitpodJsonRpcProxyFactory extends JsonRpcProxyFactory throw rlRejected; } log.warn({ userId }, "Rate limiter prevents accessing method due to too many requests.", rlRejected, { method }); - throw new ResponseError(ErrorCodes.TOO_MANY_REQUESTS, "too many requests", { "Retry-After": String(Math.round(rlRejected.msBeforeNext / 1000)) || 1 }); + throw new ResponseError(ErrorCodes.TOO_MANY_REQUESTS, "too many requests", { method, retryAfter: Math.round(rlRejected.msBeforeNext / 1000) || 1 }); } // access guard