[ws-daemon/manager] Use feature flag

components/common-go/kubernetes/kubernetes.go

@@ -65,8 +65,8 @@ const (
 	// WorkspaceSSHPublicKeys contains all authorized ssh public keys that can be connected to the workspace
 	WorkspaceSSHPublicKeys = "gitpod.io/sshPublicKeys"
 
-	// workspaceCpuLimit denotes the cpu limit of a workspace
-	WorkspaceCpuLimitAnnotation = "gitpod.io/cpuLimit"
+	// workspaceCpuMinLimitAnnotation denotes the minimum cpu limit of a workspace i.e. the minimum amount of resources it is guaranteed to get
+	WorkspaceCpuMinLimitAnnotation = "gitpod.io/cpuMinLimit"
 
 	// workspaceCpuBurstLimit denotes the cpu burst limit of a workspace
 	WorkspaceCpuBurstLimitAnnotation = "gitpod.io/cpuBurstLimit"

components/ee/agent-smith/pkg/agent/actions.go

@@ -70,7 +70,8 @@ func (agent *Smith) limitCPUUse(podname string) error {
 			return err
 		}
 
-		pod.Annotations[wsk8s.WorkspaceCpuLimitAnnotation] = agent.Config.Enforcement.CPULimitPenalty
+		pod.Annotations[wsk8s.WorkspaceCpuMinLimitAnnotation] = agent.Config.Enforcement.CPULimitPenalty
+		pod.Annotations[wsk8s.WorkspaceCpuBurstLimitAnnotation] = agent.Config.Enforcement.CPULimitPenalty
 		_, err = pods.Update(ctx, pod, corev1.UpdateOptions{})
 		if err != nil {
 			return err

components/gitpod-protocol/src/protocol.ts

@@ -273,6 +273,7 @@ export const WorkspaceFeatureFlags = {
     full_workspace_backup: undefined,
     persistent_volume_claim: undefined,
     protected_secrets: undefined,
+    workspace_class_limiting: undefined,
 };
 export type NamedWorkspaceFeatureFlag = keyof typeof WorkspaceFeatureFlags;
 

components/server/src/workspace/workspace-starter.ts

@@ -761,14 +761,6 @@ export class WorkspaceStarter {
                 featureFlags = featureFlags.concat(["persistent_volume_claim"]);
             }
 
-            if (!!featureFlags) {
-                // only set feature flags if there actually are any. Otherwise we waste the
-                // few bytes of JSON in the database for no good reason.
-                configuration.featureFlags = featureFlags;
-            }
-
-            const usageAttributionId = await this.userService.getWorkspaceUsageAttributionId(user, workspace.projectId);
-
             let workspaceClass = "";
             let classesEnabled = await getExperimentsClientForBackend().getValueAsync("workspace_classes", false, {
                 user: user,
@@ -808,8 +800,18 @@ export class WorkspaceStarter {
                         workspaceClass = workspaceClass + "-pvc";
                     }
                 }
+
+                featureFlags = featureFlags.concat(["workspace_class_limiting"]);
             }
 
+            if (!!featureFlags) {
+                // only set feature flags if there actually are any. Otherwise we waste the
+                // few bytes of JSON in the database for no good reason.
+                configuration.featureFlags = featureFlags;
+            }
+
+            const usageAttributionId = await this.userService.getWorkspaceUsageAttributionId(user, workspace.projectId);
+
             const now = new Date().toISOString();
             const instance: WorkspaceInstance = {
                 id: uuidv4(),

components/ws-daemon/pkg/cpulimit/cpulimit.go

@@ -7,6 +7,7 @@ package cpulimit
 import (
 	"context"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/gitpod-io/gitpod/common-go/log"
@@ -179,7 +180,7 @@ func (d *Distributor) Tick(dt time.Duration) (DistributorDebug, error) {
 		ws := d.History[id]
 		limit, err := d.Limiter.Limit(ws)
 		if err != nil {
-			log.Errorf("unable to apply min limit: %v", err)
+			log.WithError(err).Errorf("unable to apply min limit")
 			continue
 		}
 
@@ -191,7 +192,7 @@ func (d *Distributor) Tick(dt time.Duration) (DistributorDebug, error) {
 		if totalBandwidth < d.TotalBandwidth && ws.Throttled() {
 			limit, err = d.BurstLimiter.Limit(ws)
 			if err != nil {
-				log.Errorf("unable to apply burst limit: %v", err)
+				log.WithError(err).Errorf("unable to apply burst limit")
 				continue
 			}
 
@@ -222,6 +223,12 @@ type ResourceLimiter interface {
 	Limit(wsh *WorkspaceHistory) (Bandwidth, error)
 }
 
+var _ ResourceLimiter = (*fixedLimiter)(nil)
+var _ ResourceLimiter = (*annotationLimiter)(nil)
+var _ ResourceLimiter = (*BucketLimiter)(nil)
+var _ ResourceLimiter = (*ClampingBucketLimiter)(nil)
+var _ ResourceLimiter = (*compositeLimiter)(nil)
+
 // FixedLimiter returns a fixed limit
 func FixedLimiter(limit Bandwidth) ResourceLimiter {
 	return fixedLimiter{limit}
@@ -306,7 +313,7 @@ type ClampingBucketLimiter struct {
 }
 
 // Limit decides on a CPU use limit
-func (bl *ClampingBucketLimiter) Limit(wsh *WorkspaceHistory) (newLimit Bandwidth) {
+func (bl *ClampingBucketLimiter) Limit(wsh *WorkspaceHistory) (Bandwidth, error) {
 	budgetSpent := wsh.Usage()
 
 	if bl.lastBucketLock {
@@ -315,25 +322,54 @@ func (bl *ClampingBucketLimiter) Limit(wsh *WorkspaceHistory) (newLimit Bandwidt
 		}
 	}
 	if bl.lastBucketLock {
-		return bl.Buckets[len(bl.Buckets)-1].Limit
+		return bl.Buckets[len(bl.Buckets)-1].Limit, nil
 	}
 
 	for i, bkt := range bl.Buckets {
 		if i+1 == len(bl.Buckets) {
 			// We've reached the last bucket - budget doesn't matter anymore
 			bl.lastBucketLock = true
-			return bkt.Limit
+			return bkt.Limit, nil
 		}
 
 		budgetSpent -= bkt.Budget
 		if budgetSpent <= 0 {
 			// BudgetSpent value is in this bucket, hence we have found our current bucket
-			return bkt.Limit
+			return bkt.Limit, nil
 		}
 	}
 
 	// empty bucket list
-	return 0
+	return 0, nil
+}
+
+type compositeLimiter struct {
+	limiters []ResourceLimiter
+}
+
+func CompositeLimiter(limiters ...ResourceLimiter) ResourceLimiter {
+	return &compositeLimiter{
+		limiters: limiters,
+	}
+}
+
+func (cl *compositeLimiter) Limit(wsh *WorkspaceHistory) (Bandwidth, error) {
+	var errs []error
+	for _, limiter := range cl.limiters {
+		limit, err := limiter.Limit(wsh)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+
+		return limit, nil
+	}
+
+	allerr := make([]string, len(errs))
+	for i, err := range errs {
+		allerr[i] = err.Error()
+	}
+	return 0, xerrors.Errorf("no limiter was able to provide a limit", strings.Join(allerr, ", "))
 }
 
 type CFSController interface {

components/ws-daemon/pkg/cpulimit/dispatch.go

@@ -29,6 +29,8 @@ import (
 type Config struct {
 	Enabled        bool              `json:"enabled"`
 	TotalBandwidth resource.Quantity `json:"totalBandwidth"`
+	Limit          resource.Quantity `json:"limit"`
+	BurstLimit     resource.Quantity `json:"burstLimit"`
 
 	ControlPeriod  util.Duration `json:"controlPeriod"`
 	CGroupBasePath string        `json:"cgroupBasePath"`
@@ -65,8 +67,8 @@ func NewDispatchListener(cfg *Config, prom prometheus.Registerer) *DispatchListe
 
 	if cfg.Enabled {
 		dist := NewDistributor(d.source, d.sink,
-			AnnotationLimiter(kubernetes.WorkspaceCpuLimitAnnotation),
-			AnnotationLimiter(kubernetes.WorkspaceCpuBurstLimitAnnotation),
+			CompositeLimiter(AnnotationLimiter(kubernetes.WorkspaceCpuMinLimitAnnotation), FixedLimiter(BandwidthFromQuantity(d.Config.Limit))),
+			CompositeLimiter(AnnotationLimiter(kubernetes.WorkspaceCpuBurstLimitAnnotation), FixedLimiter(BandwidthFromQuantity(d.Config.BurstLimit))),
 			BandwidthFromQuantity(d.Config.TotalBandwidth),
 		)
 		go dist.Run(context.Background(), time.Duration(d.Config.ControlPeriod))

components/ws-daemon/pkg/cpulimit/limiter_test.go

@@ -37,15 +37,18 @@ func TestBucketLimiter(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.Desc, func(t *testing.T) {
-			//limiter := cpulimit.BucketLimiter(test.Buckets)
-			// ws := &cpulimit.WorkspaceHistory {
-
-			// }
+			limiter := cpulimit.BucketLimiter(test.Buckets)
+			ws := &cpulimit.WorkspaceHistory{
+				LastUpdate: &cpulimit.Workspace{
+					Usage: test.BudgetSpent,
+				},
+				UsageT0: 0,
+			}
 
-			// limit,_ := limiter.Limit(test.BudgetSpent)
-			// if limit != test.ExpectedLimit {
-			// 	t.Errorf("unexpected limit %d: expected %d", limit, test.ExpectedLimit)
-			// }
+			limit, _ := limiter.Limit(ws)
+			if limit != test.ExpectedLimit {
+				t.Errorf("unexpected limit %d: expected %d", limit, test.ExpectedLimit)
+			}
 		})
 	}
 }

components/ws-manager-api/core.proto

@@ -597,6 +597,9 @@ enum WorkspaceFeatureFlag {
 
     // PROTECTED_SECRETS feature flag for enable secrets support
     PROTECTED_SECRETS = 8;
+
+    // WORKSPACE_CLASS_LIMITING feature flag for enabling resuorce limiting based on workspace class
+    WORKSPACE_CLASS_LIMITING = 9;
 }
 
 // GitSpec configures the Git available within the workspace