From bc6c9f72c0d0a5a1085e5d37d67c01f1a5af4c31 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Fri, 9 Jul 2021 12:49:13 +0000 Subject: [PATCH] [ws-daemon] cgroup customizer on workspace add --- components/ws-daemon/go.mod | 1 + components/ws-daemon/go.sum | 4 ++ .../ws-daemon/pkg/daemon/cgroup_customizer.go | 71 +++++++++++++++++++ components/ws-daemon/pkg/daemon/daemon.go | 3 + 4 files changed, 79 insertions(+) create mode 100644 components/ws-daemon/pkg/daemon/cgroup_customizer.go diff --git a/components/ws-daemon/go.mod b/components/ws-daemon/go.mod index db84d672d71c35..37fe63ce15c9dd 100644 --- a/components/ws-daemon/go.mod +++ b/components/ws-daemon/go.mod @@ -6,6 +6,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect github.com/alecthomas/jsonschema v0.0.0-20190504002508-159cbd5dba26 github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c + github.com/containerd/cgroups v1.0.1 github.com/containerd/containerd v1.5.2 github.com/containerd/typeurl v1.0.2 github.com/docker/docker v20.10.5+incompatible diff --git a/components/ws-daemon/go.sum b/components/ws-daemon/go.sum index d06b5d0eb18d76..ab814534f67be9 100644 --- a/components/ws-daemon/go.sum +++ b/components/ws-daemon/go.sum @@ -236,8 +236,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -332,7 +334,9 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= diff --git a/components/ws-daemon/pkg/daemon/cgroup_customizer.go b/components/ws-daemon/pkg/daemon/cgroup_customizer.go new file mode 100644 index 00000000000000..937869591d648f --- /dev/null +++ b/components/ws-daemon/pkg/daemon/cgroup_customizer.go @@ -0,0 +1,71 @@ +// Copyright (c) 2021 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package daemon + +import ( + "context" + + "github.com/containerd/cgroups" + "github.com/gitpod-io/gitpod/ws-daemon/pkg/dispatch" + "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/xerrors" +) + +var ( + fuseDeviceMajor int64 = 10 + fuseDeviceMinor int64 = 229 +) + +type CgroupCustomizer struct { + cgroupBasePath string +} + +func (c *CgroupCustomizer) WithCgroupBasePath(basePath string) { + c.cgroupBasePath = basePath +} + +// WorkspaceAdded will customize the cgroups for every workspace that is started +func (c *CgroupCustomizer) WorkspaceAdded(ctx context.Context, ws *dispatch.Workspace) error { + disp := dispatch.GetFromContext(ctx) + if disp == nil { + return xerrors.Errorf("no dispatch available") + } + + cgroupPath, err := disp.Runtime.ContainerCGroupPath(context.Background(), ws.ContainerID) + if err != nil { + return xerrors.Errorf("cannot start governer: %w", err) + } + + control, err := cgroups.Load(c.customV1, cgroups.StaticPath(cgroupPath)) + + if err != nil { + return xerrors.Errorf("error loading cgroup at path: %s %w", cgroupPath, err) + } + + res := &specs.LinuxResources{ + Devices: []specs.LinuxDeviceCgroup{ + // /dev/fuse + { + Type: "c", + Minor: &fuseDeviceMinor, + Major: &fuseDeviceMajor, + Access: "rwm", + Allow: true, + }, + }, + } + + if err := control.Update(res); err != nil { + return xerrors.Errorf("cgroup update failed: %w", err) + } + + return nil +} + +func (c *CgroupCustomizer) customV1() ([]cgroups.Subsystem, error) { + return []cgroups.Subsystem{ + cgroups.NewDevices(c.cgroupBasePath), + }, nil +} diff --git a/components/ws-daemon/pkg/daemon/daemon.go b/components/ws-daemon/pkg/daemon/daemon.go index 0e36f093dc0bb9..855fda0a23c282 100644 --- a/components/ws-daemon/pkg/daemon/daemon.go +++ b/components/ws-daemon/pkg/daemon/daemon.go @@ -50,9 +50,12 @@ func NewDaemon(config Config, reg prometheus.Registerer) (*Daemon, error) { if nodename == "" { return nil, xerrors.Errorf("NODENAME env var isn't set") } + cgCustomizer := &CgroupCustomizer{} + cgCustomizer.WithCgroupBasePath(config.Resources.CGroupsBasePath) dsptch, err := dispatch.NewDispatch(containerRuntime, clientset, config.Runtime.KubernetesNamespace, nodename, resources.NewDispatchListener(&config.Resources, reg), &Containerd4214Workaround{}, + cgCustomizer, ) if err != nil { return nil, err