Merge pull request #831 from Mirantis/jell/ucr

Set cpuset for emulator during UpdateContainerResources

deploy/data/virtlet-ds.yaml

@@ -213,6 +213,8 @@ spec:
         - name: netns-dir
           mountPath: /var/run/netns
           mountPropagation: Bidirectional
+        - name: cgroup
+          mountPath: /sys/fs/cgroup
         securityContext:
           privileged: true
         readinessProbe:

pkg/libvirttools/cpusets.go

@@ -0,0 +1,138 @@
+/*
+Copyright 2018 Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package libvirttools
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	libvirtxml "github.com/libvirt/libvirt-go-xml"
+
+	vconfig "github.com/Mirantis/virtlet/pkg/config"
+	"github.com/Mirantis/virtlet/pkg/utils/cgroups"
+)
+
+const (
+	procfsLocation      = "/proc"
+	emulatorProcessName = "qemu-system-x86_64"
+)
+
+// UpdateCpusetsInContainerDefinition updates libvirt domain definition for the VM
+// setting the environment variable which is used by vmwrapper to pin to the specified cpuset
+func (v *VirtualizationTool) UpdateCpusetsInContainerDefinition(containerID, cpusets string) error {
+	domain, err := v.domainConn.LookupDomainByUUIDString(containerID)
+	if err != nil {
+		return err
+	}
+
+	domainxml, err := domain.XML()
+	if err != nil {
+		return err
+	}
+
+	found := false
+	envvars := domainxml.QEMUCommandline.Envs
+	for _, envvar := range envvars {
+		if envvar.Name == vconfig.CpusetsEnvVarName {
+			envvar.Value = cpusets
+			found = true
+		}
+	}
+	if !found && cpusets != "" {
+		domainxml.QEMUCommandline.Envs = append(envvars, libvirtxml.DomainQEMUCommandlineEnv{
+			Name:  vconfig.CpusetsEnvVarName,
+			Value: cpusets,
+		})
+	}
+
+	if err := domain.Undefine(); err != nil {
+		return err
+	}
+
+	_, err = v.domainConn.DefineDomain(domainxml)
+	return err
+}
+
+// UpdateCpusetsForEmulatorProcess looks through /proc for emulator process
+// to find its cgroup manager for cpusets then uses it to adjust the setting
+func (v *VirtualizationTool) UpdateCpusetsForEmulatorProcess(containerID, cpusets string) (bool, error) {
+	// TODO: replace iterating over the procfs with reading pid from
+	// /run/libvirt/qemu/virtlet-CONTAINER_ID[:12]-DOMAIN_NAME.pid
+	d, err := os.Open(procfsLocation)
+	if err != nil {
+		return false, err
+	}
+	defer d.Close()
+
+	entries, err := d.Readdirnames(-1)
+	if err != nil {
+		return false, err
+	}
+
+	for _, name := range entries {
+		_, err := strconv.ParseInt(name, 10, 32)
+		if err != nil {
+			// skip non numeric names
+			continue
+		}
+
+		isContainerPid, err := isEmulatorPid(name, containerID)
+		if err != nil {
+			return false, err
+		}
+
+		if isContainerPid {
+			controller, err := cgroups.GetProcessController(name, "cpuset")
+			if err != nil {
+				return false, err
+			}
+
+			if err := controller.Set("cpus", cpusets); err != nil {
+				return false, err
+			}
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func isEmulatorPid(pid, containerID string) (bool, error) {
+	data, err := ioutil.ReadFile(filepath.Join(procfsLocation, pid, "cmdline"))
+	if err != nil {
+		return false, err
+	}
+
+	cmdline := bytes.Split(data, []byte{0})
+
+	if string(cmdline[0]) != emulatorProcessName {
+		return false, nil
+	}
+
+	searchTerm := "virtlet-" + containerID[:12]
+	for _, param := range cmdline {
+		if strings.Contains(string(param), searchTerm) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

pkg/libvirttools/virtualization.go

@@ -818,43 +818,6 @@ func (v *VirtualizationTool) ContainerInfo(containerID string) (*types.Container
 	return containerInfo, nil
 }
 
-// UpdateCpusetsInContainerDefinition updates domain definition in libvirt for the VM
-// setting environment variable for vmwrapper with info about to which cpuset it should
-// pin itself
-func (v *VirtualizationTool) UpdateCpusetsInContainerDefinition(containerID, cpusets string) error {
-	domain, err := v.domainConn.LookupDomainByUUIDString(containerID)
-	if err != nil {
-		return err
-	}
-
-	domainxml, err := domain.XML()
-	if err != nil {
-		return err
-	}
-
-	found := false
-	envvars := domainxml.QEMUCommandline.Envs
-	for _, envvar := range envvars {
-		if envvar.Name == vconfig.CpusetsEnvVarName {
-			envvar.Value = cpusets
-			found = true
-		}
-	}
-	if !found && cpusets != "" {
-		domainxml.QEMUCommandline.Envs = append(envvars, libvirtxml.DomainQEMUCommandlineEnv{
-			Name:  vconfig.CpusetsEnvVarName,
-			Value: cpusets,
-		})
-	}
-
-	if err := domain.Undefine(); err != nil {
-		return err
-	}
-
-	_, err = v.domainConn.DefineDomain(domainxml)
-	return err
-}
-
 // VMStats returns current cpu/memory/disk usage for VM
 func (v *VirtualizationTool) VMStats(containerID string, name string) (*types.VMStats, error) {
 	domain, err := v.domainConn.LookupDomainByUUIDString(containerID)

pkg/manager/runtime.go

@@ -430,12 +430,19 @@ func (v *VirtletRuntimeService) UpdateRuntimeConfig(context.Context, *kubeapi.Up
 	return &kubeapi.UpdateRuntimeConfigResponse{}, nil
 }
 
-// UpdateContainerResources passes info about Cpusets for particular container
-// to virttool
+// UpdateContainerResources stores in domain on libvirt info about Cpuset
+// for container then looks for running emulator and tries to adjust its
+// current settings through cgroups
 func (v *VirtletRuntimeService) UpdateContainerResources(ctx context.Context, req *kubeapi.UpdateContainerResourcesRequest) (*kubeapi.UpdateContainerResourcesResponse, error) {
-	if err := v.virtTool.UpdateCpusetsInContainerDefinition(req.GetContainerId(), req.GetLinux().CpusetCpus); err != nil {
+	setByCgroup, err := v.virtTool.UpdateCpusetsForEmulatorProcess(req.GetContainerId(), req.GetLinux().CpusetCpus)
+	if err != nil {
 		return nil, err
 	}
+	if !setByCgroup {
+		if err = v.virtTool.UpdateCpusetsInContainerDefinition(req.GetContainerId(), req.GetLinux().CpusetCpus); err != nil {
+			return nil, err
+		}
+	}
 	return &kubeapi.UpdateContainerResourcesResponse{}, nil
 }
 

pkg/tools/TestGenCommand__compat.out.yaml

@@ -108,6 +108,8 @@ spec:
         - mountPath: /var/run/netns
           mountPropagation: Bidirectional
           name: netns-dir
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
       - command:
         - /vms.sh
         image: mirantis/virtlet

pkg/tools/TestGenCommand__compat_dev.out.yaml

@@ -110,6 +110,8 @@ spec:
         - mountPath: /var/run/netns
           mountPropagation: Bidirectional
           name: netns-dir
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
         - mountPath: /dind
           name: dind
       - command:

pkg/tools/TestGenCommand__dev.out.yaml

@@ -111,6 +111,8 @@ spec:
         - mountPath: /var/run/netns
           mountPropagation: Bidirectional
           name: netns-dir
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
         - mountPath: /dind
           name: dind
       - command:

pkg/tools/TestGenCommand__plain.out.yaml

@@ -109,6 +109,8 @@ spec:
         - mountPath: /var/run/netns
           mountPropagation: Bidirectional
           name: netns-dir
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
       - command:
         - /vms.sh
         image: mirantis/virtlet

pkg/tools/TestGenCommand__tag.out.yaml

@@ -109,6 +109,8 @@ spec:
         - mountPath: /var/run/netns
           mountPropagation: Bidirectional
           name: netns-dir
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
       - command:
         - /vms.sh
         image: mirantis/virtlet:0.9.42

pkg/utils/cgroups/controllers.go

@@ -37,15 +37,15 @@ const (
 func GetProcessControllers(pid interface{}) (map[string]string, error) {
 	sPid := utils.Stringify(pid)
 
-	file, err := os.Open(filepath.Join("/proc", sPid, "cgroups"))
+	file, err := os.Open(filepath.Join("/proc", sPid, "cgroup"))
 	if err != nil {
 		return nil, err
 	}
 	defer file.Close()
 
 	reader := bufio.NewReader(file)
 
-	var ctrls map[string]string
+	ctrls := make(map[string]string)
 
 	for {
 		line, err := reader.ReadString('\n')