Merge pull request #4494 from ianlewis/gvisor-containerd-shim_v0.0.2

Update gvisor runsc version

deploy/addons/gvisor/README.md

@@ -1,5 +1,5 @@
 ## gVisor Addon
-[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
+[gVisor](https://gvisor.dev/), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
 
 ### Starting Minikube
 gVisor depends on the containerd runtime to run in Minikube.
@@ -17,21 +17,27 @@ To enable this addon, simply run:
 $ minikube addons enable gvisor
 ```
 
-Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod:
+Within one minute, the addon manager should pick up the change and you should
+see the `gvisor` pod and `gvisor` [Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/):
 
 ```
-$ kubectl get pod gvisor -n kube-system
-NAME      READY     STATUS    RESTARTS   AGE
-gvisor    1/1       Running   0          3m
+$ kubectl get pod,runtimeclass gvisor -n kube-system
+NAME         READY   STATUS    RESTARTS   AGE
+pod/gvisor   1/1     Running   0          2m52s
+
+NAME                              CREATED AT
+runtimeclass.node.k8s.io/gvisor   2019-06-15T04:35:09Z
 ```
 
-Once the pod has status `Running`, gVisor is enabled in Minikube. 
+Once the pod has status `Running`, gVisor is enabled in Minikube.
 
 ### Running pods in gVisor
-To run a pod in gVisor, add this annotation to the Kubernetes yaml:
+
+To run a pod in gVisor, add the `gvisor` runtime class to the Pod spec in your
+Kubernetes yaml:
 
 ```
-io.kubernetes.cri.untrusted-workload: "true"
+runtimeClassName: gvisor
 ```
 
 An example Pod is shown below:
@@ -41,17 +47,15 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: nginx-untrusted
-  annotations:
-    io.kubernetes.cri.untrusted-workload: "true"
 spec:
+  runtimeClassName: gvisor
   containers:
   - name: nginx
     image: nginx
 ```
 
-_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._
-
 ### Disabling gVisor
+
 To disable gVisor, run:
 
 ```
@@ -67,4 +71,4 @@ NAME      READY     STATUS        RESTARTS   AGE
 gvisor    1/1       Terminating   0          5m
 ```
 
-_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._
+_Note: Once gVisor is disabled, any pod with the `gvisor` Runtime Class or `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._

deploy/addons/gvisor/gvisor-config.toml

@@ -41,10 +41,10 @@ oom_score = 0
         runtime_type = "io.containerd.runtime.v1.linux"
         runtime_engine = ""
         runtime_root = ""
-      [plugins.cri.containerd.untrusted_workload_runtime]
-        runtime_type = "io.containerd.runtime.v1.linux"
-        runtime_engine = "/usr/local/bin/runsc"
-        runtime_root = "/run/containerd/runsc"
+      [plugins.cri.containerd.runtimes.untrusted]
+        runtime_type = "io.containerd.runsc.v1"
+      [plugins.cri.containerd.runtimes.runsc]
+        runtime_type = "io.containerd.runsc.v1"
     [plugins.cri.cni]
       bin_dir = "/opt/cni/bin"
       conf_dir = "/etc/cni/net.d"
@@ -56,7 +56,6 @@ oom_score = 0
   [plugins.diff-service]
     default = ["walking"]
   [plugins.linux]
-    shim = "gvisor-containerd-shim"
     runtime = "runc"
     runtime_root = ""
     no_shim = false
@@ -66,4 +65,4 @@ oom_score = 0
     deletion_threshold = 0
     mutation_threshold = 100
     schedule_delay = "0s"
-    startup_delay = "100ms"
+    startup_delay = "100ms"

deploy/addons/gvisor/gvisor-pod.yaml.tmpl

@@ -14,15 +14,15 @@
 
 apiVersion: v1
 kind: Pod
-metadata: 
+metadata:
   name: gvisor
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
     kubernetes.io/minikube-addons: gvisor
-spec: 
+spec:
   hostPID: true
-  containers: 
+  containers:
     - name: gvisor
       image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest
       securityContext:

deploy/addons/gvisor/gvisor-runtimeclass.yaml

@@ -0,0 +1,22 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: node.k8s.io/v1beta1
+kind: RuntimeClass
+metadata:
+  name: gvisor
+  labels:
+    kubernetes.io/minikube-addons: gvisor
+    addonmanager.kubernetes.io/mode: Reconcile
+handler: runsc

pkg/gvisor/enable.go

@@ -80,12 +80,6 @@ func makeGvisorDirs() error {
 		return errors.Wrap(err, "creating runsc dir")
 	}
 
-	// Make /usr/local/bin to store the runsc binary
-	fp = filepath.Join(nodeDir, "usr/local/bin")
-	if err := os.MkdirAll(fp, 0755); err != nil {
-		return errors.Wrap(err, "creating usr/local/bin dir")
-	}
-
 	// Make /tmp/runsc to also hold logs
 	fp = filepath.Join(nodeDir, "tmp/runsc")
 	if err := os.MkdirAll(fp, 0755); err != nil {
@@ -107,13 +101,13 @@ func downloadBinaries() error {
 
 // downloads the gvisor-containerd-shim
 func gvisorContainerdShim() error {
-	dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim")
+	dest := filepath.Join(nodeDir, "usr/bin/containerd-shim-runsc-v1")
 	return downloadFileToDest(constants.GvisorContainerdShimURL, dest)
 }
 
 // downloads the runsc binary and returns a path to the binary
 func runsc() error {
-	dest := filepath.Join(nodeDir, "usr/local/bin/runsc")
+	dest := filepath.Join(nodeDir, "usr/bin/runsc")
 	return downloadFileToDest(constants.GvisorURL, dest)
 }
 
@@ -159,10 +153,6 @@ func copyConfigFiles() error {
 	if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying default config.toml")
 	}
-	log.Print("Copying gvisor-containerd-shim.toml...")
-	if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil {
-		return errors.Wrap(err, "copying gvisor-containerd-shim.toml")
-	}
 	log.Print("Copying containerd config.toml with gvisor...")
 	if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying gvisor version of config.toml")

pkg/minikube/assets/addons.go

@@ -314,18 +314,18 @@ var Addons = map[string]*Addon{
 			"gvisor-pod.yaml",
 			"0640",
 			true),
+		MustBinAsset(
+			"deploy/addons/gvisor/gvisor-runtimeclass.yaml",
+			constants.GuestAddonsDir,
+			"gvisor-runtimeclass.yaml",
+			"0640",
+			false),
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-config.toml",
 			constants.GvisorFilesPath,
 			constants.GvisorConfigTomlTargetName,
 			"0640",
 			true),
-		MustBinAsset(
-			"deploy/addons/gvisor/gvisor-containerd-shim.toml",
-			constants.GvisorFilesPath,
-			constants.GvisorContainerdShimTargetName,
-			"0640",
-			false),
 	}, false, "gvisor"),
 }
 

pkg/minikube/constants/constants.go

@@ -384,20 +384,16 @@ const (
 	GvisorFilesPath = "/tmp/gvisor"
 	// ContainerdConfigTomlPath is the path to the containerd config.toml
 	ContainerdConfigTomlPath = "/etc/containerd/config.toml"
-	// GvisorContainerdShimTomlPath is the path to gvisor-containerd-shim.toml
-	GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml"
 	// StoredContainerdConfigTomlPath is the path where the default config.toml will be stored
 	StoredContainerdConfigTomlPath = "/tmp/config.toml"
 
 	// GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml
 	GvisorConfigTomlTargetName = "gvisor-config.toml"
-	// GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim
-	GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml"
 
 	// GvisorContainerdShimURL is the url to download gvisor-containerd-shim
-	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64"
+	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.3/containerd-shim-runsc-v1.linux-amd64"
 	// GvisorURL is the url to download gvisor
-	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc"
+	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2019-01-14/runsc"
 )
 
 const (

test/integration/containerd_test.go

@@ -29,27 +29,57 @@ import (
 	"k8s.io/minikube/test/integration/util"
 )
 
-func TestContainerd(t *testing.T) {
+// TestGvisorWorkload tests the gVisor addon for pods with the untrusted
+// workload annotation and gvisor runtime class.
+func TestGvisorWorkload(t *testing.T) {
 	if isTestNoneDriver(t) {
 		t.Skip("Can't run containerd backend with none driver")
 	}
-	if shouldRunInParallel(t) {
-		t.Parallel()
+
+	p := profileName(t)
+	mk := NewMinikubeRunner(t, p, "--wait=false")
+	defer mk.TearDown(t)
+
+	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
+	mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
+	// NOTE: addons are enabled globally.
+	mk.MustRun("addons enable gvisor")
+
+	t.Log("waiting for gvisor controller to come up")
+	if err := waitForGvisorControllerRunning(p); err != nil {
+		t.Fatalf("waiting for gvisor controller to be up: %v", err)
 	}
 
-	t.Run("GvisorRestart", testGvisorRestart)
+	// Test a pod with the untrusted workload annotation.
+	createUntrustedWorkload(t, p)
+	t.Log("making sure untrusted workload is Running")
+	if err := waitForUntrustedNginxRunning(p); err != nil {
+		t.Fatalf("waiting for untrusted-workload nginx to be up: %v", err)
+	}
+	deleteUntrustedWorkload(t, p)
+
+	// Test a pod with the gvisor runtime class.
+	createGvisorWorkload(t, p)
+	t.Log("making sure gvisor workload is Running")
+	if err := waitForGvisorNginxRunning(p); err != nil {
+		t.Fatalf("waiting for gvisor nginx to be up: %v", err)
+	}
+	deleteGvisorWorkload(t, p)
 }
 
-func testGvisorRestart(t *testing.T) {
-	p := profileName(t)
-	if shouldRunInParallel(t) {
-		t.Parallel()
+// TestGvisorRestart tests the gVisor addon is functional after a VM restart.
+func TestGvisorRestart(t *testing.T) {
+	if isTestNoneDriver(t) {
+		t.Skip("Can't run containerd backend with none driver")
 	}
+
+	p := profileName(t)
 	mk := NewMinikubeRunner(t, p, "--wait=false")
 	defer mk.TearDown(t)
 
 	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
 	mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
+	// NOTE: addons are enabled globally.
 	mk.MustRun("addons enable gvisor")
 
 	t.Log("waiting for gvisor controller to come up")
@@ -98,7 +128,24 @@ func deleteUntrustedWorkload(t *testing.T, profile string) {
 	}
 }
 
-// waitForGvisorControllerRunning waits for the gvisor controller pod to be running
+func createGvisorWorkload(t *testing.T, profile string) {
+	kr := util.NewKubectlRunner(t, profile)
+	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
+	t.Log("creating pod with gvisor workload annotation")
+	if _, err := kr.RunCommand([]string{"replace", "-f", gvisorPath, "--force"}); err != nil {
+		t.Fatalf("creating gvisor nginx resource: %v", err)
+	}
+}
+
+func deleteGvisorWorkload(t *testing.T, profile string) {
+	kr := util.NewKubectlRunner(t, profile)
+	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
+	if _, err := kr.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil {
+		t.Logf("error deleting gvisor nginx resource: %v", err)
+	}
+}
+
+// waitForGvisorControllerRunning waits for the gvisor controller pod to be running.
 func waitForGvisorControllerRunning(p string) error {
 	client, err := kapi.Client(p)
 	if err != nil {
@@ -112,14 +159,30 @@ func waitForGvisorControllerRunning(p string) error {
 	return nil
 }
 
-// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start running
+// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start
+// running.
 func waitForUntrustedNginxRunning(miniProfile string) error {
 	client, err := kapi.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
-	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "untrusted": "true"}))
+	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+		return errors.Wrap(err, "waiting for nginx pods")
+	}
+	return nil
+}
+
+// waitForGvisorNginxRunning waits for the nginx pod with gvisor runtime class
+// to start running.
+func waitForGvisorNginxRunning(miniProfile string) error {
+	client, err := kapi.Client(miniProfile)
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "runtime": "gvisor"}))
 	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return errors.Wrap(err, "waiting for nginx pods")
 	}

test/integration/testdata/nginx-gvisor.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-gvisor
+  labels:
+    run: nginx
+    runtime: gvisor
+spec:
+  runtimeClassName: gvisor
+  containers:
+  - name: nginx
+    image: nginx

test/integration/testdata/nginx-untrusted.yaml

@@ -4,6 +4,7 @@ metadata:
   name: nginx-untrusted
   labels:
     run: nginx
+    untrusted: "true"
   annotations:
     io.kubernetes.cri.untrusted-workload: "true"
 spec: