Add first crictl e2e test framework and suite

This adds a new Makefile target `test-e2e`, which uses the existing
ginkgo framework to run end-to-end tests for `crictl`.

The test framework and suite setups CRI-O in a tmp sandbox, whereas
every test can specify if a runtime is needed or not. This way it is
possible to create a new isolated environment where `crictl` can operate
on. Please be aware that external dependencies like `runc` and the CNI
plugins have to be existend on the system to acually work. The tests
have to be executed as root for now.

An example test suite for the `help` as well as the `info` command has
been added as well.

Signed-off-by: Sascha Grunert <[email protected]>

.travis.yml

@@ -25,6 +25,7 @@ before_install:
       fi
     )
 
+
 install:
   - make install.tools
 
@@ -64,6 +65,27 @@ jobs:
         - powershell -c "Set-ExecutionPolicy Bypass -Scope CURRENTUSER -Force"
         - travis_wait powershell hack/install-kubelet.ps1
         # Skip hack/run-critest.sh temporarily.
+    - stage: Test
+      name: crictl e2e
+      os: linux
+      script:
+        - |
+          sudo apt-get update &&\
+          sudo apt-get install -y libseccomp-dev
+        - |
+          VERSION=v0.8.2 &&\
+          sudo mkdir -p /opt/cni/bin &&\
+          sudo wget -qO- https://github.com/containernetworking/plugins/releases/download/$VERSION/cni-plugins-linux-amd64-$VERSION.tgz \
+              | sudo tar xfz - -C /opt/cni/bin &&\
+          ls -lah /opt/cni/bin
+        - |
+          VERSION=v1.0.0-rc8 &&\
+          sudo wget -q -O \
+            /usr/bin/runc \
+            https://github.com/opencontainers/runc/releases/download/$VERSION/runc.amd64 &&\
+          sudo chmod +x /usr/bin/runc &&\
+          runc --version
+        - sudo -E env "PATH=$PATH" make all install test-e2e TESTFLAGS=-v
 
 stages:
   - Static check

Makefile

@@ -95,6 +95,15 @@ $(GINKGO):
 release:
 	hack/release.sh
 
+# needs to run as root to work
+test-e2e: $(GINKGO)
+	$(GINKGO) $(TESTFLAGS) \
+		-r -p \
+		--randomizeAllSpecs \
+		--randomizeSuites \
+		--succinct \
+		test
+
 vendor:
 	export GO111MODULE=on \
 		$(GO) mod tidy && \
@@ -116,4 +125,5 @@ vendor:
 	lint \
 	install.tools \
 	release \
+	test-e2e \
 	vendor

test/e2e/help_test.go

@@ -0,0 +1,28 @@
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo"
+)
+
+// The actual test suite
+var _ = t.Describe("help", func() {
+
+	const helpMessageIdentifier = "crictl - client for CRI"
+
+	It("should succeed with `help` subcommand", func() {
+		t.CrictlExpectSuccess("help", helpMessageIdentifier)
+	})
+
+	It("should succeed with `--help` flag", func() {
+		t.CrictlExpectSuccess("--help", helpMessageIdentifier)
+	})
+
+	It("should succeed with `-h` flag", func() {
+		t.CrictlExpectSuccess("-h", helpMessageIdentifier)
+	})
+
+	It("should show help on invalid flag", func() {
+		t.CrictlExpectFailure("--invalid", helpMessageIdentifier,
+			"flag provided but not defined")
+	})
+})

test/e2e/suite_test.go

@@ -0,0 +1,31 @@
+package e2e
+
+import (
+	"testing"
+
+	. "github.com/kubernetes-sigs/cri-tools/test/framework"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// TestE2E runs the created specs
+func TestE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "e2e")
+}
+
+var t *TestFramework
+
+var _ = SynchronizedBeforeSuite(func() []byte {
+	// Setup only once
+	dir := SetupCrio()
+	return []byte(dir)
+
+}, func(dir []byte) {
+	t = NewTestFramework()
+	t.Setup(string(dir))
+})
+
+var _ = AfterSuite(func() {
+	t.Teardown()
+})

test/e2e/version_test.go

@@ -0,0 +1,19 @@
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo"
+)
+
+// The actual test suite
+var _ = t.Describe("version", func() {
+	It("should succeed", func() {
+		// Given
+		endpoint, testDir, crio := t.StartCrio()
+
+		// When
+		t.CrictlExpectSuccessWithEndpoint(endpoint, "version", "RuntimeName:  cri-o")
+
+		// Then
+		t.StopCrio(testDir, crio)
+	})
+})

test/framework/framework.go

@@ -0,0 +1,183 @@
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gbytes"
+	. "github.com/onsi/gomega/gexec"
+	"github.com/sirupsen/logrus"
+)
+
+// TestFramework is used to support commonly used test features
+type TestFramework struct {
+	crioDir string
+}
+
+// NewTestFramework creates a new test framework instance
+func NewTestFramework() *TestFramework {
+	return &TestFramework{""}
+}
+
+// Setup is the global initialization function which runs before each test
+// suite
+func (t *TestFramework) Setup(dir string) {
+	// Global initialization for the whole framework goes in here
+	logrus.SetLevel(logrus.DebugLevel)
+	logrus.SetOutput(GinkgoWriter)
+	t.crioDir = dir
+}
+
+// Teardown is the global deinitialization function which runs after each test
+// suite
+func (t *TestFramework) Teardown() {
+}
+
+// Describe is a convenience wrapper around the `ginkgo.Describe` function
+func (t *TestFramework) Describe(text string, body func()) bool {
+	return Describe("crictl: "+text, body)
+}
+
+// Convenience method for command creation
+func cmd(workDir, format string, args ...interface{}) *Session {
+	c := strings.Split(fmt.Sprintf(format, args...), " ")
+	command := exec.Command(c[0], c[1:]...)
+	if workDir != "" {
+		command.Dir = workDir
+	}
+
+	session, err := Start(command, GinkgoWriter, GinkgoWriter)
+	Expect(err).To(BeNil())
+
+	return session
+}
+
+// Convenience method for command creation in the current working directory
+func lcmd(format string, args ...interface{}) *Session {
+	return cmd("", format, args...)
+}
+
+// Run crictl and return the resulting session
+func (t *TestFramework) Crictl(args string) *Session {
+	return lcmd("crictl %s", args).Wait()
+}
+
+// Run crictl on the specified endpoint and return the resulting session
+func (t *TestFramework) CrictlWithEndpoint(endpoint, args string) *Session {
+	return lcmd("crictl --runtime-endpoint=%s %s", endpoint, args).Wait()
+}
+
+// Run crictl and expect success containing the specified output
+func (t *TestFramework) CrictlExpectSuccess(args, expectedOut string) {
+	t.CrictlExpectSuccessWithEndpoint("", args, expectedOut)
+}
+
+// Run crictl and expect success containing the specified output
+func (t *TestFramework) CrictlExpectSuccessWithEndpoint(endpoint, args, expectedOut string) {
+	// When
+	res := t.CrictlWithEndpoint(endpoint, args)
+
+	// Then
+	Expect(res).To(Exit(0))
+	Expect(res.Out).To(Say(expectedOut))
+	Expect(res.Err.Contents()).To(BeEmpty())
+}
+
+// Run crictl and expect error containing the specified outputs
+func (t *TestFramework) CrictlExpectFailure(
+	args string, expectedOut, expectedErr string,
+) {
+	// When
+	res := t.Crictl(args)
+
+	// Then
+	Expect(res).To(Exit(1))
+	Expect(res.Out).To(Say(expectedOut))
+	Expect(res.Err).To(Say(expectedErr))
+}
+
+func SetupCrio() string {
+	const (
+		crioURL = "https://github.com/cri-o/cri-o"
+		timeout = 10 * time.Minute
+	)
+	tmpDir := filepath.Join(os.TempDir(), "crio-tmp")
+
+	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
+		logrus.Info("cloning and building CRI-O")
+		lcmd("git clone --depth=1 %s %s", crioURL, tmpDir).Wait(timeout)
+		cmd(tmpDir, "make").Wait(timeout)
+	}
+
+	return tmpDir
+}
+
+// Start the container runtime process
+func (t *TestFramework) StartCrio() (string, string, *Session) {
+	// Create a new sandbox directory
+	tmpDir, err := ioutil.TempDir("", "crictl-e2e-")
+	Expect(err).To(BeNil())
+
+	// Copy everything together
+	lcmd("cp -R %s %s", filepath.Join(t.crioDir, "bin"), tmpDir).Wait()
+
+	lcmd("cp %s %s", filepath.Join(t.crioDir, "test", "policy.json"),
+		tmpDir).Wait()
+
+	lcmd("cp %s %s", filepath.Join(t.crioDir, "contrib", "cni",
+		"10-crio-bridge.conf"), tmpDir).Wait()
+
+	for _, d := range []string{
+		"cni-config", "root", "runroot", "log", "exits", "attach",
+	} {
+		Expect(os.MkdirAll(filepath.Join(tmpDir, d), 0755)).To(BeNil())
+	}
+
+	endpoint := filepath.Join(tmpDir, "crio.sock")
+
+	session := cmd(tmpDir, "%s"+
+		" --listen=%s"+
+		" --conmon=%s"+
+		" --container-exits-dir=%s"+
+		" --container-attach-socket-dir=%s"+
+		" --log-dir=%s"+
+		" --signature-policy=%s"+
+		" --cni-config-dir=%s"+
+		" --root=%s"+
+		" --runroot=%s"+
+		" --storage-driver=vfs",
+		filepath.Join(tmpDir, "bin", "crio"),
+		endpoint,
+		filepath.Join(tmpDir, "bin", "conmon"),
+		filepath.Join(tmpDir, "exits"),
+		filepath.Join(tmpDir, "attach"),
+		filepath.Join(tmpDir, "log"),
+		filepath.Join(tmpDir, "policy.json"),
+		filepath.Join(tmpDir, "cni-config"),
+		filepath.Join(tmpDir, "root"),
+		filepath.Join(tmpDir, "runroot"),
+	)
+
+	// Wait for the connection to be available
+	for i := 0; i < 100; i++ {
+		res := t.CrictlWithEndpoint(endpoint, "--timeout=200ms info")
+		if res.ExitCode() == 0 {
+			break
+		}
+		logrus.Info("waiting for CRI-O to become ready")
+	}
+	return endpoint, tmpDir, session
+}
+
+// Stop the container runtime process
+func (t *TestFramework) StopCrio(testDir string, session *Session) {
+	Expect(session.Interrupt().Wait()).To(Exit(0))
+	Expect(os.RemoveAll(testDir)).To(BeNil())
+}