From 8f92c531e70d1ae70eac06561b47d2f838c980e3 Mon Sep 17 00:00:00 2001
From: Claudia Beresford <claudiaberesford@gmail.com>
Date: Mon, 8 Nov 2021 12:33:31 +0000
Subject: [PATCH] Add thinpool setup to general test Setup/Teardown

I had this being done in bash via a Docker Entrypoint + `trap`
combination, but it was quite brittle, and sometimes the teardown would
not run which then meant you had to run a manual step.

This changes brings that into the test so the setup and teardown can be
handled programatically.
---
 Makefile                   |   3 +-
 go.mod                     |   7 +-
 go.sum                     |   5 +-
 test/docker/Dockerfile.e2e |   4 +-
 test/docker/entrypoint.sh  |  42 ------------
 test/e2e/utils/runner.go   | 128 ++++++++++++++++++++++++++++++++-----
 test/tools/metal.py        |   2 +-
 test/tools/userdata.sh     |  24 -------
 8 files changed, 120 insertions(+), 95 deletions(-)
 delete mode 100755 test/docker/entrypoint.sh

diff --git a/Makefile b/Makefile
index 9d4bc6491..1514b8d63 100644
--- a/Makefile
+++ b/Makefile
@@ -114,10 +114,11 @@ test-e2e-docker: compile-e2e ## Run e2e tests locally in a container
 		--ipc=host \
 		--workdir=/src/flintlock \
 		$(test_image):latest \
-		"make test-e2e"
+		/bin/bash -c "make test-e2e"
 
 .PHONY: test-e2e-metal
 test-e2e-metal: ## Run e2e tests in Equinix
+	echo "must set EQUINIX_ORG_ID"
 	./test/tools/run.py run-e2e -o $(EQUINIX_ORG_ID)
 
 .PHONY: compile-e2e
diff --git a/go.mod b/go.mod
index 9180ca890..e2e9200c2 100644
--- a/go.mod
+++ b/go.mod
@@ -24,6 +24,7 @@ require (
 	github.com/onsi/gomega v1.16.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/pelletier/go-toml v1.9.4
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/afero v1.6.0
@@ -74,10 +75,10 @@ require (
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/subcommands v1.0.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
@@ -90,7 +91,6 @@ require (
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
 	github.com/opencontainers/selinux v1.8.2 // indirect
-	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
@@ -103,12 +103,9 @@ require (
 	go.mongodb.org/mongo-driver v1.5.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
 	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.5 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/genproto v0.0.0-20211021150943-2b146023228c // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
diff --git a/go.sum b/go.sum
index 47c2519ce..73d4e84a2 100644
--- a/go.sum
+++ b/go.sum
@@ -281,6 +281,7 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6Uezg
 github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
@@ -524,7 +525,6 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k=
 github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -595,6 +595,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
@@ -1023,7 +1024,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1292,7 +1292,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/test/docker/Dockerfile.e2e b/test/docker/Dockerfile.e2e
index dd7a185c7..ad34ad101 100644
--- a/test/docker/Dockerfile.e2e
+++ b/test/docker/Dockerfile.e2e
@@ -7,6 +7,4 @@ RUN chmod +x /tmp/bootstrap.sh
 RUN /tmp/bootstrap.sh
 
 COPY hack/scripts/devpool.sh /tmp/devpool.sh
-RUN  chmod +x /tmp/devpool.sh
-COPY test/docker/entrypoint.sh /entrypoint
-ENTRYPOINT ["/entrypoint"]
+RUN chmod +x /tmp/devpool.sh
diff --git a/test/docker/entrypoint.sh b/test/docker/entrypoint.sh
deleted file mode 100755
index 0c0ceda7f..000000000
--- a/test/docker/entrypoint.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash -ex
-
-THINPOOL="dev-thinpool-e2e"
-LOOP_TAG="e2e"
-CTRD_ROOT="/var/lib/containerd-dev"
-CTRD_STATE="/run/containerd-dev"
-CTRD_CFG="/etc/containerd"
-DM_ROOT="$CTRD_ROOT/snapshotter/devmapper"
-
-cleanup() {
-    dmsetup ls | awk -v pool="$THINPOOL" '$0 ~ pool {print $1}' | xargs -I {} dmsetup remove {} --force
-    losetup | awk -v loop="$LOOP_TAG" '$0 ~ loop {print $1}' | xargs losetup -d
-}
-
-trap 'true' SIGINT SIGTERM
-
-/tmp/devpool.sh "$THINPOOL" "$LOOP_TAG"
-
-mkdir -p "$DM_ROOT" "$CTRD_STATE" "$CTRD_CFG"
-cat > "$CTRD_CFG/config-dev.toml" <<EOF
-version = 2
-root = "$CTRD_ROOT"
-state = "$CTRD_STATE"
-[grpc]
-  address = "$CTRD_STATE/containerd.sock"
-[metrics]
-  address = "127.0.0.1:1338"
-[plugins]
-  [plugins."io.containerd.snapshotter.v1.devmapper"]
-    pool_name = "$THINPOOL"
-    root_path = "$DM_ROOT"
-    base_image_size = "10GB"
-    discard_blocks = true
-[debug]
-  level = "trace"
-EOF
-
-exec /bin/bash -c "${@}" &
-
-wait $!
-
-cleanup
diff --git a/test/e2e/utils/runner.go b/test/e2e/utils/runner.go
index 0d2008dda..ecc155568 100644
--- a/test/e2e/utils/runner.go
+++ b/test/e2e/utils/runner.go
@@ -1,27 +1,36 @@
-//go:build e2e
-// +build e2e
-
 package utils
 
 import (
 	"bufio"
 	"errors"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
+	ccfg "github.com/containerd/containerd/services/server/config"
+	"github.com/containerd/containerd/snapshots/devmapper/dmsetup"
 	gk "github.com/onsi/ginkgo"
 	gm "github.com/onsi/gomega"
 	"github.com/onsi/gomega/gexec"
+	"github.com/pelletier/go-toml"
 	"github.com/weaveworks/flintlock/api/services/microvm/v1alpha1"
 	"google.golang.org/grpc"
 )
 
 const (
-	containerdBin    = "containerd"
-	flintlockCmdDir  = "github.com/weaveworks/flintlock/cmd/flintlockd"
-	containerdSocket = "/run/containerd-dev/containerd.sock"
-	containerdCfg    = "/etc/containerd/config-dev.toml"
-	grpcDialTarget   = "127.0.0.1:9090"
+	containerdBin         = "containerd"
+	flintlockCmdDir       = "github.com/weaveworks/flintlock/cmd/flintlockd"
+	containerdSocket      = "/run/containerd-e2e/containerd.sock"
+	containerdCfgDir      = "/etc/containerd/config-e2e.toml"
+	containerdCfg         = containerdCfgDir + "/config-e2e.toml"
+	containerdRootDir     = "/var/lib/containerd-e2e"
+	containerdStateDir    = "/run/containerd-e2e"
+	grpcDialTarget        = "127.0.0.1:9090"
+	thinpoolName          = "dev-thinpool-e2e"
+	loopDeviceTag         = "e2e"
+	containerdGrpcAddress = containerdStateDir + "/containerd.sock"
+	devMapperRoot         = containerdRootDir + "/snapshotter/devmapper"
 )
 
 // Runner is a very poorly named thing and honestly idk what to call it.
@@ -36,6 +45,8 @@ type Runner struct {
 }
 
 // Setup is a helper for the e2e tests which:
+// - sets up up devicemapper thinpools
+// - writes containerd config
 // - compiles flintlockd
 // - starts containerd
 // - starts flintlockd
@@ -44,6 +55,9 @@ type Runner struct {
 // All opened connections and started processes are saved for later shutdown.
 // Teardown should be called before Setup in a defer.
 func (r *Runner) Setup() v1alpha1.MicroVMClient {
+	makeDirectories()
+	createThinPools()
+	writeContainerdConfig()
 	r.buildFLBinary()
 	r.startContainerd()
 	r.startFlintlockd()
@@ -73,9 +87,78 @@ func (r *Runner) Teardown() {
 		r.containerdSession.Terminate().Wait()
 	}
 
+	cleanupThinPools()
+	cleanupDirectories()
+
 	gexec.CleanupBuildArtifacts()
 }
 
+func makeDirectories() {
+	gm.Expect(os.MkdirAll(devMapperRoot, os.ModePerm)).To(gm.Succeed())
+	gm.Expect(os.MkdirAll(containerdStateDir, os.ModePerm)).To(gm.Succeed())
+	gm.Expect(os.MkdirAll(containerdCfgDir, os.ModePerm)).To(gm.Succeed())
+}
+
+func cleanupDirectories() {
+	gm.Expect(os.RemoveAll(containerdCfgDir)).To(gm.Succeed())
+	gm.Expect(os.RemoveAll(containerdRootDir)).To(gm.Succeed())
+	gm.Expect(os.RemoveAll(containerdStateDir)).To(gm.Succeed())
+}
+
+func createThinPools() {
+	scriptPath := filepath.Join(baseDir(), "hack", "scripts", "devpool.sh")
+	command := exec.Command(scriptPath, thinpoolName, loopDeviceTag)
+	session, err := gexec.Start(command, gk.GinkgoWriter, gk.GinkgoWriter)
+	gm.Expect(err).NotTo(gm.HaveOccurred())
+	gm.Eventually(session).Should(gexec.Exit(0))
+}
+
+func cleanupThinPools() {
+	gm.Expect(dmsetup.RemoveDevice(thinpoolName, dmsetup.RemoveWithForce)).To(gm.Succeed())
+
+	cmd := exec.Command("losetup")
+	loopDevices := grep(cmd, loopDeviceTag, 0)
+	for _, dev := range loopDevices {
+		command := exec.Command("losetup", "-d", dev)
+		session, err := gexec.Start(command, gk.GinkgoWriter, gk.GinkgoWriter)
+		gm.Expect(err).NotTo(gm.HaveOccurred())
+		gm.Eventually(session).Should(gexec.Exit(0))
+	}
+}
+
+func writeContainerdConfig() {
+	dmplug := map[string]interface{}{
+		"pool_name":       thinpoolName,
+		"root_path":       devMapperRoot,
+		"base_image_size": "10GB",
+		"discard_blocks":  "true",
+	}
+	pluginTree, err := toml.TreeFromMap(dmplug)
+	gm.Expect(err).NotTo(gm.HaveOccurred())
+	cfg := ccfg.Config{
+		Version: 2, //nolint:gomnd
+		Root:    containerdRootDir,
+		State:   containerdStateDir,
+		GRPC: ccfg.GRPCConfig{
+			Address: containerdGrpcAddress,
+		},
+		Metrics: ccfg.MetricsConfig{
+			Address: "127.0.0.1:1338",
+		},
+		Plugins: map[string]toml.Tree{
+			"io.containerd.snapshotter.v1.devmapper": *pluginTree,
+		},
+		Debug: ccfg.Debug{
+			Level: "trace",
+		},
+	}
+
+	f, err := os.Create(containerdCfg)
+	gm.Expect(err).NotTo(gm.HaveOccurred())
+	defer f.Close()
+	gm.Expect(toml.NewEncoder(f).Encode(cfg)).To(gm.Succeed())
+}
+
 func (r *Runner) buildFLBinary() {
 	flBin, err := gexec.Build(flintlockCmdDir)
 	gm.Expect(err).NotTo(gm.HaveOccurred())
@@ -105,22 +188,35 @@ func (r *Runner) dialGRPCServer() {
 }
 
 func getParentInterface() (string, error) {
-	// If there is a go package which lets me do this without shelling out lmk.
-	// I could not find one after a quick search.
 	cmd := exec.Command("ip", "route", "show")
-	output, err := cmd.Output()
-	if err != nil {
-		return "", err
+	iface := grep(cmd, "default", 4) //nolint:gomnd
+	if len(iface) == 0 {
+		return "", errors.New("parent interface not found") //nolint:goerr113
 	}
+
+	return iface[0], nil
+}
+
+func grep(cmd *exec.Cmd, match string, loc int) []string {
+	output, err := cmd.Output()
+	gm.Expect(err).NotTo(gm.HaveOccurred())
 	scanner := bufio.NewScanner(strings.NewReader(string(output)))
+	out := []string{}
 	for scanner.Scan() {
 		line := scanner.Text()
-		if strings.Contains(line, "default") {
+		if strings.Contains(line, match) {
 			parts := strings.Split(line, " ")
 
-			return parts[4], nil
+			out = append(out, parts[loc])
 		}
 	}
 
-	return "", errors.New("parent interface not found") //nolint:goerr113
+	return out
+}
+
+func baseDir() string {
+	wd, err := os.Getwd()
+	gm.Expect(err).NotTo(gm.HaveOccurred())
+
+	return filepath.Dir(filepath.Dir(wd))
 }
diff --git a/test/tools/metal.py b/test/tools/metal.py
index 2d4a1f401..4c7190a4e 100644
--- a/test/tools/metal.py
+++ b/test/tools/metal.py
@@ -75,7 +75,7 @@ def create_new_key(self, project_id, name):
       return key
 
    def create_user_data(self):
-      files = ["hack/scripts/bootstrap.sh", "hack/scripts/devpool.sh", "test/tools/userdata.sh"]
+      files = ["hack/scripts/bootstrap.sh", "test/tools/userdata.sh"]
       userdata = ""
       for file in files:
          with open(self.base+"/"+file) as f:
diff --git a/test/tools/userdata.sh b/test/tools/userdata.sh
index 6bc6559aa..e7befb3ae 100644
--- a/test/tools/userdata.sh
+++ b/test/tools/userdata.sh
@@ -1,29 +1,5 @@
 #!/bin/bash
 
-CTRD_ROOT="/var/lib/containerd-dev"
-CTRD_STATE="/run/containerd-dev"
-CTRD_CFG="/etc/containerd"
-DM_ROOT="$CTRD_ROOT/snapshotter/devmapper"
-
-mkdir -p "$DM_ROOT" "$CTRD_STATE" "$CTRD_CFG"
-cat > "$CTRD_CFG/config-dev.toml" <<EOF
-version = 2
-root = "$CTRD_ROOT"
-state = "$CTRD_STATE"
-[grpc]
-  address = "$CTRD_STATE/containerd.sock"
-[metrics]
-  address = "127.0.0.1:1338"
-[plugins]
-  [plugins."io.containerd.snapshotter.v1.devmapper"]
-    pool_name = "$POOL"
-    root_path = "$DM_ROOT"
-    base_image_size = "10GB"
-    discard_blocks = true
-[debug]
-  level = "trace"
-EOF
-
 mv /usr/local/go/bin/go /usr/local/bin
 
 mkdir -p /root/work && cd /root/work