Add initial version of runtimetest command:

README.md

@@ -66,21 +66,22 @@ OPTIONS:
    --path       path to a bundle
 
 ```
-
-Testing OCI runtimes
+    
+Testing OCI runtimes   
 ------------------------------------------
 
 ```
-$ make
-$ sudo make install
-$ sudo ./test_runtime.sh -r runc
------------------------------------------------------------------------------------
-VALIDATING RUNTIME: runc
------------------------------------------------------------------------------------
-validating container process
-validating capabilities
-validating hostname
-validating rlimits
-validating sysctls
-Runtime runc passed validation
-```
+# ocitools runtimetest --help
+NAME:
+   runtimetest - test if a runtime is comlpliant to oci specs
+
+USAGE:
+   command runtimetest [command options] [arguments...]
+
+OPTIONS:
+   --runtime, -r        runtime to be tested
+   --output, -o         output format,
+-o=all: ouput sucessful details and statistics, -o=err-only: ouput failure details and statistics
+   --debug, -d  switch of debug mode, defaults to false, with '--debug' to enable debug mode
+```    
+

cases.conf

@@ -0,0 +1,2 @@
+process= --args=./runtimetest --rootfs=rootfs --read-only=false;--args=./runtimetest --rootfs=rootfs --read-only=true
+hostname= --args=./runtimetest --rootfs=rootfs --hostname=zenlin

config/config.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+)
+
+const configPath = "cases.conf"
+
+var (
+	// BundleMap for config, key is the bundlename, value is the params
+	BundleMap = make(map[string]string)
+	configLen int
+)
+
+func init() {
+	f, err := os.Open(configPath)
+	if err != nil {
+		logrus.Fatalf("open file %v error %v", configPath, err)
+	}
+	defer f.Close()
+
+	rd := bufio.NewReader(f)
+	count := 0
+	for {
+		line, err := rd.ReadString('\n')
+		if err != nil || io.EOF == err {
+			break
+		}
+		prefix := strings.Split(line, "=")
+		caseName := strings.TrimSpace(prefix[0])
+		caseArg := strings.TrimPrefix(line, caseName+"=")
+		for i, arg := range splitArgs(caseArg) {
+			BundleMap[caseName+strconv.FormatInt(int64(i), 10)] = arg
+			count = count + 1
+		}
+	}
+	configLen = count
+}
+
+func splitArgs(args string) []string {
+	argArray := strings.Split(args, ";")
+	resArray := make([]string, len(argArray))
+	for count, arg := range argArray {
+		resArray[count] = strings.TrimSpace(arg)
+	}
+	return resArray
+}

main.go

@@ -16,6 +16,7 @@ func main() {
 	app.Commands = []cli.Command{
 		generateCommand,
 		bundleValidateCommand,
+		runtimeTestCommand,
 	}
 
 	if err := app.Run(os.Args); err != nil {

runtimetest.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+	"github.com/opencontainers/ocitools/units"
+)
+
+const bundleCacheDir = "./bundles"
+
+var runtimetestFlags = []cli.Flag{
+	cli.StringFlag{Name: "runtime, r", Usage: "runtime to be tested"},
+	cli.StringFlag{Name: "output, o", Usage: "output format, \n" +
+		"-o=all: ouput sucessful details and statistics, -o=err-only: ouput failure details and statistics"},
+	cli.BoolFlag{Name: "debug, d", Usage: "switch of debug mode, defaults to false, with '--debug' to enable debug mode"},
+}
+
+var runtimeTestCommand = cli.Command{
+	Name:  "runtimetest",
+	Usage: "test if a runtime is comlpliant to oci specs",
+	Flags: runtimetestFlags,
+	Action: func(context *cli.Context) {
+		if os.Geteuid() != 0 {
+			logrus.Fatalln("runtimetest should be run as root")
+		}
+		var runtime string
+		if runtime = context.String("runtime"); runtime != "runc" {
+			logrus.Fatalf("runtimetest does not support %v\n", runtime)
+		}
+		output := context.String("output")
+		setDebugMode(context.Bool("debug"))
+
+		units.LoadTestUnits("./cases.conf")
+
+		if err := os.MkdirAll(bundleCacheDir, os.ModePerm); err != nil {
+			logrus.Printf("create cache dir for bundle cases err: %v\ns", bundleCacheDir)
+			return
+		}
+
+		for _, tu := range *units.Units {
+			testTask(tu, runtime)
+		}
+
+		units.OutputResult(output)
+
+		if err := os.RemoveAll(bundleCacheDir); err != nil {
+			logrus.Fatalf("remove cache dir of bundles %v err: %v\n", bundleCacheDir, err)
+		}
+
+		if err := os.Remove("./runtime.json"); err != nil {
+			logrus.Fatalf("remove ./runtime.json err: %v\n", err)
+		}
+
+		if err := os.Remove("./config.json"); err != nil {
+			logrus.Fatalf("remove ./config.json err: %v\n", err)
+		}
+	},
+}
+
+func setDebugMode(debug bool) {
+	if !debug {
+		logrus.SetLevel(logrus.InfoLevel)
+	} else {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+}
+
+func testTask(unit *units.TestUnit, runtime string) {
+	logrus.Debugf("test bundle name: %v, Test args: %v\n", unit.Name, unit.Args)
+	if err := unit.SetRuntime(runtime); err != nil {
+		logrus.Fatalf("failed to setup runtime %s , error: %v\n", runtime, err)
+	} else {
+		unit.Run()
+	}
+	return
+}