Refactor CLI tests

cmd/amp/cli/cli_test.go

@@ -1,134 +1,116 @@
 package cli_test
 
 import (
+	"fmt"
 	"github.com/appcelerator/amp/api/server"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"path"
 	"regexp"
 	"strings"
 	"testing"
-	"time"
 )
 
 type TestSpec struct {
-	fileName string
-	contents []byte
-	valid    bool
+	Name     string
+	Commands []CommandSpec
 }
 
 type CommandSpec struct {
-	Cmd         string   `yaml:"cmd"`
-	Args        []string `yaml:"args"`
-	Options     []string `yaml:"options"`
-	Expectation string   `yaml:"expectation"`
+	Cmd               string   `yaml:"cmd"`
+	Args              []string `yaml:"args"`
+	Options           []string `yaml:"options"`
+	Expectation       string   `yaml:"expectation"`
+	ExpectErrorStatus bool     `yaml:"expectErrorStatus"`
 }
 
 var (
 	testDir = "./test_samples"
 )
 
-func TestMain(t *testing.T) {
-	_, conn := server.StartTestServer()
-	t.Log(conn)
+func TestMain(m *testing.M) {
+	server.StartTestServer()
+	os.Exit(m.Run())
 }
 
 func TestCmds(t *testing.T) {
-	tests := loadFiles(t)
+	tests, err := loadTestSpecs()
+	if err != nil {
+		t.Errorf("unable to load test specs, reason: %v", err)
+		return
+	}
+
 	for _, test := range tests {
 		t.Log("-----------------------------------------------------------------------------------------")
-		t.Logf("test %s\n", test.fileName)
-		parseCmd(t, test)
+		t.Logf("Running spec: %s", test.Name)
+		if err := runTestSpec(t, test); err != nil {
+			t.Error(err)
+			return
+		}
 	}
 }
 
-func loadFiles(t *testing.T) []*TestSpec {
-	tests := []*TestSpec{}
+func loadTestSpecs() ([]*TestSpec, error) {
 	files, err := ioutil.ReadDir(testDir)
 	if err != nil {
-		t.Error(err)
-		return nil
+		return nil, err
 	}
-	for _, f := range files {
-		name := f.Name()
-		t.Log("Loading file:", name)
-		valid := false
-		if !strings.HasPrefix(name, "00-") {
-			valid = true
-		}
-		contents, err := ioutil.ReadFile(path.Join(testDir, name))
+
+	tests := []*TestSpec{}
+	for _, file := range files {
+		test, err := loadTestSpec(path.Join(testDir, file.Name()))
 		if err != nil {
-			t.Errorf("unable to load test sample: %s. Error: %v", name, err)
-		}
-		testSpec := &TestSpec{
-			fileName: name,
-			contents: contents,
-			valid:    valid,
+			return nil, err
 		}
-		tests = append(tests, testSpec)
+		tests = append(tests, test)
 	}
-	return tests
+	return tests, nil
 }
 
-func parseCmd(t *testing.T, test *TestSpec) {
-	commandMap, err := generateCmdSpec(test.contents)
+func loadTestSpec(fileName string) (*TestSpec, error) {
+	content, err := ioutil.ReadFile(fileName)
 	if err != nil {
-		t.Error(err)
-		return
+		return nil, fmt.Errorf("unable to load test spec: %s. Error: %v", fileName, err)
 	}
-	for _, cmdSpec := range commandMap {
-		cmdString := generateCmdString(cmdSpec)
-		t.Log(cmdString, "Command passed.")
-		for i := 0; i < 10; i++ {
-			t.Log(cmdString, "Running...")
-			t.Log(cmdString, "Iteration:", i+1)
-			result, err := runCmd(cmdString)
-			validID := regexp.MustCompile(cmdSpec.Expectation)
-			if test.valid == false {
-				if err == nil {
-					t.Log(cmdString, "Error:", err)
-					t.Log(cmdString, "Invalid Sample Command has failed, retrying.")
-					time.Sleep(1 * time.Second)
-				} else {
-					if !validID.MatchString(string(result)) {
-						t.Log(cmdString, "Error: miss matched expectation")
-						t.Fail()
-						break
-					}
-					t.Log(cmdString, "Invalid Sample Command result:\n", string(result))
-					break
-				}
-			} else {
-				if err != nil {
-					t.Log(cmdString, "Error:", err)
-					t.Log(cmdString, "Command failed, retrying.")
-					time.Sleep(1 * time.Second)
-				} else {
-					if !validID.MatchString(string(result)) {
-						t.Log(cmdString, "Error: miss matched expectation")
-						t.Fail()
-						break
-					}
-					t.Log(cmdString, "Command result:\n", string(result))
-					break
-				}
-			}
-			if i >= 9 {
-				t.Log(cmdString, "Error:", err)
-				t.Log(cmdString, "Command has failed, exiting.")
-				t.Fail()
-			}
-		}
+	testSpec := &TestSpec{
+		Name: fileName,
 	}
+
+	commandMap := map[string]CommandSpec{}
+	if err := yaml.Unmarshal(content, &commandMap); err != nil {
+		return nil, fmt.Errorf("unable to parse test spec: %s. Error: %v", fileName, err)
+	}
+
+	// Keep values only
+	for _, command := range commandMap {
+		testSpec.Commands = append(testSpec.Commands, command)
+	}
+
+	return testSpec, nil
 }
 
-func generateCmdSpec(b []byte) (out map[string]CommandSpec, err error) {
-	err = yaml.Unmarshal(b, &out)
-	return
+func runTestSpec(t *testing.T, test *TestSpec) error {
+	for _, cmdSpec := range test.Commands {
+		cmdString := generateCmdString(&cmdSpec)
+		t.Logf("Running: %s", strings.Join(cmdString, " "))
+		actualOutput, err := exec.Command(cmdString[0], cmdString[1:]...).CombinedOutput()
+		expectedOutput := regexp.MustCompile(cmdSpec.Expectation)
+		if !expectedOutput.MatchString(string(actualOutput)) {
+			return fmt.Errorf("miss matched expected output: %s", actualOutput)
+		}
+		if err != nil && !cmdSpec.ExpectErrorStatus {
+			return fmt.Errorf("Command was expected to exit with zero status but got: %v", err)
+		}
+		if err == nil && cmdSpec.ExpectErrorStatus {
+			return fmt.Errorf("Command was expected to exit with error status but existed with zero")
+		}
+	}
+	return nil
 }
 
-func generateCmdString(cmdSpec CommandSpec) (cmdString []string) {
+func generateCmdString(cmdSpec *CommandSpec) (cmdString []string) {
 	cmdSplit := strings.Fields(cmdSpec.Cmd)
 	optionsSplit := []string{}
 	for _, val := range cmdSpec.Options {
@@ -138,9 +120,3 @@ func generateCmdString(cmdSpec CommandSpec) (cmdString []string) {
 	cmdString = append(cmdString, optionsSplit...)
 	return
 }
-
-func runCmd(cmdString []string) (result []byte, err error) {
-	cmd := exec.Command(cmdString[0], cmdString[1:]...)
-	result, err = cmd.CombinedOutput()
-	return
-}

cmd/amp/cli/test_samples/21-all-logs.yml → cmd/amp/cli/test_samples/logs-all.yml

@@ -1,4 +1,4 @@
-all-logs:
+logs-all:
   cmd: amp logs
   args:
   options:

cmd/amp/cli/test_samples/20-metadata-logs.yml → cmd/amp/cli/test_samples/logs-metadata.yml

@@ -1,4 +1,4 @@
-metadata-logs:
+logs-metadata:
   cmd: amp logs
   args:
     -

cmd/amp/cli/test_samples/19-numbered-logs.yml → cmd/amp/cli/test_samples/logs-numbered.yml

@@ -1,4 +1,4 @@
-numbered-logs:
+logs-numbered:
   cmd: amp logs
   args:
     -

cmd/amp/cli/test_samples/18-stack-logs.yml → cmd/amp/cli/test_samples/logs-stack.yml

@@ -1,4 +1,4 @@
-stack-logs:
+logs-stack:
   cmd: amp logs
   args:
     -

cmd/amp/cli/test_samples/01-create-service.yml → cmd/amp/cli/test_samples/service-create.yml

@@ -1,4 +1,4 @@
-create-service:
+service-create:
   cmd: amp service create
   args:
     - appcelerator/pinger

cmd/amp/cli/test_samples/03-curl-service.yml → cmd/amp/cli/test_samples/service-curl.yml

@@ -1,4 +1,4 @@
-curl-service:
+service-curl:
   cmd: curl
   args:
     - localhost:90/ping

cmd/amp/cli/test_samples/02-list-service.yml → cmd/amp/cli/test_samples/service-list.yml

@@ -1,4 +1,4 @@
-list-service:
+service-list:
   cmd: docker service ls
   args:
   options:

cmd/amp/cli/test_samples/04-remove-service.yml → cmd/amp/cli/test_samples/service-remove.yml

@@ -1,10 +1,10 @@
-remove-service:
+service-remove:
   cmd: amp service rm
   args:
     - pinger
   options:
   expectation: (pinger)
-list-service:
+service-list:
   cmd: docker service ls
   args:
   options:

cmd/amp/cli/test_samples/05-create-stack.yml → cmd/amp/cli/test_samples/stack-create.yml

@@ -1,4 +1,4 @@
-create-stack:
+stack-create:
   cmd: amp stack up
   args:
   options:

cmd/amp/cli/test_samples/06-list-stack.yml → cmd/amp/cli/test_samples/stack-list.yml

@@ -1,4 +1,4 @@
-list-stack:
+stack-list:
   cmd: amp stack ls
   args:
   options:

cmd/amp/cli/test_samples/09-remove-stack.yml → cmd/amp/cli/test_samples/stack-remove.yml

@@ -1,16 +1,16 @@
-stop-stack:
+stack-stop:
   cmd: amp stack stop
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
-remove-stack:
+stack-remove:
   cmd: amp stack rm
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
-list-stack:
+stack-list:
   cmd: amp stack ls
   args:
   options:

cmd/amp/cli/test_samples/08-restart-stack.yml → cmd/amp/cli/test_samples/stack-restart.yml

@@ -1,10 +1,10 @@
-restart-stack:
+stack-restart:
   cmd: amp stack start
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
-list-stack:
+stack-list:
   cmd: amp stack ls
   args:
   options:

cmd/amp/cli/test_samples/07-stop-stack.yml → cmd/amp/cli/test_samples/stack-stop.yml

@@ -1,10 +1,10 @@
-stop-stack:
+stack-stop:
   cmd: amp stack stop
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
-list-stack:
+stack-list:
   cmd: amp stack ls
   args:
   options:

cmd/amp/cli/test_samples/14-container-stats.yml → cmd/amp/cli/test_samples/stats-container.yml

@@ -1,4 +1,4 @@
-container-stats:
+stats-container:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/10-cpu-stats.yml → cmd/amp/cli/test_samples/stats-cpu.yml

@@ -1,4 +1,4 @@
-cpu-stats:
+stats-cpu:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/12-io-stats.yml → cmd/amp/cli/test_samples/stats-io.yml

@@ -1,4 +1,4 @@
-io-stats:
+stats-io:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/11-mem-stats.yml → cmd/amp/cli/test_samples/stats-mem.yml

@@ -1,4 +1,4 @@
-mem-stats:
+stats-mem:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/15-net-stats.yml → cmd/amp/cli/test_samples/stats-net.yml

@@ -1,4 +1,4 @@
-net-stats:
+stats-net:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/16-node-stats.yml → cmd/amp/cli/test_samples/stats-node.yml

@@ -1,4 +1,4 @@
-node-stats:
+stats-node:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/17-service-stats.yml → cmd/amp/cli/test_samples/stats-service.yml

@@ -1,4 +1,4 @@
-service-stats:
+stats-service:
   cmd: amp stats
   args:
     -

cmd/amp/cli/test_samples/13-task-stats.yml → cmd/amp/cli/test_samples/stats-task.yml

@@ -1,4 +1,4 @@
-task-stats:
+stats-task:
   cmd: amp stats
   args:
     -