e2e: refactor CLI utils out of rescheduling test (#8905)

The CLI helpers in the rescheduling test were intended for shared use, but
until some other tests were written we didn't want to waste time making them
generic. This changeset refactors them and adds some new helpers associated
with the node drain tests (under separate PR).

e2e/e2eutil/allocs.go

@@ -0,0 +1,146 @@
+package e2eutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/testutil"
+)
+
+// WaitForAllocStatusExpected polls 'nomad job status' and exactly compares
+// the status of all allocations (including any previous versions) against the
+// expected list.
+func WaitForAllocStatusExpected(jobID string, expected []string) error {
+	return WaitForAllocStatusComparison(
+		func() ([]string, error) { return AllocStatuses(jobID) },
+		func(got []string) bool { return reflect.DeepEqual(got, expected) },
+		nil,
+	)
+}
+
+// WaitForAllocStatusComparison is a convenience wrapper that polls the query
+// function until the comparison function returns true.
+func WaitForAllocStatusComparison(query func() ([]string, error), comparison func([]string) bool, wc *WaitConfig) error {
+	var got []string
+	var err error
+	interval, retries := wc.OrDefault()
+	testutil.WaitForResultRetries(retries, func() (bool, error) {
+		time.Sleep(interval)
+		got, err = query()
+		if err != nil {
+			return false, err
+		}
+		return comparison(got), nil
+	}, func(e error) {
+		err = fmt.Errorf("alloc status check failed: got %#v", got)
+	})
+	return err
+}
+
+// AllocsForJob returns a slice of key->value maps, each describing the values
+// of the 'nomad job status' Allocations section (not actual
+// structs.Allocation objects, query the API if you want those)
+func AllocsForJob(jobID string) ([]map[string]string, error) {
+
+	out, err := Command("nomad", "job", "status", "-verbose", "-all-allocs", jobID)
+	if err != nil {
+		return nil, fmt.Errorf("'nomad job status' failed: %w", err)
+	}
+
+	section, err := GetSection(out, "Allocations")
+	if err != nil {
+		return nil, fmt.Errorf("could not find Allocations section: %w", err)
+	}
+
+	allocs, err := ParseColumns(section)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse Allocations section: %w", err)
+	}
+	return allocs, nil
+}
+
+// AllocsForNode returns a slice of key->value maps, each describing the values
+// of the 'nomad node status' Allocations section (not actual
+// structs.Allocation objects, query the API if you want those)
+func AllocsForNode(nodeID string) ([]map[string]string, error) {
+
+	out, err := Command("nomad", "node", "status", "-verbose", nodeID)
+	if err != nil {
+		return nil, fmt.Errorf("'nomad node status' failed: %w", err)
+	}
+
+	section, err := GetSection(out, "Allocations")
+	if err != nil {
+		return nil, fmt.Errorf("could not find Allocations section: %w", err)
+	}
+
+	allocs, err := ParseColumns(section)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse Allocations section: %w", err)
+	}
+	return allocs, nil
+}
+
+// AllocStatuses returns a slice of client statuses
+func AllocStatuses(jobID string) ([]string, error) {
+
+	allocs, err := AllocsForJob(jobID)
+	if err != nil {
+		return nil, err
+	}
+
+	statuses := []string{}
+	for _, alloc := range allocs {
+		statuses = append(statuses, alloc["Status"])
+	}
+	return statuses, nil
+}
+
+// AllocStatusesRescheduled is a helper function that pulls
+// out client statuses only from rescheduled allocs.
+func AllocStatusesRescheduled(jobID string) ([]string, error) {
+
+	out, err := Command("nomad", "job", "status", "-verbose", jobID)
+	if err != nil {
+		return nil, fmt.Errorf("nomad job status failed: %w", err)
+	}
+
+	section, err := GetSection(out, "Allocations")
+	if err != nil {
+		return nil, fmt.Errorf("could not find Allocations section: %w", err)
+	}
+
+	allocs, err := ParseColumns(section)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse Allocations section: %w", err)
+	}
+
+	statuses := []string{}
+	for _, alloc := range allocs {
+
+		allocID := alloc["ID"]
+
+		// reschedule tracker isn't exposed in the normal CLI output
+		out, err := Command("nomad", "alloc", "status", "-json", allocID)
+		if err != nil {
+			return nil, fmt.Errorf("nomad alloc status failed: %w", err)
+		}
+
+		dec := json.NewDecoder(strings.NewReader(out))
+		alloc := &api.Allocation{}
+		err = dec.Decode(alloc)
+		if err != nil {
+			return nil, fmt.Errorf("could not decode alloc status JSON: %w", err)
+		}
+
+		if (alloc.RescheduleTracker != nil &&
+			len(alloc.RescheduleTracker.Events) > 0) || alloc.FollowupEvalID != "" {
+			statuses = append(statuses, alloc.ClientStatus)
+		}
+	}
+	return statuses, nil
+}

e2e/e2eutil/deployments.go

@@ -0,0 +1,38 @@
+package e2eutil
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/hashicorp/nomad/testutil"
+)
+
+func WaitForLastDeploymentStatus(jobID, status string, wc *WaitConfig) error {
+	var got string
+	var err error
+	interval, retries := wc.OrDefault()
+	testutil.WaitForResultRetries(retries, func() (bool, error) {
+		time.Sleep(interval)
+
+		out, err := Command("nomad", "job", "status", jobID)
+		if err != nil {
+			return false, fmt.Errorf("could not get job status: %v\n%v", err, out)
+		}
+
+		section, err := GetSection(out, "Latest Deployment")
+		if err != nil {
+			return false, fmt.Errorf("could not find Latest Deployment section: %w", err)
+		}
+
+		fields, err := ParseFields(section)
+		if err != nil {
+			return false, fmt.Errorf("could not parse Latest Deployment section: %w", err)
+		}
+
+		got = fields["Status"]
+		return got == status, nil
+	}, func(e error) {
+		err = fmt.Errorf("deployment status check failed: got %#v", got)
+	})
+	return err
+}

e2e/e2eutil/job.go

@@ -0,0 +1,41 @@
+package e2eutil
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"regexp"
+)
+
+// Register registers a jobspec from a file but with a unique ID.
+// The caller is responsible for recording that ID for later cleanup.
+func Register(jobID, jobFilePath string) error {
+
+	cmd := exec.Command("nomad", "job", "run", "-")
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("could not open stdin?: %w", err)
+	}
+
+	content, err := ioutil.ReadFile(jobFilePath)
+	if err != nil {
+		return fmt.Errorf("could not open job file: %w", err)
+	}
+
+	// hack off the first line to replace with our unique ID
+	var re = regexp.MustCompile(`^job "\w+" \{`)
+	jobspec := re.ReplaceAllString(string(content),
+		fmt.Sprintf("job \"%s\" {", jobID))
+
+	go func() {
+		defer stdin.Close()
+		io.WriteString(stdin, jobspec)
+	}()
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("could not register job: %w\n%v", err, string(out))
+	}
+	return nil
+}

e2e/e2eutil/node.go

@@ -144,3 +144,43 @@ func listClientNodesByOS(client *api.Client, osName string) ([]string, error) {
 	}
 	return nodeIDs, nil
 }
+
+func NodeStatusList() ([]map[string]string, error) {
+
+	out, err := Command("nomad", "node", "status", "-verbose")
+	if err != nil {
+		return nil, fmt.Errorf("'nomad node status' failed: %w", err)
+	}
+
+	nodes, err := ParseColumns(out)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse node status output: %w", err)
+	}
+	return nodes, nil
+}
+
+func NodeStatusListFiltered(filterFn func(string) bool) ([]map[string]string, error) {
+
+	out, err := Command("nomad", "node", "status", "-verbose")
+	if err != nil {
+		return nil, fmt.Errorf("'nomad node status' failed: %w", err)
+	}
+
+	allNodes, err := ParseColumns(out)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse node status output: %w", err)
+	}
+	nodes := []map[string]string{}
+
+	for _, node := range allNodes {
+		out, err := Command("nomad", "node", "status", "-verbose", node["ID"])
+		if err != nil {
+			return nil, fmt.Errorf("could not node status output: %w", err)
+		}
+		if filterFn(out) {
+			nodes = append(nodes, node)
+		}
+	}
+
+	return nodes, nil
+}

e2e/e2eutil/wait.go

@@ -0,0 +1,25 @@
+package e2eutil
+
+import "time"
+
+// WaitConfig is an interval and wait time that can be passed to a waiter
+// function, but with a default value that comes from the OrDefault method
+// if the config is nil
+type WaitConfig struct {
+	Interval time.Duration
+	Retries  int64
+}
+
+// Return a default wait config of 10s
+func (wc *WaitConfig) OrDefault() (time.Duration, int64) {
+	if wc == nil {
+		return time.Millisecond * 100, 100
+	}
+	if wc.Interval == 0 {
+		wc.Interval = time.Millisecond * 100
+	}
+	if wc.Retries == 0 {
+		wc.Retries = 100
+	}
+	return wc.Interval, wc.Retries
+}