groups: implementation of named groups

load/dependencyresolver.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"sort"
+	"sync"
 	"text/template"
 
 	"github.com/asteris-llc/converge/graph"
@@ -36,8 +38,10 @@ func ResolveDependencies(ctx context.Context, g *graph.Graph) (*graph.Graph, err
 	logger := logging.GetLogger(ctx).WithField("function", "ResolveDependencies")
 	logger.Debug("resolving dependencies")
 
-	return g.Transform(ctx, func(meta *node.Node, out *graph.Graph) error {
-		if meta.ID == "root" { // skip root
+	groupLock := new(sync.RWMutex)
+	groupMap := make(map[string][]string)
+	g, err := g.Transform(ctx, func(meta *node.Node, out *graph.Graph) error {
+		if graph.IsRoot(meta.ID) { // skip root
 			return nil
 		}
 
@@ -60,8 +64,60 @@ func ResolveDependencies(ctx context.Context, g *graph.Graph) (*graph.Graph, err
 			}
 		}
 
+		// collect groups information
+		group, err := groupName(node)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve group from node %s", meta.ID)
+		}
+		if group != "" {
+			groupLock.Lock()
+			groupMap[group] = append(groupMap[group], meta.ID)
+			groupLock.Unlock()
+		}
+
 		return nil
 	})
+
+	// create dependencies between nodes in each group
+	for _, ids := range groupMap {
+		// sort ids so that intra-group dependencies are prioritized
+		sort.Strings(ids)
+		for i, id := range ids {
+			if i > 0 {
+				from := id
+				to := ids[i-1]
+
+				groupDep := func(id string) string {
+					pid := graph.ParentID(id)
+					if !graph.IsRoot(pid) {
+						id = pid
+					}
+					return id
+				}
+
+				if !graph.AreSiblingIDs(from, to) {
+					from = groupDep(from)
+					to = groupDep(to)
+				}
+
+				g.Connect(from, to)
+			}
+		}
+	}
+
+	return g, err
+}
+
+func groupName(node *parse.Node) (string, error) {
+	group, err := node.GetString("group")
+	switch err {
+	case parse.ErrNotFound:
+		return "", nil
+	case nil:
+		return group, nil
+	default:
+		return "", err
+	}
 }
 
 func getDepends(g *graph.Graph, id string, node *parse.Node) ([]string, error) {

load/dependencyresolver_test.go

@@ -96,3 +96,46 @@ func TestDependencyResolverResolvesParam(t *testing.T) {
 		"root/param.message",
 	)
 }
+
+// TestDependencyResolverResolvesGroupDependencies tests whether group
+// dependencies are wired correctly
+func TestDependencyResolverResolvesGroupDependencies(t *testing.T) {
+	t.Parallel()
+	defer logging.HideLogs(t)()
+
+	t.Run("intra-module", func(t *testing.T) {
+		nodes, err := load.Nodes(context.Background(), "../samples/groups.hcl", false)
+		require.NoError(t, err)
+
+		resolved, err := load.ResolveDependencies(context.Background(), nodes)
+		assert.NoError(t, err)
+
+		assert.Empty(t, graph.Targets(resolved.DownEdges("root/task.install-build-essential")))
+		assert.Equal(
+			t,
+			[]string{"root/task.install-build-essential"},
+			graph.Targets(resolved.DownEdges("root/task.install-jq")),
+		)
+		assert.Equal(
+			t,
+			[]string{"root/task.install-jq"},
+			graph.Targets(resolved.DownEdges("root/task.install-tree")),
+		)
+	})
+
+	t.Run("inter-module", func(t *testing.T) {
+		nodes, err := load.Nodes(context.Background(), "../samples/groupedIncludeModule.hcl", false)
+		require.NoError(t, err)
+
+		resolved, err := load.ResolveDependencies(context.Background(), nodes)
+		assert.NoError(t, err)
+
+		// first module is not dependent on other modules
+		assert.NotContains(t, resolved.Dependencies("root/module.test1"), "root/module.test2")
+		assert.NotContains(t, resolved.Dependencies("root/module.test1"), "root/module.test3")
+
+		// other modules should depend on each other
+		assert.Contains(t, resolved.Dependencies("root/module.test2"), "root/module.test1")
+		assert.Contains(t, resolved.Dependencies("root/module.test3"), "root/module.test2")
+	})
+}

resource/preparer.go

@@ -110,6 +110,7 @@ func (p *Preparer) validateExtra(typ reflect.Type) error {
 
 	// add special fields
 	fieldNames["depends"] = struct{}{}
+	fieldNames["group"] = struct{}{}
 
 	var err error
 	for key := range p.Source {

samples/groupedIncludeModule.hcl

@@ -0,0 +1,39 @@
+/* demonstrates using modules containing named groups */
+param "jsonfile" {
+  default = "test.json"
+}
+
+file.content "jsonfile" {
+  destination = "{{param `jsonfile`}}"
+  content     = "{}"
+}
+
+module "groupedModule.hcl" "test1" {
+  params = {
+    jsonfile = "{{param `jsonfile`}}"
+    name     = "test1"
+    value    = "test1-val"
+  }
+
+  depends = ["file.content.jsonfile"]
+}
+
+module "groupedModule.hcl" "test2" {
+  params = {
+    jsonfile = "{{param `jsonfile`}}"
+    name     = "test2"
+    value    = "test2-val"
+  }
+
+  depends = ["file.content.jsonfile"]
+}
+
+module "groupedModule.hcl" "test3" {
+  params = {
+    jsonfile = "{{param `jsonfile`}}"
+    name     = "test3"
+    value    = "test3-val"
+  }
+
+  depends = ["file.content.jsonfile"]
+}

samples/groupedModule.hcl

@@ -0,0 +1,21 @@
+/* this module allows you to set a field on the root object of a json file.
+   Because this relies on updating a file in place using a temporary file, it
+   uses a group to ensure it is not run multiple times in parallel. */
+
+param "jsonfile" {
+  default = "test.json"
+}
+
+param "name" {
+  default = "field"
+}
+
+param "value" {
+  default = "value"
+}
+
+task "update" {
+  check = "cat {{param `jsonfile`}} | jq -r -e '.{{param `name`}}'"
+  apply = "cat {{param `jsonfile`}} | jq '. + {\"{{param `name`}}\": \"{{param `value`}}\"}' > /tmp/{{param `jsonfile`}} && mv /tmp/{{param `jsonfile`}} {{param `jsonfile`}}"
+  group = "groupedModule"
+}

samples/groups.hcl

@@ -0,0 +1,17 @@
+task "install-tree" {
+  check = "dpkg -s tree >/dev/null 2>&1"
+  apply = "apt-get install -y tree"
+  group = "apt"
+}
+
+task "install-jq" {
+  check = "dpkg -s jq >/dev/null 2>&1"
+  apply = "apt-get install -y jq"
+  group = "apt"
+}
+
+task "install-build-essential" {
+  check = "dpkg -s build-essential >/dev/null 2>&1"
+  apply = "apt-get install -y build-essential"
+  group = "apt"
+}