Playbook update (for linuxkit demo); misc fixes

cmd/infrakit/plugin/plugin.go

@@ -122,15 +122,15 @@ func Command(plugins func() discovery.Plugins) *cobra.Command {
 		}
 
 		if !*quiet {
-			fmt.Printf("%-20s\t%-50s\t%-s\n", "NAME", "LISTEN", "INTERFACE")
+			fmt.Printf("%-30s\t%-50s\t%-s\n", "NAME", "LISTEN", "INTERFACE")
 		}
 
 		sort.Strings(keys)
 
 		for _, k := range keys {
 
 			ep := view[k]
-			fmt.Printf("%-20s\t%-50s\t%-s\n", ep.name, ep.listen, ep.spi)
+			fmt.Printf("%-30s\t%-50s\t%-s\n", ep.name, ep.listen, ep.spi)
 
 		}
 

docs/playbooks/linuxkit/examples/sshd/build-image.sh

@@ -3,7 +3,7 @@
 {{ $defaultKeyDir := cat (env `HOME`) `/.ssh` | nospace }}
 
 {{ $keyDir := flag "ssh-dir" "string" "Directory of ssh keys" | prompt "SSH key dir?" "string" $defaultKeyDir }}
-{{ $keyFile := flag "key-name" "string" "Key name" | prompt "Which public key to include?" "string" "id_rsa" }}
+{{ $keyFile := flag "key-name" "string" "Key name" | prompt "Which public key to include?" "string" "id_rsa.pub" }}
 
 {{ $output := `sshd.yml` }}
 
@@ -23,3 +23,6 @@ EOF
 echo "Generated config file. Running moby."
 
 moby build {{$output}}
+
+{{ $bundle := `sshd` }}
+echo "Checksum:"{{ fetch (cat `file://` (env `PWD`) $bundle "-initrd.img" | nospace) | sha256sum }}

docs/playbooks/linuxkit/examples/sshd/destroy-instances.sh

@@ -0,0 +1,5 @@
+{{/* =% sh %= */}}
+
+{{ $plugin := flag "instance-plugin" "string" "Name of plugin" | prompt "What instance plugin to target?" "string" "instance-hyperkit" }}
+
+infrakit instance --name {{ $plugin }} describe -q | awk '{print $1}' | xargs infrakit instance --name {{ $plugin }} destroy

docs/playbooks/linuxkit/examples/sshd/index.yml

@@ -7,3 +7,12 @@ run-instance : provision-instance.ikt
 
 # Manage a group of nodes
 scale-group : scale-group.ikt
+
+# List instances
+list-instances : list-instances.sh
+
+# Inspect global state
+inspect : inspect.sh
+
+# Destroy all
+destroy-all : destroy-instances.sh

docs/playbooks/linuxkit/examples/sshd/inspect.sh

@@ -0,0 +1,3 @@
+{{/* =% sh %= */}}
+
+infrakit manager inspect -y

docs/playbooks/linuxkit/examples/sshd/list-instances.sh

@@ -0,0 +1,3 @@
+{{/* =% sh %= */}}
+
+infrakit instance --name instance-hyperkit describe

docs/playbooks/linuxkit/examples/sshd/provision-instance.ikt

@@ -10,11 +10,13 @@
 {{ $memory := flag "memory" "int" "Memory in MB" | prompt "How many mb for memory?" "int" 1024 }}
 {{ $disk := flag "disk" "int" "Disk in MB" | prompt "How many mb for disk?" "int" 100 }}
 
+{{ $checksum := fetch (cat "file://" $linuxkitPath "/" $bundle "-initrd.img" | nospace) | sha256sum }}
+
 LogicalID: {{ $privateIP }}
 Tags:
   infrakit.created: {{ now | htmlDate }}
   infrakit.project: {{ $project }}
-  infrakit.checksum: {{ include (cat $linuxkitPath "/" $bundle "-initrd.img" | nospace) | sha256sum }}
+  infrakit.checksum: {{ $checksum }}
 
 Properties:
   kernel: {{ cat $linuxkitPath "/" $bundle "-kernel" | nospace }}
@@ -23,3 +25,4 @@ Properties:
   memory: {{ $memory }}
   disk_size: {{ $disk }}
   cmdline: {{ include (cat "file://" $linuxkitPath "/" $bundle "-cmdline" | nospace) }}
+  checksum: {{ $checksum }}

docs/playbooks/linuxkit/examples/sshd/scale-group.ikt

@@ -1,9 +1,6 @@
 {{/* Input to create a group using the hyperkit instance plugin and manager */}}
-<<<<<<< HEAD
 {{/* =% managerCommit true false %= */}}
-=======
-{{/* =% managerCommit true false  %= */}}
->>>>>>> a8e9cb894b8d878d79ab036cda9bb1af9a8cf86f
+
 
 {{ $groupName := flag "group-name" "string" "Name of group" | prompt "Name of the group?" "string" "mygroup" }}
 {{ $size := flag "size" "int" "Size of the group" | prompt "How many nodes?" "int" 2 }}
@@ -16,6 +13,8 @@
 {{ $memory := flag "memory" "int" "Memory in MB" | prompt "How many mb for memory?" "int" 1024 }}
 {{ $disk := flag "disk" "int" "Disk in MB" | prompt "How many mb for disk?" "int" 100 }}
 
+{{ $checksum := fetch (cat "file://" $linuxkitPath "/" $bundle "-initrd.img" | nospace) | sha256sum }}
+
 - Plugin: group
   Properties:
     ID: {{ $groupName }}
@@ -28,7 +27,7 @@
           Tags:
             infrakit.created: {{ now | htmlDate }}
             infrakit.project: {{ $project }}
-            infrakit.checksum: {{ include (cat $linuxkitPath "/" $bundle "-initrd.img" | nospace) | sha256sum }}
+            infrakit.checksum: {{ $checksum }}
 
       Instance:
         Plugin: instance-hyperkit
@@ -39,3 +38,4 @@
           memory: {{ $memory }}
           disk_size: {{ $disk }}
           cmdline: {{ include (cat "file://" $linuxkitPath "/" $bundle "-cmdline" | nospace) }}
+          checksum: {{ $checksum }}

docs/playbooks/linuxkit/examples/sshd/sshd.yml

@@ -50,6 +50,7 @@ files:
 
 outputs:
   - format: kernel+initrd
-  - format: iso-bios
-  - format: iso-efi
-  - format: gcp-img
+
+#  - format: iso-bios
+#  - format: iso-efi
+#  - format: gcp-img

docs/playbooks/linuxkit/index.yml

@@ -7,10 +7,10 @@ install-moby: install-moby.sh
 install-hyperkit : install-hyperkit.sh
 
 # Starts the infrakit daemons using a common data volume for sockets and files
-start-infrakit : start-infrakit.sh
+start : start-infrakit.sh
 
 # Stops all the infrakit daemons
-stop-infrakit  : stop-infrakit.sh
+stop  : stop-infrakit.sh
 
 # Examples
 examples : examples/index.yml

docs/playbooks/linuxkit/start-infrakit.sh

@@ -27,7 +27,7 @@ docker run  -d --name infrakit \
 docker run  -d --volumes-from infrakit --name time {{ $image }} infrakit-event-time
 docker run  -d --volumes-from infrakit --name vanilla {{ $image }} infrakit-flavor-vanilla
 docker run  -d --volumes-from infrakit --name group-stateless {{ $image }} infrakit-group-default \
-       --name group-stateless --poll-interval 30s
+       --name group-stateless --poll-interval 10s
 
 # The leader file -- only required for local store
 docker run --rm --volumes-from infrakit {{ $image }} /bin/sh -c "echo group > /infrakit/leader"
@@ -42,7 +42,24 @@ docker run  -d --volumes-from infrakit --name manager \
 {{ source "start-instance-gcp.sh" }}
 
 echo "Updating hosts file"
-{{ $hostsFile := list (env `INFRAKIT_HOME`) `/hosts` | join `` }} 
-{{ $hosts :=  include (list `file://` $hostsFile | join ``) | yamlDecode }} 
+{{ $hostsFile := list (env `INFRAKIT_HOME`) `/hosts` | join `` }}
+{{ $hosts :=  include (list `file://` $hostsFile | join ``) | yamlDecode }}
 {{ $_ := set $hosts `localhost` (list `localhost` $port | join `:`) }}
 echo "{{ $hosts | yamlEncode }}" > {{ $hostsFile }}
+
+echo "Started hyperkit: {{ var `started-hyperkit` }}"
+echo "Started gcp:      {{ var `started-gcp` }}"
+
+tracked=``
+# Start any tracker of resources
+{{ if var `started-hyperkit`}}
+tracked="$tracked --instance instance-hyperkit"
+{{ end }}
+{{ if var `started-gcp`}}
+tracked="$tracked --instance instance-gcp"
+{{ end }}
+
+if [[ "$tracked" != "" ]]; then
+docker run -d --volumes-from infrakit --name tracker \
+       {{ $image }} infrakit util track --name tracker $tracked
+fi

docs/playbooks/linuxkit/start-instance-gcp.sh

@@ -25,4 +25,6 @@ docker run -d --volumes-from infrakit --name instance-gcp \
        --namespace-tags {{cat "infrakit.scope=" $project | nospace}} \
        --zone {{ $zone }} --log 5 --project {{ $project }}
 
+{{ var `started-gcp` true }}
+
 {{ end }}

docs/playbooks/linuxkit/start-instance-hyperkit.sh

@@ -13,4 +13,6 @@ echo "This plugin is listening at localhost:24865"
 
 infrakit-instance-hyperkit --log 5 > {{env `INFRAKIT_HOME`}}/logs/instance-hyperkit.log 2>&1 &
 
+{{ var `started-hyperkit` true }}
+
 {{ end }}

pkg/cli/context.go

@@ -251,6 +251,23 @@ func (c *Context) Funcs() []template.Function {
 				return c.defineFlag(n, ftype, desc, defaultValue)
 			},
 		},
+		{
+			Name: "fetch",
+			Func: func(p string, opt ...interface{}) (string, error) {
+				// Overrides the base 'file' to account for the fact that
+				// some variables in the flag building phase are not set and would
+				// cause errors.  In general, use include for loading files whose
+				// paths are computed from some flags.  Use 'source' for including
+				// sibling templates that also include other flag definitions.
+				if c.exec {
+					content, err := c.template.Fetch(p, opt...)
+					if err == nil {
+						return content, nil
+					}
+				}
+				return "", nil
+			},
+		},
 		{
 			Name: "include",
 			Func: func(p string, opt ...interface{}) (string, error) {

pkg/plugin/instance/hyperkit/instance.go

@@ -44,15 +44,24 @@ type hyperkitPlugin struct {
 	DiskDir string
 }
 
+// Properties is the struct that holds the input
+type Properties struct {
+	hyperkit.HyperKit
+
+	// Checksum is a checksum for the image
+	Checksum string
+}
+
 // Validate performs local validation on a provision request.
 func (p hyperkitPlugin) Validate(req *types.Any) error {
-	// The guest is just the same data structure used by hyperkit for full fidelity config
-	guest := hyperkit.HyperKit{}
-
-	if err := req.Decode(&guest); err != nil {
+	properties := Properties{}
+	if err := req.Decode(&properties); err != nil {
 		return fmt.Errorf("error decoding guest configuration: %s, err=%v", req.String(), err)
 	}
 
+	// The guest is just the same data structure used by hyperkit for full fidelity config
+	guest := properties.HyperKit
+
 	for key, check := range map[string]int{
 		"CPUs":     guest.CPUs,
 		"Memory":   guest.Memory,
@@ -82,16 +91,19 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 		return nil, fmt.Errorf("missing properties in spec")
 	}
 
-	// The guest is just the same data structure used by hyperkit for full fidelity config
-	guest := &hyperkit.HyperKit{
-		HyperKit:   p.HyperKit,
-		VPNKitSock: p.VPNKitSock,
+	properties := Properties{
+		HyperKit: hyperkit.HyperKit{
+			HyperKit:   p.HyperKit,
+			VPNKitSock: p.VPNKitSock,
+		},
 	}
-
-	if err := spec.Properties.Decode(guest); err != nil {
-		return nil, fmt.Errorf("error decoding guest configuration: %s", spec.Properties.String())
+	if err := spec.Properties.Decode(&properties); err != nil {
+		return nil, fmt.Errorf("error decoding guest configuration: err=%v", err)
 	}
 
+	// The guest is just the same data structure used by hyperkit for full fidelity config
+	guest := properties.HyperKit
+
 	// directory for instance state
 	instanceDir, err := ioutil.TempDir(p.VMDir, "infrakit-")
 	if err != nil {

pkg/template/funcs.go

@@ -233,6 +233,28 @@ func setHeaders(req *http.Request, headers map[string][]string) {
 	}
 }
 
+// Fetch opens and reads a url and returns its content
+func (t *Template) Fetch(p string, opt ...interface{}) (string, error) {
+	headers, _ := headersAndContext(opt...)
+	loc := p
+	if strings.Index(loc, "str://") == -1 {
+		u, err := GetURL(t.url, p)
+		if err != nil {
+			return "", err
+		}
+		loc = u.String()
+	}
+	prev := t.options.CustomizeFetch
+	t.options.CustomizeFetch = func(req *http.Request) {
+		setHeaders(req, headers)
+		if prev != nil {
+			prev(req)
+		}
+	}
+	buff, err := Fetch(loc, t.options)
+	return string(buff), err
+}
+
 // Source 'sources' the input file at url, also inherits all the variables.
 func (t *Template) Source(p string, opt ...interface{}) (string, error) {
 	headers, context := headersAndContext(opt...)
@@ -309,6 +331,13 @@ func (t *Template) Include(p string, opt ...interface{}) (string, error) {
 func (t *Template) DefaultFuncs() []Function {
 
 	return []Function{
+		{
+			Name: "fetch",
+			Description: []string{
+				"Fetches a resource without evaluation as template",
+			},
+			Func: t.Fetch,
+		},
 		{
 			Name: "source",
 			Description: []string{