Add multipart mime support and refactor

This change adds multipart mime userdata support and refactors the code
to allow coreos-cloudinit to run multiple userdata parts as separate
steps.

Signed-off-by: Gabriel Adrian Samfira <[email protected]>

config/config.go

@@ -56,6 +56,15 @@ func IsCloudConfig(userdata string) bool {
 	return (header == "#cloud-config")
 }
 
+func IsMultipartMime(userdata string) bool {
+	header := strings.SplitN(userdata, "\n", 2)[0]
+
+	// Trim trailing whitespaces
+	header = strings.TrimRightFunc(header, unicode.IsSpace)
+
+	return strings.Contains(header, "Content-Type: multipart/mixed;")
+}
+
 // NewCloudConfig instantiates a new CloudConfig from the given contents (a
 // string of YAML), returning any error encountered. It will ignore unknown
 // fields but log encountering them.

config/validate/validate.go

@@ -44,9 +44,11 @@ func Validate(userdataBytes []byte) (Report, error) {
 		return Report{}, nil
 	case config.IsCloudConfig(string(userdataBytes)):
 		return validateCloudConfig(userdataBytes, Rules)
+	case config.IsMultipartMime(string(userdataBytes)):
+		return Report{}, nil
 	default:
 		return Report{entries: []Entry{
-			{kind: entryError, message: `must be "#cloud-config" or begin with "#!"`, line: 1},
+			{kind: entryError, message: `must be "#cloud-config", multipart mime or begin with "#!"`, line: 1},
 		}}, nil
 	}
 }

config/validate/validate_test.go

@@ -28,7 +28,7 @@ func TestParseCloudConfig(t *testing.T) {
 	}{
 		{},
 		{
-			config: "	",
+			config:  "	",
 			entries: []Entry{{entryError, "found character that cannot start any token", 1}},
 		},
 		{
@@ -113,7 +113,7 @@ func TestValidate(t *testing.T) {
 		},
 		{
 			config: "{}",
-			report: Report{entries: []Entry{{entryError, `must be "#cloud-config" or begin with "#!"`, 1}}},
+			report: Report{entries: []Entry{{entryError, `must be "#cloud-config", multipart mime or begin with "#!"`, 1}}},
 		},
 		{
 			config: `{"ignitionVersion":0}`,

coreos-cloudinit.go

@@ -161,7 +161,7 @@ func main() {
 		os.Exit(2)
 	}
 
-	if flags.printVersion == true {
+	if flags.printVersion {
 		fmt.Printf("coreos-cloudinit %s\n", version)
 		os.Exit(0)
 	}
@@ -188,6 +188,22 @@ func main() {
 		os.Exit(1)
 	}
 
+	log.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
+	metadata, err := ds.FetchMetadata()
+	if err != nil {
+		log.Printf("Failed fetching meta-data from datasource: %v\n", err)
+		os.Exit(1)
+	}
+	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.sshKeyName, metadata)
+
+	// Setup networking units
+	if flags.convertNetconf != "" {
+		if err := setupNetworkUnits(metadata.NetworkConfig, env, flags.convertNetconf); err != nil {
+			log.Printf("Failed to setup network units: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
 	log.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
 	userdataBytes, err := ds.FetchUserdata()
 	if err != nil {
@@ -216,66 +232,40 @@ func main() {
 		}
 	}
 
-	log.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
-	metadata, err := ds.FetchMetadata()
+	udata, err := initialize.NewUserData(string(userdataBytes), env)
 	if err != nil {
-		log.Printf("Failed fetching meta-data from datasource: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Apply environment to user-data
-	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.sshKeyName, metadata)
-	userdata := env.Apply(string(userdataBytes))
-
-	var ccu *config.CloudConfig
-	var script *config.Script
-	switch ud, err := initialize.ParseUserData(userdata); err {
-	case initialize.ErrIgnitionConfig:
-		fmt.Printf("Detected an Ignition config. Exiting...")
-		os.Exit(0)
-	case nil:
-		switch t := ud.(type) {
-		case *config.CloudConfig:
-			ccu = t
-		case *config.Script:
-			script = t
-		}
-	default:
-		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
+		log.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
 		failure = true
 	}
 
-	log.Println("Merging cloud-config from meta-data and user-data")
-	cc := mergeConfigs(ccu, metadata)
+	mustStop := false
+	hostname := determineHostname(metadata, udata)
+	if err := initialize.ApplyHostname(hostname); err != nil {
+		log.Printf("Failed to set hostname: %v", err)
+		mustStop = true
+	}
 
-	var ifaces []network.InterfaceGenerator
-	if flags.convertNetconf != "" {
-		var err error
-		switch flags.convertNetconf {
-		case "debian":
-			ifaces, err = network.ProcessDebianNetconf(metadata.NetworkConfig.([]byte))
-		case "packet":
-			ifaces, err = network.ProcessPacketNetconf(metadata.NetworkConfig.(packet.NetworkData))
-		case "vmware":
-			ifaces, err = network.ProcessVMwareNetconf(metadata.NetworkConfig.(map[string]string))
-		default:
-			err = fmt.Errorf("Unsupported network config format %q", flags.convertNetconf)
-		}
-		if err != nil {
-			log.Printf("Failed to generate interfaces: %v\n", err)
-			os.Exit(1)
-		}
+	mergedKeys := mergeSSHKeysFromSources(metadata, udata)
+	if err := initialize.ApplyCoreUserSSHKeys(mergedKeys, env); err != nil {
+		log.Printf("Failed to apply SSH keys: %v", err)
+		mustStop = true
 	}
 
-	if err = initialize.Apply(cc, ifaces, env); err != nil {
-		log.Printf("Failed to apply cloud-config: %v\n", err)
+	if mustStop {
+		// We try to set both the hostname and SSH keys. If either fails, we stop.
+		// We don't stop if hostname fails to be set, because we may still be able to set
+		// the SSH keys and access the server to debug. However, if an error is encountered
+		// in either of the two operations, we exit with a non-zero status.
 		os.Exit(1)
 	}
 
-	if script != nil {
-		if err = runScript(*script, env); err != nil {
-			log.Printf("Failed to run script: %v\n", err)
-			os.Exit(1)
+	if !failure && udata != nil {
+		for _, part := range udata.Parts {
+			log.Printf("Running part %q", part.PartType())
+			if err := part.RunPart(env); err != nil {
+				log.Printf("Failed to run part: %v", err)
+				failure = true
+			}
 		}
 	}
 
@@ -284,25 +274,56 @@ func main() {
 	}
 }
 
-// mergeConfigs merges certain options from md (meta-data from the datasource)
-// onto cc (a CloudConfig derived from user-data), if they are not already set
-// on cc (i.e. user-data always takes precedence)
-func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.CloudConfig) {
-	if cc != nil {
-		out = *cc
-	}
-
-	if md.Hostname != "" {
-		if out.Hostname != "" {
-			log.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
-		} else {
-			out.Hostname = md.Hostname
+// determineHostname returns either the hostname from the metadata, or the hostname from the
+// supplied cloud-config. The cloud-config hostname takes precedence, and we stop after the first
+// cloud-config that gives us a hostname.
+func determineHostname(md datasource.Metadata, udata *initialize.UserData) string {
+	hostname := md.Hostname
+	if udata != nil {
+		udataHostname := udata.FindHostname()
+		if udataHostname != "" {
+			hostname = udataHostname
 		}
 	}
+	return hostname
+}
+
+// mergeSSHKeysFromSources creates a list of all SSH keys from meta-data and the supplied
+// cloud-config sources.
+func mergeSSHKeysFromSources(md datasource.Metadata, udata *initialize.UserData) []string {
+	keys := []string{}
 	for _, key := range md.SSHPublicKeys {
-		out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
+		keys = append(keys, key)
+	}
+
+	if udata != nil {
+		return udata.FindSSHKeys(keys)
+	}
+
+	return keys
+}
+
+func setupNetworkUnits(netConfig interface{}, env *initialize.Environment, netconf string) error {
+	var ifaces []network.InterfaceGenerator
+	var err error
+	switch netconf {
+	case "debian":
+		ifaces, err = network.ProcessDebianNetconf(netConfig.([]byte))
+	case "packet":
+		ifaces, err = network.ProcessPacketNetconf(netConfig.(packet.NetworkData))
+	case "vmware":
+		ifaces, err = network.ProcessVMwareNetconf(netConfig.(map[string]string))
+	default:
+		err = fmt.Errorf("Unsupported network config format %q", netconf)
+	}
+	if err != nil {
+		return fmt.Errorf("error generating interfaces: %w", err)
+	}
+
+	if err := initialize.ApplyNetworkConfig(ifaces, env); err != nil {
+		return fmt.Errorf("error applying network config: %w", err)
 	}
-	return
+	return nil
 }
 
 // getDatasources creates a slice of possible Datasources for cloudinit based

coreos-cloudinit_test.go

@@ -18,79 +18,9 @@ import (
 	"bytes"
 	"encoding/base64"
 	"errors"
-	"reflect"
 	"testing"
-
-	"github.com/flatcar/coreos-cloudinit/config"
-	"github.com/flatcar/coreos-cloudinit/datasource"
 )
 
-func TestMergeConfigs(t *testing.T) {
-	tests := []struct {
-		cc *config.CloudConfig
-		md datasource.Metadata
-
-		out config.CloudConfig
-	}{
-		{
-			// If md is empty and cc is nil, result should be empty
-			out: config.CloudConfig{},
-		},
-		{
-			// If md and cc are empty, result should be empty
-			cc:  &config.CloudConfig{},
-			out: config.CloudConfig{},
-		},
-		{
-			// If cc is empty, cc should be returned unchanged
-			cc:  &config.CloudConfig{SSHAuthorizedKeys: []string{"abc", "def"}, Hostname: "cc-host"},
-			out: config.CloudConfig{SSHAuthorizedKeys: []string{"abc", "def"}, Hostname: "cc-host"},
-		},
-		{
-			// If cc is empty, cc should be returned unchanged(overridden)
-			cc:  &config.CloudConfig{},
-			md:  datasource.Metadata{Hostname: "md-host", SSHPublicKeys: map[string]string{"key": "ghi"}},
-			out: config.CloudConfig{SSHAuthorizedKeys: []string{"ghi"}, Hostname: "md-host"},
-		},
-		{
-			// If cc is nil, cc should be returned unchanged(overridden)
-			md:  datasource.Metadata{Hostname: "md-host", SSHPublicKeys: map[string]string{"key": "ghi"}},
-			out: config.CloudConfig{SSHAuthorizedKeys: []string{"ghi"}, Hostname: "md-host"},
-		},
-		{
-			// user-data should override completely in the case of conflicts
-			cc:  &config.CloudConfig{SSHAuthorizedKeys: []string{"abc", "def"}, Hostname: "cc-host"},
-			md:  datasource.Metadata{Hostname: "md-host"},
-			out: config.CloudConfig{SSHAuthorizedKeys: []string{"abc", "def"}, Hostname: "cc-host"},
-		},
-		{
-			// Mixed merge should succeed
-			cc:  &config.CloudConfig{SSHAuthorizedKeys: []string{"abc", "def"}, Hostname: "cc-host"},
-			md:  datasource.Metadata{Hostname: "md-host", SSHPublicKeys: map[string]string{"key": "ghi"}},
-			out: config.CloudConfig{SSHAuthorizedKeys: []string{"abc", "def", "ghi"}, Hostname: "cc-host"},
-		},
-		{
-			// Completely non-conflicting merge should be fine
-			cc:  &config.CloudConfig{Hostname: "cc-host"},
-			md:  datasource.Metadata{SSHPublicKeys: map[string]string{"zaphod": "beeblebrox"}},
-			out: config.CloudConfig{Hostname: "cc-host", SSHAuthorizedKeys: []string{"beeblebrox"}},
-		},
-		{
-			// Non-mergeable settings in user-data should not be affected
-			cc:  &config.CloudConfig{Hostname: "cc-host", ManageEtcHosts: config.EtcHosts("lolz")},
-			md:  datasource.Metadata{Hostname: "md-host"},
-			out: config.CloudConfig{Hostname: "cc-host", ManageEtcHosts: config.EtcHosts("lolz")},
-		},
-	}
-
-	for i, tt := range tests {
-		out := mergeConfigs(tt.cc, tt.md)
-		if !reflect.DeepEqual(tt.out, out) {
-			t.Errorf("bad config (%d): want %#v, got %#v", i, tt.out, out)
-		}
-	}
-}
-
 func mustDecode(in string) []byte {
 	out, err := base64.StdEncoding.DecodeString(in)
 	if err != nil {

go.mod

@@ -8,14 +8,18 @@ require (
 	github.com/coreos/yaml v0.0.0-20141224210557-6b16a5714269
 	github.com/dotcloud/docker v0.11.2-0.20140522020950-55d41c3e21e1
 	github.com/sigma/vmw-ovflib v0.0.0-20150531125353-56b4f44581ca
+	github.com/stretchr/testify v1.8.2
 	github.com/vmware/vmw-guestinfo v0.0.0-20170622145319-ab8497750719
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/guelfey/go.dbus v0.0.0-20131113121618-f6a3a2366cc3 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/smartystreets/goconvey v1.7.2 // indirect
 	github.com/tarm/goserial v0.0.0-20140420040555-cdabc8d44e8e // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 replace github.com/vmware/vmw-guestinfo => github.com/sigma/vmw-guestinfo v0.0.0-20170622145319-ab8497750719

go.sum

@@ -4,6 +4,9 @@ github.com/coreos/go-systemd v0.0.0-20140326023052-4fbc5060a317 h1:OJi3CY9dHDNVE
 github.com/coreos/go-systemd v0.0.0-20140326023052-4fbc5060a317/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/yaml v0.0.0-20141224210557-6b16a5714269 h1:/1sjrpK5Mb6IwyFOKd+u7321tXfNAsj0Ci8CivZmSlo=
 github.com/coreos/yaml v0.0.0-20141224210557-6b16a5714269/go.mod h1:Bl1D/T9QJhVdu6eFoLrGxN90+admDLGaLz2HXH/VzDc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dotcloud/docker v0.11.2-0.20140522020950-55d41c3e21e1 h1:YgK/YfnYMOO3tJL9N0lYOd7IG2dhs9+kPUDXqPZPQ9c=
 github.com/dotcloud/docker v0.11.2-0.20140522020950-55d41c3e21e1/go.mod h1:ILmsztPoNJzhFWqgN1w/CccGRGm7j5uaU6kYvDN9scw=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -17,6 +20,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sigma/vmw-guestinfo v0.0.0-20170622145319-ab8497750719 h1:RdpTT9XV4QBLYPeqHUjGAhydFTimEjgvLDFyBfLAelI=
 github.com/sigma/vmw-guestinfo v0.0.0-20170622145319-ab8497750719/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U=
 github.com/sigma/vmw-ovflib v0.0.0-20150531125353-56b4f44581ca h1:zVbnn0fCxftRipg4oHS6mZIP6hcm6axhUff5jV1qcC8=
@@ -25,12 +30,23 @@ github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N
 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
 github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tarm/goserial v0.0.0-20140420040555-cdabc8d44e8e h1:DFPBoKQ4NuBYyj8GVNALoQbQqESZTJ8azlYNuvrAFTA=
 github.com/tarm/goserial v0.0.0-20140420040555-cdabc8d44e8e/go.mod h1:jcMo2Odv5FpDA6rp8bnczbUolcICW6t54K3s9gOlgII=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=