Merge pull request #941 from hashicorp/f-interface-name

Use an interface name to find the bind addr for rpc, http and serf components

command/agent/agent.go

@@ -127,9 +127,29 @@ func (a *Agent) serverConfig() (*nomad.Config, error) {
 	}
 	if addr := a.config.Addresses.RPC; addr != "" {
 		conf.RPCAddr.IP = net.ParseIP(addr)
+	} else if device := a.config.Interfaces.RPC; device != "" {
+		ip, err := ipOfDevice(device)
+		if err != nil {
+			return nil, err
+		}
+		conf.RPCAddr.IP = ip
 	}
 	if addr := a.config.Addresses.Serf; addr != "" {
 		conf.SerfConfig.MemberlistConfig.BindAddr = addr
+	} else if device := a.config.Interfaces.Serf; device != "" {
+		ip, err := ipOfDevice(device)
+		if err != nil {
+			return nil, err
+		}
+		conf.SerfConfig.MemberlistConfig.BindAddr = ip.String()
+	}
+
+	if device := a.config.Interfaces.HTTP; device != "" && a.config.Addresses.HTTP == "" {
+		ip, err := ipOfDevice(device)
+		if err != nil {
+			return nil, err
+		}
+		a.config.Addresses.HTTP = ip.String()
 	}
 
 	// Set up the ports
@@ -209,12 +229,21 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 	conf.Node.Name = a.config.NodeName
 	conf.Node.Meta = a.config.Client.Meta
 	conf.Node.NodeClass = a.config.Client.NodeClass
+
+	// Setting the proper HTTP Addr
 	httpAddr := fmt.Sprintf("%s:%d", a.config.BindAddr, a.config.Ports.HTTP)
-	if a.config.Addresses.HTTP != "" && a.config.AdvertiseAddrs.HTTP == "" {
+	if a.config.Addresses.HTTP != "" && a.config.AdvertiseAddrs.HTTP == "" && a.config.Interfaces.HTTP == "" {
 		httpAddr = fmt.Sprintf("%s:%d", a.config.Addresses.HTTP, a.config.Ports.HTTP)
 		if _, err := net.ResolveTCPAddr("tcp", httpAddr); err != nil {
 			return nil, fmt.Errorf("error resolving http addr: %v:", err)
 		}
+	} else if a.config.Interfaces.HTTP != "" && a.config.AdvertiseAddrs.HTTP == "" {
+		ip, err := ipOfDevice(a.config.Interfaces.HTTP)
+		if err != nil {
+			return nil, fmt.Errorf("error finding ip address from interface %q: %v", a.config.Interfaces.HTTP, err)
+		}
+		a.config.Addresses.HTTP = ip.String()
+		httpAddr = fmt.Sprintf("%s:%d", ip.String(), a.config.Ports.HTTP)
 	} else if a.config.AdvertiseAddrs.HTTP != "" {
 		addr, err := net.ResolveTCPAddr("tcp", a.config.AdvertiseAddrs.HTTP)
 		if err != nil {
@@ -223,7 +252,6 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 		httpAddr = fmt.Sprintf("%s:%d", addr.IP.String(), addr.Port)
 	}
 	conf.Node.HTTPAddr = httpAddr
-	conf.Version = a.config.Version
 
 	// Reserve resources on the node.
 	r := conf.Node.Reserved
@@ -237,6 +265,8 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 	r.IOPS = a.config.Client.Reserved.IOPS
 	conf.GloballyReservedPorts = a.config.Client.Reserved.ParsedReservedPorts
 
+	conf.Version = a.config.Version
+
 	return conf, nil
 }
 

command/agent/config.go

@@ -46,6 +46,10 @@ type Config struct {
 	// Addresses is used to override the network addresses we bind to.
 	Addresses *Addresses `mapstructure:"addresses"`
 
+	// Interfaces is used to override the network addresses we bind to by
+	// providing device names
+	Interfaces *Interfaces `mapstructure:"interfaces"`
+
 	// AdvertiseAddrs is used to control the addresses we advertise.
 	AdvertiseAddrs *AdvertiseAddrs `mapstructure:"advertise"`
 
@@ -255,6 +259,14 @@ type Addresses struct {
 	Serf string `mapstructure:"serf"`
 }
 
+// Interfaces provides an alternative to the Addresses configuration. We pick an
+// ip configured on the devide specified and use that to bind.
+type Interfaces struct {
+	HTTP string `mapstructure:"http"`
+	RPC  string `mapstructure:"rpc"`
+	Serf string `mapstructure:"serf"`
+}
+
 // AdvertiseAddrs is used to control the addresses we advertise out for
 // different network services. Not all network services support an
 // advertise address. All are optional and default to BindAddr.
@@ -366,6 +378,7 @@ func DefaultConfig() *Config {
 			Serf: 4648,
 		},
 		Addresses:      &Addresses{},
+		Interfaces:     &Interfaces{},
 		AdvertiseAddrs: &AdvertiseAddrs{},
 		Atlas:          &AtlasConfig{},
 		Client: &ClientConfig{
@@ -496,6 +509,14 @@ func (c *Config) Merge(b *Config) *Config {
 		result.Addresses = result.Addresses.Merge(b.Addresses)
 	}
 
+	// Apply the interfaces config
+	if result.Interfaces == nil && b.Interfaces != nil {
+		interfaces := *b.Interfaces
+		result.Interfaces = &interfaces
+	} else if b.Interfaces != nil {
+		result.Interfaces = result.Interfaces.Merge(b.Interfaces)
+	}
+
 	// Apply the advertise addrs config
 	if result.AdvertiseAddrs == nil && b.AdvertiseAddrs != nil {
 		advertise := *b.AdvertiseAddrs
@@ -675,6 +696,22 @@ func (a *Addresses) Merge(b *Addresses) *Addresses {
 	return &result
 }
 
+// Merge is used to merge two interfaces configs together.
+func (i *Interfaces) Merge(b *Interfaces) *Interfaces {
+	result := *i
+
+	if b.HTTP != "" {
+		result.HTTP = b.HTTP
+	}
+	if b.RPC != "" {
+		result.RPC = b.RPC
+	}
+	if b.Serf != "" {
+		result.Serf = b.Serf
+	}
+	return &result
+}
+
 // Merge merges two advertise addrs configs together.
 func (a *AdvertiseAddrs) Merge(b *AdvertiseAddrs) *AdvertiseAddrs {
 	result := *a

command/agent/config_parse.go

@@ -78,6 +78,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 		"enable_debug",
 		"ports",
 		"addresses",
+		"interfaces",
 		"advertise",
 		"client",
 		"server",
@@ -102,6 +103,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 	}
 	delete(m, "ports")
 	delete(m, "addresses")
+	delete(m, "interfaces")
 	delete(m, "advertise")
 	delete(m, "client")
 	delete(m, "server")
@@ -128,6 +130,13 @@ func parseConfig(result *Config, list *ast.ObjectList) error {
 		}
 	}
 
+	// Parse interfaces
+	if o := list.Filter("interfaces"); len(o.Items) > 0 {
+		if err := parseInterfaces(&result.Interfaces, o); err != nil {
+			return multierror.Prefix(err, "interfaces ->")
+		}
+	}
+
 	// Parse advertise
 	if o := list.Filter("advertise"); len(o.Items) > 0 {
 		if err := parseAdvertise(&result.AdvertiseAddrs, o); err != nil {
@@ -244,6 +253,38 @@ func parseAddresses(result **Addresses, list *ast.ObjectList) error {
 	return nil
 }
 
+func parseInterfaces(result **Interfaces, list *ast.ObjectList) error {
+	list = list.Elem()
+	if len(list.Items) > 1 {
+		return fmt.Errorf("only one 'interfaces' block allowed")
+	}
+
+	// Get our interfaces object
+	listVal := list.Items[0].Val
+
+	// Check for the invalid keys
+	valid := []string{
+		"http",
+		"rpc",
+		"serf",
+	}
+	if err := checkHCLKeys(listVal, valid); err != nil {
+		return err
+	}
+
+	var m map[string]interface{}
+	if err := hcl.DecodeObject(&m, listVal); err != nil {
+		return err
+	}
+
+	var interfaces Interfaces
+	if err := mapstructure.WeakDecode(m, &interfaces); err != nil {
+		return err
+	}
+	*result = &interfaces
+	return nil
+}
+
 func parseAdvertise(result **AdvertiseAddrs, list *ast.ObjectList) error {
 	list = list.Elem()
 	if len(list.Items) > 1 {

command/agent/util.go

@@ -1,11 +1,53 @@
 package agent
 
 import (
+	"fmt"
 	"math/rand"
+	"net"
 	"time"
 )
 
 // Returns a random stagger interval between 0 and the duration
 func randomStagger(intv time.Duration) time.Duration {
 	return time.Duration(uint64(rand.Int63()) % uint64(intv))
 }
+
+// IpOfDevice returns a routable ip addr of a device
+func ipOfDevice(name string) (net.IP, error) {
+	intf, err := net.InterfaceByName(name)
+	if err != nil {
+		return nil, err
+	}
+	addrs, err := intf.Addrs()
+	if err != nil {
+		return nil, err
+	}
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("no ips were detected on the interface: %v", name)
+	}
+
+	// Iterating through the IPs configured for that device and returning the
+	// the first ipv4 address configured. If no ipv4 addresses are configured,
+	// we return the first ipv6 addr if any ipv6 addr is configured.
+	var ipv6Addrs []net.IP
+	for _, addr := range addrs {
+		var ip net.IP
+		switch v := (addr).(type) {
+		case *net.IPNet:
+			ip = v.IP
+			if ip.To4() != nil {
+				return ip, nil
+			}
+			if ip.To16() != nil {
+				ipv6Addrs = append(ipv6Addrs, ip)
+				continue
+			}
+		case *net.IPAddr:
+			continue
+		}
+	}
+	if len(ipv6Addrs) > 0 {
+		return ipv6Addrs[0], nil
+	}
+	return nil, fmt.Errorf("no ips were detected on the interface: %v", name)
+}

website/source/docs/agent/config.html.md

@@ -144,6 +144,22 @@ nodes, unless otherwise specified:
     server nodes from the same datacenter if possible. Used only on server
     nodes.
 
+* <a id="interfaces">`interfaces`</a>: Provides an alternative to the
+ `addresses` configuration. Operators can provide network device names to which
+ Nomad binds individual network services. Nomad looks for the first IPv4
+ address configured for the device and uses it, and if no IPv4 address is
+ present then it looks for an IPv6 address. The value is a map of device names of
+ network interfaces and supports the following keys:
+  <br>
+  * `http`: The device name the HTTP server is bound to. Applies to both clients and servers.
+  * `rpc`: The device name to bind the internal RPC interfaces to. Should be exposed
+    only to other cluster members if possible. Used only on server nodes, but
+    must be accessible from all agents.
+  * `serf`: The device name used to bind the gossip layer to. Both a TCP and UDP
+    listener will be exposed on this address. Should be restricted to only
+    server nodes from the same datacenter if possible. Used only on server
+    nodes.
+
 * `advertise`: Controls the advertise address for individual network services.
   This can be used to advertise a different address to the peers of a server
   node to support more complex network configurations such as NAT. This