Task DNS Options (#7661)

Co-Authored-By: Tim Gross <[email protected]>
Co-Authored-By: Seth Hoenig <[email protected]>

api/resources.go

@@ -89,6 +89,12 @@ type Port struct {
 	To    int `mapstructure:"to"`
 }
 
+type DNSConfig struct {
+	Servers  []string `mapstructure:"servers"`
+	Searches []string `mapstructure:"searches"`
+	Options  []string `mapstructure:"options"`
+}
+
 // NetworkResource is used to describe required network
 // resources of a given task.
 type NetworkResource struct {
@@ -97,6 +103,7 @@ type NetworkResource struct {
 	CIDR          string
 	IP            string
 	MBits         *int
+	DNS           *DNSConfig
 	ReservedPorts []Port
 	DynamicPorts  []Port
 }

client/allocrunner/taskrunner/task_runner.go

@@ -940,6 +940,18 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig {
 	tr.networkIsolationLock.Lock()
 	defer tr.networkIsolationLock.Unlock()
 
+	var dns *drivers.DNSConfig
+	if alloc.AllocatedResources != nil && len(alloc.AllocatedResources.Shared.Networks) > 0 {
+		allocDNS := alloc.AllocatedResources.Shared.Networks[0].DNS
+		if allocDNS != nil {
+			dns = &drivers.DNSConfig{
+				Servers:  allocDNS.Servers,
+				Searches: allocDNS.Searches,
+				Options:  allocDNS.Options,
+			}
+		}
+	}
+
 	return &drivers.TaskConfig{
 		ID:            fmt.Sprintf("%s/%s/%s", alloc.ID, task.Name, invocationid),
 		Name:          task.Name,
@@ -963,6 +975,7 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig {
 		StderrPath:       tr.logmonHookConfig.stderrFifo,
 		AllocID:          tr.allocID,
 		NetworkIsolation: tr.networkIsolationSpec,
+		DNS:              dns,
 	}
 }
 

command/agent/job_endpoint.go

@@ -1095,6 +1095,14 @@ func ApiNetworkResourceToStructs(in []*api.NetworkResource) []*structs.NetworkRe
 			MBits: *nw.MBits,
 		}
 
+		if nw.DNS != nil {
+			out[i].DNS = &structs.DNSConfig{
+				Servers:  nw.DNS.Servers,
+				Searches: nw.DNS.Searches,
+				Options:  nw.DNS.Options,
+			}
+		}
+
 		if l := len(nw.DynamicPorts); l != 0 {
 			out[i].DynamicPorts = make([]structs.Port, l)
 			for j, dp := range nw.DynamicPorts {

command/assets/connect.nomad

@@ -256,6 +256,12 @@ job "countdash" {
       # port "http" {
       #   to = "8080"
       # }
+
+      # The "dns" stanza allows operators to override the DNS configuration
+      # inherited by the host client.
+      # dns {
+      #   servers = ["1.1.1.1"] 
+      # }
     }
     # The "service" stanza enables Consul Connect.
     service {

drivers/docker/driver.go

@@ -885,6 +885,19 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 		hostConfig.ShmSize = driverConfig.ShmSize
 	}
 
+	// setup Nomad DNS options, these are overridden by docker driver specific options
+	if task.DNS != nil {
+		hostConfig.DNS = task.DNS.Servers
+		hostConfig.DNSSearch = task.DNS.Searches
+		hostConfig.DNSOptions = task.DNS.Options
+	}
+
+	if len(driverConfig.DNSSearchDomains) > 0 {
+		hostConfig.DNSSearch = driverConfig.DNSSearchDomains
+	}
+	if len(driverConfig.DNSOptions) > 0 {
+		hostConfig.DNSOptions = driverConfig.DNSOptions
+	}
 	// set DNS servers
 	for _, ip := range driverConfig.DNSServers {
 		if net.ParseIP(ip) != nil {
@@ -948,9 +961,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 		hostConfig.Mounts = append(hostConfig.Mounts, hm)
 	}
 
-	// set DNS search domains and extra hosts
-	hostConfig.DNSSearch = driverConfig.DNSSearchDomains
-	hostConfig.DNSOptions = driverConfig.DNSOptions
 	hostConfig.ExtraHosts = driverConfig.ExtraHosts
 
 	hostConfig.IpcMode = driverConfig.IPCMode

drivers/docker/driver_unix_test.go

@@ -805,3 +805,59 @@ func TestDocker_ExecTaskStreaming(t *testing.T) {
 	dtestutil.ExecTaskStreamingConformanceTests(t, d, task.ID)
 
 }
+
+// Tests that a given DNSConfig properly configures dns
+func Test_dnsConfig(t *testing.T) {
+	if !tu.IsCI() {
+		t.Parallel()
+	}
+	testutil.DockerCompatible(t)
+	require := require.New(t)
+	harness := dockerDriverHarness(t, nil)
+	defer harness.Kill()
+
+	cases := []struct {
+		name string
+		cfg  *drivers.DNSConfig
+	}{
+		{
+			name: "nil DNSConfig",
+		},
+		{
+			name: "basic",
+			cfg: &drivers.DNSConfig{
+				Servers: []string{"1.1.1.1", "1.0.0.1"},
+			},
+		},
+		{
+			name: "full",
+			cfg: &drivers.DNSConfig{
+				Servers:  []string{"1.1.1.1", "1.0.0.1"},
+				Searches: []string{"local.test", "node.consul"},
+				Options:  []string{"ndots:2", "edns0"},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		taskCfg := newTaskConfig("", []string{"/bin/sleep", "1000"})
+		task := &drivers.TaskConfig{
+			ID:        uuid.Generate(),
+			Name:      "nc-demo",
+			AllocID:   uuid.Generate(),
+			Resources: basicResources,
+			DNS:       c.cfg,
+		}
+		require.NoError(task.EncodeConcreteDriverConfig(&taskCfg))
+
+		cleanup := harness.MkAllocDir(task, false)
+		defer cleanup()
+
+		_, _, err := harness.StartTask(task)
+		require.NoError(err)
+		defer harness.DestroyTask(task.ID, true)
+
+		dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg)
+	}
+
+}

drivers/exec/driver.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/drivers/shared/eventer"
 	"github.com/hashicorp/nomad/drivers/shared/executor"
+	"github.com/hashicorp/nomad/drivers/shared/resolvconf"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/plugins/base"
@@ -369,6 +370,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		user = "nobody"
 	}
 
+	if cfg.DNS != nil {
+		dnsMount, err := resolvconf.GenerateDNSMount(cfg.TaskDir().Dir, cfg.DNS)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err)
+		}
+		cfg.Mounts = append(cfg.Mounts, dnsMount)
+	}
+
 	execCmd := &executor.ExecCommand{
 		Cmd:              driverConfig.Command,
 		Args:             driverConfig.Args,

drivers/exec/driver_unix_test.go

@@ -107,3 +107,61 @@ func TestExec_ExecTaskStreaming(t *testing.T) {
 	dtestutil.ExecTaskStreamingConformanceTests(t, harness, task.ID)
 
 }
+
+// Tests that a given DNSConfig properly configures dns
+func TestExec_dnsConfig(t *testing.T) {
+	t.Parallel()
+	ctestutils.RequireRoot(t)
+	ctestutils.ExecCompatible(t)
+	require := require.New(t)
+	d := NewExecDriver(testlog.HCLogger(t))
+	harness := dtestutil.NewDriverHarness(t, d)
+	defer harness.Kill()
+
+	cases := []struct {
+		name string
+		cfg  *drivers.DNSConfig
+	}{
+		{
+			name: "nil DNSConfig",
+		},
+		{
+			name: "basic",
+			cfg: &drivers.DNSConfig{
+				Servers: []string{"1.1.1.1", "1.0.0.1"},
+			},
+		},
+		{
+			name: "full",
+			cfg: &drivers.DNSConfig{
+				Servers:  []string{"1.1.1.1", "1.0.0.1"},
+				Searches: []string{"local.test", "node.consul"},
+				Options:  []string{"ndots:2", "edns0"},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		task := &drivers.TaskConfig{
+			ID:   uuid.Generate(),
+			Name: "sleep",
+			DNS:  c.cfg,
+		}
+
+		cleanup := harness.MkAllocDir(task, false)
+		defer cleanup()
+
+		tc := &TaskConfig{
+			Command: "/bin/sleep",
+			Args:    []string{"9000"},
+		}
+		require.NoError(task.EncodeConcreteDriverConfig(&tc))
+
+		_, _, err := harness.StartTask(task)
+		require.NoError(err)
+		defer d.DestroyTask(task.ID, true)
+
+		dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg)
+	}
+
+}

drivers/java/driver.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/drivers/shared/eventer"
 	"github.com/hashicorp/nomad/drivers/shared/executor"
+	"github.com/hashicorp/nomad/drivers/shared/resolvconf"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/drivers"
@@ -344,6 +345,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		user = "nobody"
 	}
 
+	if cfg.DNS != nil {
+		dnsMount, err := resolvconf.GenerateDNSMount(cfg.TaskDir().Dir, cfg.DNS)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err)
+		}
+		cfg.Mounts = append(cfg.Mounts, dnsMount)
+	}
+
 	execCmd := &executor.ExecCommand{
 		Cmd:              absPath,
 		Args:             args,

drivers/java/driver_test.go

@@ -345,3 +345,56 @@ config {
 
 	require.EqualValues(t, expected, tc)
 }
+
+// Tests that a given DNSConfig properly configures dns
+func Test_dnsConfig(t *testing.T) {
+	t.Parallel()
+	ctestutil.RequireRoot(t)
+	javaCompatible(t)
+	require := require.New(t)
+	d := NewDriver(testlog.HCLogger(t))
+	harness := dtestutil.NewDriverHarness(t, d)
+	defer harness.Kill()
+
+	cases := []struct {
+		name string
+		cfg  *drivers.DNSConfig
+	}{
+		{
+			name: "nil DNSConfig",
+		},
+		{
+			name: "basic",
+			cfg: &drivers.DNSConfig{
+				Servers: []string{"1.1.1.1", "1.0.0.1"},
+			},
+		},
+		{
+			name: "full",
+			cfg: &drivers.DNSConfig{
+				Servers:  []string{"1.1.1.1", "1.0.0.1"},
+				Searches: []string{"local.test", "node.consul"},
+				Options:  []string{"ndots:2", "edns0"},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		tc := &TaskConfig{
+			Class: "Hello",
+			Args:  []string{"900"},
+		}
+		task := basicTask(t, "demo-app", tc)
+		task.DNS = c.cfg
+
+		cleanup := harness.MkAllocDir(task, false)
+		defer cleanup()
+
+		_, _, err := harness.StartTask(task)
+		require.NoError(err)
+		defer d.DestroyTask(task.ID, true)
+
+		dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg)
+	}
+
+}

drivers/qemu/driver.go

@@ -346,6 +346,17 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		"-nographic",
 	}
 
+	var netdevArgs []string
+	if cfg.DNS != nil {
+		if len(cfg.DNS.Servers) > 0 {
+			netdevArgs = append(netdevArgs, "dns="+cfg.DNS.Servers[0])
+		}
+
+		for _, s := range cfg.DNS.Searches {
+			netdevArgs = append(netdevArgs, "dnssearch="+s)
+		}
+	}
+
 	var monitorPath string
 	if driverConfig.GracefulShutdown {
 		if runtime.GOOS == "windows" {
@@ -384,7 +395,6 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		// Loop through the port map and construct the hostfwd string, to map
 		// reserved ports to the ports listenting in the VM
 		// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
-		var forwarding []string
 		taskPorts := cfg.Resources.NomadResources.Networks[0].PortLabels()
 		for label, guest := range driverConfig.PortMap {
 			host, ok := taskPorts[label]
@@ -393,14 +403,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 			}
 
 			for _, p := range protocols {
-				forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
+				netdevArgs = append(netdevArgs, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
 			}
 		}
 
-		if len(forwarding) != 0 {
+		if len(netdevArgs) != 0 {
 			args = append(args,
 				"-netdev",
-				fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")),
+				fmt.Sprintf("user,id=user.0,%s", strings.Join(netdevArgs, ",")),
 				"-device", "virtio-net,netdev=user.0",
 			)
 		}