Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

client: interpolate driver configurations #4843

Merged
merged 8 commits into from
Nov 16, 2018
Merged
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
__BACKWARDS INCOMPATIBILITIES:__
* core: Switch to structured logging using [go-hclog](https://github.com/hashicorp/go-hclog)
* core: Allow the != constraint to match against keys that do not exist [[GH-4875](https://github.com/hashicorp/nomad/pull/4875)]
* client: Task config interpolation requires names to be valid identifiers
(`node.region` or `NOMAD_DC`). Interpolating other variables requires a new
indexing syntax: `env[".invalid.identifier."]`. [[GH-4843](https://github.com/hashicorp/nomad/issues/4843)]

IMPROVEMENTS:
* core: Added advertise address to client node meta data [[GH-4390](https://github.com/hashicorp/nomad/issues/4390)]
Expand Down
23 changes: 22 additions & 1 deletion client/allocrunner/taskrunner/task_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -462,8 +462,29 @@ func (tr *TaskRunner) runDriver() error {
// TODO(nickethier): make sure this uses alloc.AllocatedResources once #4750 is rebased
taskConfig := tr.buildTaskConfig()

// TODO: load variables
// Build hcl context variables
vars, errs, err := tr.envBuilder.Build().AllValues()
if err != nil {
return fmt.Errorf("error building environment variables: %v", err)
}

// Handle per-key errors
if len(errs) > 0 {
keys := make([]string, 0, len(errs))
for k, err := range errs {
keys = append(keys, k)

if tr.logger.IsTrace() {
// Verbosely log every diagnostic for debugging
tr.logger.Trace("error building environment variables", "key", k, "error", err)
}
}

tr.logger.Warn("some environment variables not available for rendering", "keys", strings.Join(keys, ", "))
schmichael marked this conversation as resolved.
Show resolved Hide resolved
}

evalCtx := &hcl.EvalContext{
Variables: vars,
Functions: shared.GetStdlibFuncs(),
}

Expand Down
9 changes: 6 additions & 3 deletions client/allocrunner/taskrunner/task_runner_getters.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,22 @@ func (tr *TaskRunner) setVaultToken(token string) {
tr.envBuilder.SetVaultToken(token, tr.task.Vault.Env)
}

// getDriverHandle returns a driver handle and its result proxy. Use the
// result proxy instead of the handle's WaitCh.
// getDriverHandle returns a driver handle.
func (tr *TaskRunner) getDriverHandle() *DriverHandle {
tr.handleLock.Lock()
defer tr.handleLock.Unlock()
return tr.handle
}

// setDriverHanlde sets the driver handle and creates a new result proxy.
// setDriverHandle sets the driver handle and updates the driver network in the
// task's environment.
func (tr *TaskRunner) setDriverHandle(handle *DriverHandle) {
tr.handleLock.Lock()
defer tr.handleLock.Unlock()
tr.handle = handle

// Update the environment's driver network
tr.envBuilder.SetDriverNetwork(handle.net)
}

func (tr *TaskRunner) clearDriverHandle() {
Expand Down
56 changes: 56 additions & 0 deletions client/allocrunner/taskrunner/task_runner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ import (
"fmt"
"path/filepath"
"testing"
"time"

"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
consulapi "github.com/hashicorp/nomad/client/consul"
cstate "github.com/hashicorp/nomad/client/state"
"github.com/hashicorp/nomad/client/vaultclient"
mockdriver "github.com/hashicorp/nomad/drivers/mock"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
Expand Down Expand Up @@ -154,3 +156,57 @@ func TestTaskRunner_Restore_Running(t *testing.T) {
}
assert.Equal(t, 1, started)
}

// TestTaskRunner_TaskEnv asserts driver configurations are interpolated.
func TestTaskRunner_TaskEnv(t *testing.T) {
t.Parallel()
require := require.New(t)

alloc := mock.BatchAlloc()
alloc.Job.TaskGroups[0].Meta = map[string]string{
"common_user": "somebody",
}
task := alloc.Job.TaskGroups[0].Tasks[0]
task.Name = "testtask_taskenv"
task.Driver = "mock_driver"
task.Meta = map[string]string{
"foo": "bar",
}

// Use interpolation from both node attributes and meta vars
task.Config = map[string]interface{}{
"run_for": "1ms",
"stdout_string": `${node.region} ${NOMAD_META_foo} ${NOMAD_META_common_user}`,
}

conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name)
defer cleanup()

// Run the first TaskRunner
tr, err := NewTaskRunner(conf)
require.NoError(err)
go tr.Run()
defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))

// Wait for task to complete
select {
case <-tr.WaitCh():
case <-time.After(3 * time.Second):
}

// Get the mock driver plugin
driverPlugin, err := conf.PluginSingletonLoader.Dispense(
mockdriver.PluginID.Name,
mockdriver.PluginID.PluginType,
nil,
conf.Logger,
)
require.NoError(err)
mockDriver := driverPlugin.Plugin().(*mockdriver.Driver)

// Assert its config has been properly interpolated
driverCfg, mockCfg := mockDriver.GetTaskConfig()
require.NotNil(driverCfg)
require.NotNil(mockCfg)
assert.Equal(t, "global bar somebody", mockCfg.StdoutString)
}
8 changes: 2 additions & 6 deletions client/config/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ import (

"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/mitchellh/go-testing-interface"
)

// TestClientConfig returns a default client configuration for test clients and
// a cleanup func to remove the state and alloc dirs when finished.
func TestClientConfig(t testing.T) (*Config, func()) {
conf := DefaultConfig()
conf.Node = mock.Node()
conf.Logger = testlog.HCLogger(t)

// Create a tempdir to hold state and alloc subdirs
Expand Down Expand Up @@ -42,11 +43,6 @@ func TestClientConfig(t testing.T) (*Config, func()) {

conf.VaultConfig.Enabled = helper.BoolToPtr(false)
conf.DevMode = true
conf.Node = &structs.Node{
Reserved: &structs.Resources{
DiskMB: 0,
},
}

// Loosen GC threshold
conf.GCDiskUsageThreshold = 98.0
Expand Down
65 changes: 65 additions & 0 deletions client/driver/env/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/hashicorp/nomad/helper"
hargs "github.com/hashicorp/nomad/helper/args"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/zclconf/go-cty/cty"
)

// A set of environment variables that are exported by each driver.
Expand Down Expand Up @@ -159,6 +160,70 @@ func (t *TaskEnv) All() map[string]string {
return m
}

// AllValues is a map of the task's environment variables and the node's
// attributes with cty.Value (String) values. Errors including keys are
// returned in a map by key name.
//
// In the rare case of a fatal error, only an error value is returned. This is
// likely a programming error as user input should not be able to cause a fatal
// error.
func (t *TaskEnv) AllValues() (map[string]cty.Value, map[string]error, error) {
errs := make(map[string]error)

// Intermediate map for building up nested go types
allMap := make(map[string]interface{}, len(t.EnvMap)+len(t.NodeAttrs))

// Intermediate map for all env vars including those whose keys that
// cannot be nested (eg foo...bar)
envMap := make(map[string]cty.Value, len(t.EnvMap))

// Prepare job-based variables (eg job.meta, job.group.task.env, etc)
for k, v := range t.EnvMap {
if err := addNestedKey(allMap, k, v); err != nil {
errs[k] = err
}
envMap[k] = cty.StringVal(v)
}

// Prepare node-based variables (eg node.*, attr.*, meta.*)
for k, v := range t.NodeAttrs {
if err := addNestedKey(allMap, k, v); err != nil {
errs[k] = err
}
}

// Add flat envMap as a Map to allMap so users can access any key via
// HCL2's indexing syntax: ${env["foo...bar"]}
allMap["env"] = cty.MapVal(envMap)

// Add meta and attr to node if they exist to properly namespace things
// a bit.
nodeMapI, ok := allMap["node"]
if !ok {
return nil, nil, fmt.Errorf("missing node variable")
}
nodeMap, ok := nodeMapI.(map[string]interface{})
if !ok {
return nil, nil, fmt.Errorf("invalid type for node variable: %T", nodeMapI)
}
if attrMap, ok := allMap["attr"]; ok {
nodeMap["attr"] = attrMap
}
if metaMap, ok := allMap["meta"]; ok {
nodeMap["meta"] = metaMap
}

// ctyify the entire tree of strings and maps
tree, err := ctyify(allMap)
if err != nil {
// This should not be possible and is likely a programming
// error. Invalid user input should be cleaned earlier.
return nil, nil, err
}

return tree, errs, nil
}

// ParseAndReplace takes the user supplied args replaces any instance of an
// environment variable or Nomad variable in the args with the actual value.
func (t *TaskEnv) ParseAndReplace(args []string) []string {
Expand Down
Loading