diff --git a/api/tasks.go b/api/tasks.go index d03a4cb4a9a..350a50ffc90 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -857,6 +857,7 @@ type Vault struct { Env *bool `hcl:"env,optional"` ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"` ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"` + File *bool `mapstructure:"file" hcl:"file,optional"` } func (v *Vault) Canonicalize() { @@ -872,6 +873,9 @@ func (v *Vault) Canonicalize() { if v.ChangeSignal == nil { v.ChangeSignal = stringToPtr("SIGHUP") } + if v.File == nil { + v.File = boolToPtr(true) + } } // NewTask creates and initializes a new Task. diff --git a/api/tasks_test.go b/api/tasks_test.go index 02e20506a6e..ffb0f19f8ff 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -508,6 +508,7 @@ func TestTask_Canonicalize_Vault(t *testing.T) { Namespace: stringToPtr(""), ChangeMode: stringToPtr("restart"), ChangeSignal: stringToPtr("SIGHUP"), + File: boolToPtr(true), }, }, } diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index da05aacb39d..4aa8e60348e 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -58,6 +58,10 @@ var ( // directory TaskSecrets = "secrets" + // TaskPrivate is the name of the pruvate directory inside each task + // directory + TaskPrivate = "private" + // TaskDirs is the set of directories created in each tasks directory. TaskDirs = map[string]os.FileMode{TmpDirName: os.ModeSticky | 0777} @@ -304,6 +308,13 @@ func (d *AllocDir) UnmountAll() error { } } + if pathExists(dir.PrivateDir) { + if err := removeSecretDir(dir.PrivateDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the private dir %q: %v", dir.PrivateDir, err)) + } + } + // Unmount dev/ and proc/ have been mounted. if err := dir.unmountSpecialDirs(); err != nil { mErr.Errors = append(mErr.Errors, err) @@ -441,6 +452,10 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) { d.mu.RUnlock() return nil, fmt.Errorf("Reading secret file prohibited: %s", path) } + if filepath.HasPrefix(p, dir.PrivateDir) { + d.mu.RUnlock() + return nil, fmt.Errorf("Reading private file prohibited: %s", path) + } } d.mu.RUnlock() diff --git a/client/allocdir/task_dir.go b/client/allocdir/task_dir.go index d516c313cf1..7b9b081e9e5 100644 --- a/client/allocdir/task_dir.go +++ b/client/allocdir/task_dir.go @@ -39,6 +39,10 @@ type TaskDir struct { // /secrets/ SecretsDir string + // PrivateDir is the path to private/ directory on the host + // /private/ + PrivateDir string + // skip embedding these paths in chroots. Used for avoiding embedding // client.alloc_dir recursively. skip map[string]struct{} @@ -66,6 +70,7 @@ func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string) SharedTaskDir: filepath.Join(taskDir, SharedAllocName), LocalDir: filepath.Join(taskDir, TaskLocal), SecretsDir: filepath.Join(taskDir, TaskSecrets), + PrivateDir: filepath.Join(taskDir, TaskPrivate), skip: skip, logger: logger, } @@ -128,6 +133,15 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { return err } + // Create the private directory + if err := createSecretDir(t.PrivateDir); err != nil { + return err + } + + if err := dropDirPermissions(t.PrivateDir, os.ModePerm); err != nil { + return err + } + // Build chroot if chroot filesystem isolation is going to be used if createChroot { if err := t.buildChroot(chroot); err != nil { diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 845ae4e1ce1..9057750d3f2 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1610,7 +1610,7 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { require.False(t, finalState.Failed) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) @@ -1675,7 +1675,7 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) { require.Equal(t, 1, count) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index 8aa33a429dc..d1b5460766e 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -81,6 +81,10 @@ type vaultHook struct { // tokenPath is the path in which to read and write the token tokenPath string + // sharedTokenPath is the path in which to only write, but never + // read the token from + sharedTokenPath string + // alloc is the allocation alloc *structs.Allocation @@ -129,7 +133,7 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe // Try to recover a token if it was previously written in the secrets // directory recoveredToken := "" - h.tokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) + h.tokenPath = filepath.Join(req.TaskDir.PrivateDir, vaultTokenFile) data, err := ioutil.ReadFile(h.tokenPath) if err != nil { if !os.IsNotExist(err) { @@ -141,6 +145,7 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe // Store the recovered token recoveredToken = string(data) } + h.sharedTokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) // Launch the token manager go h.run(recoveredToken) @@ -343,9 +348,14 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) { // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { - if err := ioutil.WriteFile(h.tokenPath, []byte(token), 0666); err != nil { + if err := ioutil.WriteFile(h.tokenPath, []byte(token), 0600); err != nil { return fmt.Errorf("failed to write vault token: %v", err) } + if h.vaultStanza.File { + if err := ioutil.WriteFile(h.sharedTokenPath, []byte(token), 0666); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + } return nil } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index f88a58a2e0b..6b9cb8ed1f0 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1203,6 +1203,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, Env: *apiTask.Vault.Env, ChangeMode: *apiTask.Vault.ChangeMode, ChangeSignal: *apiTask.Vault.ChangeSignal, + File: *apiTask.Vault.File, } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 28e08ed3181..d8dd84e0b63 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2723,6 +2723,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: helper.BoolToPtr(true), ChangeMode: helper.StringToPtr("c"), ChangeSignal: helper.StringToPtr("sighup"), + File: helper.BoolToPtr(true), }, Templates: []*api.Template{ { @@ -3128,6 +3129,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Env: true, ChangeMode: "c", ChangeSignal: "sighup", + File: true, }, Templates: []*structs.Template{ { diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index f9d3fb59cd1..e0166959add 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -240,6 +240,7 @@ etc/ lib/ lib64/ local/ +private/ proc/ secrets/ sys/ diff --git a/jobspec/parse.go b/jobspec/parse.go index aad9d9fbe1b..e2184317849 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -510,6 +510,7 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { "env", "change_mode", "change_signal", + "file", } if err := checkHCLKeys(listVal, valid); err != nil { return multierror.Prefix(err, "vault ->") diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index e138d44e214..0cdfd854d46 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -212,6 +212,7 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { tgVault := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), } if err := parseVault(tgVault, o); err != nil { diff --git a/jobspec/parse_job.go b/jobspec/parse_job.go index b59b3b59456..9cf9b72e0fb 100644 --- a/jobspec/parse_job.go +++ b/jobspec/parse_job.go @@ -193,6 +193,7 @@ func parseJob(result *api.Job, list *ast.ObjectList) error { jobVault := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), } if err := parseVault(jobVault, o); err != nil { diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index ff81b6ba66a..fa2e31e2712 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -291,6 +291,7 @@ func parseTask(item *ast.ObjectItem, keys []string) (*api.Task, error) { v := &api.Vault{ Env: boolToPtr(true), ChangeMode: stringToPtr("restart"), + File: boolToPtr(true), } if err := parseVault(v, o); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 7c9ff243a10..b7bf7f11e76 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -354,6 +354,7 @@ func TestParse(t *testing.T) { Policies: []string{"foo", "bar"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), }, Templates: []*api.Template{ { @@ -406,6 +407,7 @@ func TestParse(t *testing.T) { Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeSignal), ChangeSignal: stringToPtr("SIGUSR1"), + File: boolToPtr(false), }, }, }, @@ -765,6 +767,7 @@ func TestParse(t *testing.T) { Policies: []string{"group"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), }, }, { @@ -773,6 +776,7 @@ func TestParse(t *testing.T) { Policies: []string{"task"}, Env: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(false), }, }, }, @@ -786,6 +790,7 @@ func TestParse(t *testing.T) { Policies: []string{"job"}, Env: boolToPtr(true), ChangeMode: stringToPtr(vaultChangeModeRestart), + File: boolToPtr(true), }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 20a8171e4d0..7bd2a234cb6 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -350,6 +350,7 @@ job "binstore-storagelocker" { env = false change_mode = "signal" change_signal = "SIGUSR1" + file = false } } diff --git a/jobspec/test-fixtures/vault_inheritance.hcl b/jobspec/test-fixtures/vault_inheritance.hcl index 18d83d9f5de..7425cea21de 100644 --- a/jobspec/test-fixtures/vault_inheritance.hcl +++ b/jobspec/test-fixtures/vault_inheritance.hcl @@ -14,6 +14,7 @@ job "example" { vault { policies = ["task"] env = false + file = false } } } diff --git a/jobspec2/parse_job.go b/jobspec2/parse_job.go index 9b533874f50..81301a7335e 100644 --- a/jobspec2/parse_job.go +++ b/jobspec2/parse_job.go @@ -64,6 +64,9 @@ func normalizeVault(v *api.Vault) { if v.ChangeMode == nil { v.ChangeMode = stringToPtr("restart") } + if v.File == nil { + v.File = boolToPtr(true) + } } func normalizeNetworkPorts(networks []*api.NetworkResource) { diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 1a5751a8c7e..bb5e36e0162 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -6760,6 +6760,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, }, }, Expected: &TaskDiff{ @@ -6787,6 +6788,12 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "true", }, + { + Type: DiffTypeAdded, + Name: "File", + Old: "", + New: "true", + }, }, Objects: []*ObjectDiff{ { @@ -6820,6 +6827,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, }, }, New: &Task{}, @@ -6848,6 +6856,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "", }, + { + Type: DiffTypeDeleted, + Name: "File", + Old: "true", + New: "", + }, }, Objects: []*ObjectDiff{ { @@ -6882,6 +6896,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, }, }, New: &Task{ @@ -6891,6 +6906,7 @@ func TestTaskDiff(t *testing.T) { Env: false, ChangeMode: "restart", ChangeSignal: "foo", + File: false, }, }, Expected: &TaskDiff{ @@ -6918,6 +6934,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "false", }, + { + Type: DiffTypeEdited, + Name: "File", + Old: "true", + New: "false", + }, { Type: DiffTypeEdited, Name: "Namespace", @@ -6959,6 +6981,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, }, }, New: &Task{ @@ -6968,6 +6991,7 @@ func TestTaskDiff(t *testing.T) { Env: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", + File: true, }, }, Expected: &TaskDiff{ @@ -6995,6 +7019,12 @@ func TestTaskDiff(t *testing.T) { Old: "true", New: "true", }, + { + Type: DiffTypeNone, + Name: "File", + Old: "true", + New: "true", + }, { Type: DiffTypeNone, Name: "Namespace", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index b376ebfaa14..2d72f46a329 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8974,13 +8974,10 @@ type Vault struct { // ChangeSignal is the signal sent to the task when a new token is // retrieved. This is only valid when using the signal change mode. ChangeSignal string -} -func DefaultVaultBlock() *Vault { - return &Vault{ - Env: true, - ChangeMode: VaultChangeModeRestart, - } + // File marks whether the Vault Token should be exposed in the file + // vault_token in the task's secrets directory. + File bool } // Copy returns a copy of this Vault block. diff --git a/website/content/docs/internals/filesystem.mdx b/website/content/docs/internals/filesystem.mdx index 8017f73827d..779af58ef6c 100644 --- a/website/content/docs/internals/filesystem.mdx +++ b/website/content/docs/internals/filesystem.mdx @@ -29,10 +29,12 @@ allocation directory like the one below. │ └── tmp ├── task1 │ ├── local +│ ├── private │ ├── secrets │ └── tmp └── task2 ├── local + ├── private ├── secrets └── tmp ``` @@ -68,6 +70,11 @@ allocation directory like the one below. `NOMAD_TASK_DIR`. Note this is not the same as the "task working directory". This directory is private to the task. + - **«taskname»/private/**: This directory is used by nomad to store private files + related to the allocation, e.g., vault tokens, that are not always shared with tasks + when using `image` isolation. The contents of files in this directory cannot be read + by the `nomad alloc fs` command. + - **«taskname»/secrets/**: This directory is the location provided to the task as `NOMAD_SECRETS_DIR`. The contents of files in this directory cannot be read by the `nomad alloc fs` command. It can be used to store secret data that @@ -96,6 +103,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/ $ nomad alloc fs c0b2245f task1/ Mode Size Modified Time Name drwxrwxrwx 4.0 KiB 2020-10-27T18:00:33Z local/ +drwxrwxrwx 60 B 2020-10-27T18:00:32Z private/ drwxrwxrwx 60 B 2020-10-27T18:00:32Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/ ``` @@ -149,6 +157,7 @@ minimal filesystem tree: │ └── tmp └── task1 ├── local + ├── private ├── secrets └── tmp ``` @@ -164,6 +173,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z task1/ $ nomad alloc fs b0686b27 task1 Mode Size Modified Time Name drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z local/ +drwxrwxrwx 60 B 2020-10-27T18:51:54Z private/ drwxrwxrwx 60 B 2020-10-27T18:51:54Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z tmp/ @@ -286,6 +296,7 @@ contents], in addition to the `NOMAD_ALLOC_DIR`, `NOMAD_TASK_DIR`, and ├── lib32 ├── lib64 ├── local + ├── private ├── proc ├── run ├── sbin @@ -314,6 +325,7 @@ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib32/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib64/ drwxrwxrwx 4.0 KiB 2020-10-27T19:05:22Z local/ +drwxrwxrwx 60 B 2020-10-27T19:05:22Z private/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:24Z proc/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z run/ drwxr-xr-x 12 KiB 2020-10-27T19:05:22Z sbin/ @@ -333,6 +345,7 @@ $ nomad alloc exec eebd13a7 /bin/sh $ mount ... /dev/mapper/root on /alloc type ext4 (rw,relatime,errors=remount-ro,data=ordered) +tmpfs on /private type tmpfs (rw,noexec,relatime,size=1024k) tmpfs on /secrets type tmpfs (rw,noexec,relatime,size=1024k) ... ``` @@ -376,6 +389,7 @@ minimal filesystem tree: └── task3 ├── executor.out ├── local + ├── private ├── secrets └── tmp ``` @@ -387,6 +401,7 @@ $ nomad alloc fs 87ec7d12 task3 Mode Size Modified Time Name -rw-r--r-- 140 B 2020-10-27T19:15:33Z executor.out drwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z local/ +drwxrwxrwx 60 B 2020-10-27T19:15:33Z private/ drwxrwxrwx 60 B 2020-10-27T19:15:33Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z tmp/ ``` diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index e6ab5ac6a86..721c8b5732a 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -45,6 +45,7 @@ to the secret directory at `secrets/vault_token` and by injecting a `VAULT_TOKEN environment variable. If the Nomad cluster is [configured](/docs/configuration/vault#namespace) to use [Vault Namespaces](https://www.vaultproject.io/docs/enterprise/namespaces), a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` is set. +This behavior can be altered using the `env` and `file` parameters. If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or network error), the client will attempt to retrieve a new Vault token. If successful, the @@ -78,6 +79,9 @@ with Vault as well. the task requires. The Nomad client will retrieve a Vault token that is limited to those policies. +- `file` `(bool: true)` - Specifies if the Vault token should be written to + `secrets/vault_token`. + ## `vault` Examples The following examples only show the `vault` stanzas. Remember that the @@ -109,6 +113,35 @@ vault { } ``` +### Private Token and Noop + +This example retrieves a Vault token that is not shared with the task when using +a driver that provides `image` isolation like [Docker][docker]. + +This allows to get a powerful Vault token that interacts with the task's +[`template`][template] stanzas to issue all kinds of secrets (e.g., database +secrets, other vault tokens, etc.), without sharing that issuing power with +the task itself: + +```hcl +vault { + policies = ["tls-policy", "nomad-job-policy"] + change_mode = "noop" + env = false + file = false +} + +template { + data = <<-EOH + {{with secret "auth/token/create/nomad-job" "policies=examplepolicy"}}{{.Auth.ClientToken}}{{ end }} + EOH + destination = "${NOMAD_SECRETS_DIR}/examplepolicy.token" + change_mode = "noop" + perms = "600" +} +``` + + ### Vault Namespace This example shows specifying a particular Vault namespace for a given task. @@ -128,3 +161,4 @@ vault { [restart]: /docs/job-specification/restart 'Nomad restart Job Specification' [template]: /docs/job-specification/template 'Nomad template Job Specification' [vault]: https://www.vaultproject.io/ 'Vault by HashiCorp' +[docker]: (/docs/drivers/docker) 'Docker Driver'