From 86170bca5f5a372580c38d8e66a63762523ae595 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 22 Nov 2023 13:02:17 +0000 Subject: [PATCH 1/3] Support rootless plugin containers --- .github/workflows/test-go.yml | 27 +++- api/sys_plugins_runtimes.go | 1 + changelog/24236.txt | 3 + command/plugin_runtime_register.go | 11 ++ command/plugin_runtime_register_test.go | 8 +- go.mod | 5 +- go.sum | 16 +- sdk/go.mod | 5 +- sdk/go.sum | 11 +- sdk/helper/pluginruntimeutil/config.go | 1 + sdk/helper/pluginutil/run_config.go | 6 +- vault/external_plugin_container_test.go | 140 +++++++++--------- vault/external_plugin_test.go | 8 +- .../plugin/external_plugin_test.go | 24 +-- vault/logical_system.go | 7 + vault/logical_system_paths.go | 9 ++ vault/logical_system_test.go | 3 + vault/testdata/Dockerfile | 4 + vault/testing.go | 38 ++--- .../system/plugins-runtimes-catalog.mdx | 4 + .../docs/commands/plugin/runtime/register.mdx | 4 + .../docs/plugins/containerized-plugins.mdx | 38 ++--- 22 files changed, 232 insertions(+), 141 deletions(-) create mode 100644 changelog/24236.txt diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 2b498be4d011..6fcefe765992 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -286,14 +286,37 @@ jobs: "runsc": { "path": "/usr/local/bin/runsc", "runtimeArgs": [ - "--host-uds=all", - "--host-fifo=open" + "--host-uds=create" ] } } } EOF sudo systemctl reload docker + - name: Install rootless Docker + # Enterprise repo runners do not allow sudo, so can't system packages there yet. + if: ${{ !inputs.enterprise }} + run: | + sudo apt-get install -y uidmap dbus-user-session + export FORCE_ROOTLESS_INSTALL=1 + curl -fsSL https://get.docker.com/rootless | sh + mkdir -p ~/.config/docker/ + tee ~/.config/docker/daemon.json <)` – Part of the request URL. Specifies the plugin runtime name. Use the runtime name to look up plugin runtimes in the catalog. +- `rootless` `(bool: false)` - Whether the container runtime is running as a + non-privileged user. Must be set if plugin container images are also configured + to run as a non-root user. + - `oci_runtime` `(string: )` – Specifies OCI-compliant container runtime to use. Default is "runsc", gVisor's OCI runtime. diff --git a/website/content/docs/commands/plugin/runtime/register.mdx b/website/content/docs/commands/plugin/runtime/register.mdx index 698ee318db6f..75119d4e7268 100644 --- a/website/content/docs/commands/plugin/runtime/register.mdx +++ b/website/content/docs/commands/plugin/runtime/register.mdx @@ -45,6 +45,10 @@ flags](/vault/docs/commands) included on all commands. - `-type` `(string: )` - Plugin runtime type. Vault currently only supports `container` as a runtime type. +- `-rootless` `(bool: false)` - Whether the container runtime is running as a + non-privileged user. Must be set if plugin container images are also configured + to run as a non-root user. + - `-cgroup_parent` `(string: "")` - Parent cgroup to set for each container. Use `cgroup_parent` to control the total resource usage for a group of plugins. diff --git a/website/content/docs/plugins/containerized-plugins.mdx b/website/content/docs/plugins/containerized-plugins.mdx index 1bd4cb2de797..748d6d680923 100644 --- a/website/content/docs/plugins/containerized-plugins.mdx +++ b/website/content/docs/plugins/containerized-plugins.mdx @@ -39,7 +39,7 @@ increases the isolation between plugins, and between plugins and Vault. All plugins have the following basic requirements to be containerized: -- **Your plugin must be built with at least v1.5.0 of the HashiCorp +- **Your plugin must be built with at least v1.6.0 of the HashiCorp [`go-plugin`](https://github.com/hashicorp/go-plugin) library**. - **The image entrypoint should run the plugin binary**. @@ -52,39 +52,39 @@ in [supported configurations](#supported-configurations). Vault's containerized plugins are compatible with a variety of configurations. In particular, it has been tested with the following: -- Docker and Podman. -- Default and rootless container engine. -- OCI runtimes runsc and runc. -- Plugin container images with root and non-root users. +- Default and [rootless](https://docs.docker.com/engine/security/rootless/) Docker. +- OCI-compatible runtimes `runsc` and `runc`. +- Plugin container images running as root and non-root users. - [Mlock](/vault/docs/configuration#disable_mlock) disabled or enabled. Not all combinations work and some have additional requirements, listed below. If you use a configuration that matches multiple headings, you should combine the requirements from each matching heading. -### Rootless installation with non-root container user +### `runsc` runtime -Not currently supported. We are hoping to provide support in future. +- You must pass an additional `--host-uds=create` flag to the `runsc` runtime. -### runsc runtime +### Rootless Docker with `runsc` runtime -- You must pass an additional `--host-uds=all` flag to the `runsc` runtime. +- You must pass an additional `--ignore-cgroups` flag to the `runsc` runtime. + - Cgroup limits are not currently supported for this configuration. -### Rootless installation with `runsc` +### Rootless Docker with non-root container user -- Does not currently support cgroup limits. -- You must pass an additional `--ignore-cgroups` flag to the `runsc` runtime. +- You must use a container plugin runtime with + [`rootless`](/vault/docs/commands/plugin/runtime/register#rootless) enabled. +- Your filesystem must have Posix 1e ACL support, available by default in most + modern Linux file systems. +- Only supported for gVisor's `runsc` runtime. -### Non-root container user with mlock enabled +### Rootless Docker with mlock enabled -- You must set the IPC_LOCK capability on the plugin binary. +- Only supported for gVisor's `runsc` runtime. -### Rootless container engine with mlock enabled +### Non-root container user with mlock enabled -- You must set the IPC_LOCK capability on the container engine's binary. -- You do not need to set the IPC_LOCK capability if running with Docker and runsc. - The `runsc` runtime supports mlock syscalls in rootless Docker without needing - IPC_LOCK itself. +- You must set the `IPC_LOCK` capability on the plugin binary. ## Container lifecycle and metadata From 991dae091ac4e3cc952637f503b5e34dfd71486f Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 22 Nov 2023 13:22:12 +0000 Subject: [PATCH 2/3] Fix shellcheck errors --- .github/workflows/test-go.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 6fcefe765992..24825b5272b6 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -379,8 +379,9 @@ jobs: fi fi - export VAULT_TEST_LOG_DIR=$(pwd)/test-results/go-test/logs-${{ matrix.id }} - mkdir -p $VAULT_TEST_LOG_DIR + VAULT_TEST_LOG_DIR="$(pwd)/test-results/go-test/logs-${{ matrix.id }}" + export VAULT_TEST_LOG_DIR + mkdir -p "$VAULT_TEST_LOG_DIR" # shellcheck disable=SC2086 # can't quote RERUN_FAILS From cc085923938b5bc8129e6bc808c05a4992b55406 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 22 Nov 2023 13:49:00 +0000 Subject: [PATCH 3/3] Fix list response to include rootless --- vault/logical_system.go | 1 + 1 file changed, 1 insertion(+) diff --git a/vault/logical_system.go b/vault/logical_system.go index 3acdda8812f0..ee328453bc24 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -931,6 +931,7 @@ func (b *SystemBackend) handlePluginRuntimeCatalogList(ctx context.Context, _ *l "cgroup_parent": conf.CgroupParent, "cpu_nanos": conf.CPU, "memory_bytes": conf.Memory, + "rootless": conf.Rootless, }) } }