From 58c80247d2493c6901c5be7c27b5e766dd922478 Mon Sep 17 00:00:00 2001
From: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
Date: Thu, 9 Mar 2023 20:06:21 +0900
Subject: [PATCH] rootless: guide for Bottlerocket OS (`sysctl -w
 user.max_user_namespaces=N`)

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
(cherry picked from commit c67176ae94f3e51ae990148a7a04ad10ff6072a3)
Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
---
 docs/rootless.md                              | 11 ++++++++
 .../kubernetes/sysctl-userns.privileged.yaml  | 26 +++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 examples/kubernetes/sysctl-userns.privileged.yaml

diff --git a/docs/rootless.md b/docs/rootless.md
index ee25875e76ee..2dabfbdee806 100644
--- a/docs/rootless.md
+++ b/docs/rootless.md
@@ -24,6 +24,12 @@ spec:
 
 See also the [example manifests](#Kubernetes).
 
+### Bottlerocket OS
+
+Needs to run `sysctl -w user.max_user_namespaces=N` (N=positive integer, like 63359) on the host nodes.
+
+See [`../examples/kubernetes/sysctl-userns.privileged.yaml`](../examples/kubernetes/sysctl-userns.privileged.yaml).
+
 <details>
 <summary>Old distributions</summary>
 
@@ -104,6 +110,11 @@ See https://rootlesscontaine.rs/getting-started/common/subuid/
 ### Error `Options:[rbind ro]}]: operation not permitted`
 Make sure to mount an `emptyDir` volume on `/home/user/.local/share/buildkit` .
 
+### Error `fork/exec /proc/self/exe: no space left on device` with `level=warning msg="/proc/sys/user/max_user_namespaces needs to be set to non-zero."`
+Run `sysctl -w user.max_user_namespaces=N` (N=positive integer, like 63359) on the host nodes.
+
+See [`../examples/kubernetes/sysctl-userns.privileged.yaml`](../examples/kubernetes/sysctl-userns.privileged.yaml).
+
 ## Containerized deployment
 
 ### Kubernetes
diff --git a/examples/kubernetes/sysctl-userns.privileged.yaml b/examples/kubernetes/sysctl-userns.privileged.yaml
new file mode 100644
index 000000000000..1380788a63ee
--- /dev/null
+++ b/examples/kubernetes/sysctl-userns.privileged.yaml
@@ -0,0 +1,26 @@
+# Run `sysctl -w user.max_user_namespaces=63359` on all the nodes,
+# for errors like "/proc/sys/user/max_user_namespaces needs to be set to non-zero"
+# on running rootless buildkitd pods.
+#
+# This workaround is known to be needed on Bottlerocket OS.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: sysctl-userns
+  name: sysctl-userns
+spec:
+  selector:
+    matchLabels:
+      app: sysctl-userns
+  template:
+    metadata:
+      labels:
+        app: sysctl-userns
+    spec:
+      containers:
+        - name: sysctl-userns
+          image: busybox
+          command: ["sh", "-euxc", "sysctl -w user.max_user_namespaces=63359 && sleep infinity"]
+          securityContext:
+            privileged: true