From f87ce34cd6a169b4dd9c19e929daeef2594c8608 Mon Sep 17 00:00:00 2001
From: Jeev B <jeevb@users.noreply.github.com>
Date: Thu, 24 Jun 2021 11:58:00 -0700
Subject: [PATCH] Add hack for cgroup v2 to sandbox entrypoints (#1175)

Signed-off-by: Jeev B <jeev.balakrishnan@freenome.com>
---
 docker/sandbox/Dockerfile                  | 21 +++++++++------------
 docker/sandbox/cgroup-v2-hack.sh           | 15 +++++++++++++++
 docker/sandbox/flyte-entrypoint-default.sh |  5 ++++-
 docker/sandbox/flyte-entrypoint-dind.sh    |  5 ++++-
 4 files changed, 32 insertions(+), 14 deletions(-)
 create mode 100755 docker/sandbox/cgroup-v2-hack.sh

diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile
index 3d64453538..d277bbb6c5 100644
--- a/docker/sandbox/Dockerfile
+++ b/docker/sandbox/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.16.0-alpine3.13 AS go_builder_
+FROM golang:1.16.5-alpine3.13 AS go_builder_
 
 # Install dependencies
 RUN apk add --no-cache build-base git make
@@ -12,31 +12,28 @@ RUN git clone -b ${BUILDKIT_CLI_FOR_KUBECTL_VERSION} --single-branch --depth 1 h
     && make -C ${GOPATH}/src/github.com/vmware-tanzu/buildkit-cli-for-kubectl ${INSTALL_DIR}/linux/kubectl-build BIN_DIR=${INSTALL_DIR} VERSION=${BUILDKIT_CLI_FOR_KUBECTL_VERSION}
 
 
-FROM alpine:3.13.2 AS base_
-
-# Install dependencies
-RUN apk add --no-cache curl
+FROM alpine:3.13.5 AS base_
 
 # Make directory to store artifacts
 RUN mkdir -p /flyteorg/bin /flyteorg/share
 
 # Install k3s
-ARG K3S_VERSION="v1.20.2%2Bk3s1"
+ARG K3S_VERSION="v1.21.1%2Bk3s1"
 RUN wget -q -O /flyteorg/bin/k3s https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s \
     && chmod +x /flyteorg/bin/k3s
 
+# Install flytectl
+RUN wget -q -O - https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | BINDIR=/flyteorg/bin sh -s
+
 # Install buildkit-cli-for-kubectl
 COPY --from=go_builder_ /install/linux/ /flyteorg/bin/
 
-# Install flytectl
-RUN curl https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | sh
-
 # Copy flyte manifest
 COPY deployment/sandbox/flyte_generated.yaml /flyteorg/share/flyte_generated.yaml
 
 # Copy scripts
-RUN cp /bin/flytectl /flyteorg/bin/
-COPY docker/sandbox/kubectl docker/sandbox/wait-for-flyte.sh /flyteorg/bin/
+COPY docker/sandbox/kubectl docker/sandbox/cgroup-v2-hack.sh docker/sandbox/wait-for-flyte.sh /flyteorg/bin/
+
 
 FROM base_ AS default
 
@@ -62,7 +59,7 @@ EXPOSE 30081 30082 30084
 ENTRYPOINT ["tini", "flyte-entrypoint.sh"]
 
 
-FROM docker:20.10.3-dind AS dind
+FROM docker:20.10.7-dind AS dind
 
 # Install dependencies
 RUN apk add --no-cache bash git make tini
diff --git a/docker/sandbox/cgroup-v2-hack.sh b/docker/sandbox/cgroup-v2-hack.sh
new file mode 100755
index 0000000000..1156a13b01
--- /dev/null
+++ b/docker/sandbox/cgroup-v2-hack.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+###############################################################################################################
+# DISCLAIMER                                                                                                  #
+# Copied from https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/hack/dind#L28-L37    #
+# Moby License Apache 2.0: https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/LICENSE #
+###############################################################################################################
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+  # move the processes from the root group to the /init group,
+  # otherwise writing subtree_control fails with EBUSY.
+  mkdir -p /sys/fs/cgroup/init
+  busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+  # enable controllers
+  sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control"
+fi
diff --git a/docker/sandbox/flyte-entrypoint-default.sh b/docker/sandbox/flyte-entrypoint-default.sh
index 5b5c3ceb1d..bfdfdd8b2e 100755
--- a/docker/sandbox/flyte-entrypoint-default.sh
+++ b/docker/sandbox/flyte-entrypoint-default.sh
@@ -1,6 +1,9 @@
 #!/bin/sh
 
-set -e
+set -euo pipefail
+
+# Apply cgroup v2 hack
+cgroup-v2-hack.sh
 
 trap 'pkill -P $$' EXIT
 
diff --git a/docker/sandbox/flyte-entrypoint-dind.sh b/docker/sandbox/flyte-entrypoint-dind.sh
index 9db5613c34..1d8cdee181 100755
--- a/docker/sandbox/flyte-entrypoint-dind.sh
+++ b/docker/sandbox/flyte-entrypoint-dind.sh
@@ -1,6 +1,9 @@
 #!/bin/sh
 
-set -e
+set -euo pipefail
+
+# Apply cgroup v2 hack
+cgroup-v2-hack.sh
 
 trap 'pkill -P $$' EXIT