Merge pull request #48 from dalehamel/cos-header-install

feat: Automatically fetch headers for Google's Container OS linux header install via initContainer
iovisor · Mar 10, 2019 · 03c1a99 · 03c1a99
2 parents 0510804 + c934736
commit 03c1a99
Show file tree

Hide file tree

Showing 4 changed files with 222 additions and 7 deletions.
diff --git a/Makefile b/Makefile
@@ -55,6 +55,14 @@ image/build:
 		-f Dockerfile.tracerunner .
 	$(DOCKER) tag $(IMAGE_TRACERUNNER_BRANCH) $(IMAGE_TRACERUNNER_COMMIT)
 
+.PHONY: image/build-init
+image/build-init:
+	$(DOCKER) build \
+		$(IMAGE_BUILD_FLAGS) \
+		-t $(IMAGE_INITCONTAINER_BRANCH) \
+		-f ./init/Dockerfile.initcontainer ./init
+	$(DOCKER) tag $(IMAGE_INITCONTAINER_BRANCH) $(IMAGE_INITCONTAINER_COMMIT)
+
 .PHONY: image/push
 image/push:
 	$(DOCKER) push $(IMAGE_TRACERUNNER_BRANCH)

diff --git a/init/Dockerfile.initcontainer b/init/Dockerfile.initcontainer
@@ -0,0 +1,15 @@
+FROM alpine:3.8
+RUN apk add --update \
+  bash \
+  bc \
+  build-base \
+  curl \
+  libelf-dev \
+  linux-headers \
+  make
+
+WORKDIR /
+
+COPY /fetch-linux-headers.sh /
+
+ENTRYPOINT [ "/fetch-linux-headers.sh" ]
diff --git a/init/fetch-linux-headers.sh b/init/fetch-linux-headers.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -x
+
+LSB_FILE="/etc/lsb-release.host"
+OS_RELEASE_FILE="/etc/os-release.host"
+TARGET_DIR="/usr/src"
+HOST_MODULES_DIR="/lib/modules.host"
+
+generate_headers()
+{
+  echo "Generating kernel headers"
+  cd ${BUILD_DIR}
+  zcat /proc/config.gz > .config
+  make ARCH=x86 oldconfig > /dev/null
+  make ARCH=x86 prepare > /dev/null
+}
+
+fetch_cos_linux_sources()
+{
+  echo "Fetching upstream kernel sources."
+  mkdir -p ${BUILD_DIR}
+  curl -s "https://storage.googleapis.com/cos-tools/${BUILD_ID}/kernel-src.tar.gz" | tar -xzf - -C ${BUILD_DIR}
+}
+
+install_cos_linux_headers()
+{
+  if grep -q CHROMEOS_RELEASE_VERSION ${LSB_FILE};then
+    BUILD_ID=$(grep CHROMEOS_RELEASE_VERSION ${LSB_FILE} | cut -d = -f 2)
+    BUILD_DIR="/linux-lakitu-${BUILD_ID}"
+    SOURCES_DIR="${TARGET_DIR}/linux-lakitu-${BUILD_ID}"
+
+    if [ ! -e "${SOURCES_DIR}/.installed" ];then
+      echo "Installing kernel headers for for COS build ${BUILD_ID}"
+      fetch_cos_linux_sources
+      generate_headers
+      mv ${BUILD_DIR} ${TARGET_DIR}
+      touch "${SOURCES_DIR}/.installed"
+    fi
+  fi
+}
+
+install_headers()
+{
+  distro=$(grep ^NAME ${OS_RELEASE_FILE} | cut -d = -f 2)
+
+  case $distro in
+    *"Container-Optimized OS"*)
+      install_cos_linux_headers
+      HEADERS_TARGET=${SOURCES_DIR}
+      ;;
+    *)
+      echo "WARNING: ${distro} is not a supported distro, cannot install headers, ensure they are installed to /lib/modules"
+  esac
+}
+
+check_headers()
+{
+  modules_path=$1
+  utsname=$(uname -r)
+  arch=$(uname -m)
+  kdir="${modules_path}/${utsname}"
+
+  [ "${arch}" == "x86_64" ] && arch="x86"
+
+  [ ! -e ${kdir} ] && return 1
+  [ ! -e "${kdir}/source" ] && [ ! -e "${kdir}/build" ] && return 1
+
+  header_dir=$([ -e "${kdir}/source" ] && echo "${kdir}/source" || echo "${kdir}/build")
+
+  [ ! -e "${header_dir}/include/linux/kconfig.h" ] && return 1
+  [ ! -e "${header_dir}/include/generated/uapi" ] && return 1
+  [ ! -e "${header_dir}/arch/${arch}/include/generated/uapi" ] && return 1
+
+  return 0
+}
+
+if [ ! -e /lib/modules/.installed ];then
+  if ! check_headers ${HOST_MODULES_DIR}; then
+    install_headers
+  else
+    HEADERS_TARGET=${HOST_MODULES_DIR}/source
+  fi
+
+  mkdir -p "/lib/modules/$(uname -r)"
+  ln -sf ${HEADERS_TARGET} "/lib/modules/$(uname -r)/source"
+  ln -sf ${HEADERS_TARGET} "/lib/modules/$(uname -r)/build"
+  touch /lib/modules/.installed
+  exit 0
+else
+  echo "Headers already installed"
+  exit 0
+fi
diff --git a/pkg/tracejob/job.go b/pkg/tracejob/job.go
@@ -238,7 +238,7 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) {
 							},
 						},
 						apiv1.Volume{
-							Name: "modules",
+							Name: "modules-host",
 							VolumeSource: apiv1.VolumeSource{
 								HostPath: &apiv1.HostPathVolumeSource{
 									Path: "/lib/modules",
@@ -277,11 +277,6 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) {
 									MountPath: "/programs",
 									ReadOnly:  true,
 								},
-								apiv1.VolumeMount{
-									Name:      "modules",
-									MountPath: "/lib/modules",
-									ReadOnly:  true,
-								},
 								apiv1.VolumeMount{
 									Name:      "sys",
 									MountPath: "/sys",
@@ -316,6 +311,110 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) {
 		},
 	}
 
+	if nj.FetchHeaders {
+		// If we aren't downloading headers, add the initContainer and set up mounts
+		job.Spec.Template.Spec.InitContainers = []apiv1.Container{
+			apiv1.Container{
+				Name:  "kubectl-trace-init",
+				Image: nj.InitImageNameTag,
+				Resources: apiv1.ResourceRequirements{
+					Requests: apiv1.ResourceList{
+						apiv1.ResourceCPU:    resource.MustParse("100m"),
+						apiv1.ResourceMemory: resource.MustParse("100Mi"),
+					},
+					Limits: apiv1.ResourceList{
+						apiv1.ResourceCPU:    resource.MustParse("1"),
+						apiv1.ResourceMemory: resource.MustParse("1G"),
+					},
+				},
+				VolumeMounts: []apiv1.VolumeMount{
+					apiv1.VolumeMount{
+						Name:      "lsb-release",
+						MountPath: "/etc/lsb-release.host",
+						ReadOnly:  true,
+					},
+					apiv1.VolumeMount{
+						Name:      "os-release",
+						MountPath: "/etc/os-release.host",
+						ReadOnly:  true,
+					},
+					apiv1.VolumeMount{
+						Name:      "modules-dir",
+						MountPath: "/lib/modules",
+					},
+					apiv1.VolumeMount{
+						Name:      "modules-host",
+						MountPath: "/lib/modules.host",
+						ReadOnly:  true,
+					},
+					apiv1.VolumeMount{
+						Name:      "linux-headers-generated",
+						MountPath: "/usr/src/",
+					},
+				},
+			},
+		}
+
+		job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes,
+			apiv1.Volume{
+				Name: "lsb-release",
+				VolumeSource: apiv1.VolumeSource{
+					HostPath: &apiv1.HostPathVolumeSource{
+						Path: "/etc/lsb-release",
+					},
+				},
+			},
+			apiv1.Volume{
+				Name: "os-release",
+				VolumeSource: apiv1.VolumeSource{
+					HostPath: &apiv1.HostPathVolumeSource{
+						Path: "/etc/os-release",
+					},
+				},
+			},
+			apiv1.Volume{
+				Name: "modules-dir",
+				VolumeSource: apiv1.VolumeSource{
+					HostPath: &apiv1.HostPathVolumeSource{
+						Path: "/var/cache/linux-headers/modules_dir",
+					},
+				},
+			},
+			apiv1.Volume{
+				Name: "linux-headers-generated",
+				VolumeSource: apiv1.VolumeSource{
+					HostPath: &apiv1.HostPathVolumeSource{
+						Path: "/var/cache/linux-headers/generated",
+					},
+				},
+			})
+
+		job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts,
+			apiv1.VolumeMount{
+				Name:      "modules-dir",
+				MountPath: "/lib/modules",
+				ReadOnly:  true,
+			},
+			apiv1.VolumeMount{
+				Name:      "modules-host",
+				MountPath: "/lib/modules.host",
+				ReadOnly:  true,
+			},
+			apiv1.VolumeMount{
+				Name:      "linux-headers-generated",
+				MountPath: "/usr/src/",
+				ReadOnly:  true,
+			})
+
+	} else {
+		// If we aren't downloading headers, unconditionally used the ones linked in /lib/modules
+		job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts,
+			apiv1.VolumeMount{
+				Name:      "modules-host",
+				MountPath: "/lib/modules",
+				ReadOnly:  true,
+			})
+	}
 	if _, err := t.ConfigClient.Create(cm); err != nil {
 		return nil, err
 	}
@@ -393,4 +492,4 @@ func jobStatus(j batchv1.Job) TraceJobStatus {
 		return TraceJobFailed
 	}
 	return TraceJobUnknown
-}
+}