Container OS linux header install via initContainer

iovisor · Mar 10, 2019 · d88e0ed · d88e0ed
1 parent 0510804
commit d88e0ed
Show file tree

Hide file tree

Showing 5 changed files with 216 additions and 4 deletions.
diff --git a/Makefile b/Makefile
@@ -55,6 +55,11 @@ image/build:
 		-f Dockerfile.tracerunner .
 	$(DOCKER) tag $(IMAGE_TRACERUNNER_BRANCH) $(IMAGE_TRACERUNNER_COMMIT)
 
+.PHONY: image/build-init
+image/build-init:
+	$(DOCKER) build \
+		-f ./init/Dockerfile.initcontainer ./init
+
 .PHONY: image/push
 image/push:
 	$(DOCKER) push $(IMAGE_TRACERUNNER_BRANCH)

diff --git a/init/Dockerfile.initcontainer b/init/Dockerfile.initcontainer
@@ -0,0 +1,15 @@
+FROM alpine:3.8
+RUN apk add --update \
+  bash \
+  bc \
+  build-base \
+  curl \
+  libelf-dev \
+  linux-headers \
+  make
+
+WORKDIR /
+
+COPY /fetch-linux-headers.sh /
+
+ENTRYPOINT [ "/fetch-linux-headers.sh" ]
diff --git a/init/fetch-linux-headers.sh b/init/fetch-linux-headers.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -x
+
+LSB_FILE="/etc/lsb-release.host"
+OS_RELEASE_FILE="/etc/os-release.host"
+TARGET_DIR="/usr/src"
+HOST_MODULES_DIR="/lib/modules.host"
+
+generate_headers()
+{
+  echo "Generating kernel headers"
+  cd ${BUILD_DIR}
+  zcat /proc/config.gz > .config
+  make ARCH=x86 oldconfig > /dev/null
+  make ARCH=x86 prepare > /dev/null
+}
+
+fetch_cos_linux_sources()
+{
+  echo "Fetching upstream kernel sources."
+  mkdir -p ${BUILD_DIR}
+  curl -s "https://storage.googleapis.com/cos-tools/${BUILD_ID}/kernel-src.tar.gz" | tar -xzf - -C ${BUILD_DIR}
+}
+
+install_cos_linux_headers()
+{
+  if grep -q CHROMEOS_RELEASE_VERSION ${LSB_FILE};then
+    BUILD_ID=$(grep CHROMEOS_RELEASE_VERSION ${LSB_FILE} | cut -d = -f 2)
+    BUILD_DIR="/linux-lakitu-${BUILD_ID}"
+    SOURCES_DIR="${TARGET_DIR}/linux-lakitu-${BUILD_ID}"
+
+    if [ ! -e "${SOURCES_DIR}/.installed" ];then
+      echo "Installing kernel headers for for COS build ${BUILD_ID}"
+      fetch_cos_linux_sources
+      generate_headers
+      mv ${BUILD_DIR} ${TARGET_DIR}
+      touch "${SOURCES_DIR}/.installed"
+    fi
+  fi
+}
+
+install_headers()
+{
+  distro=$(grep ^NAME ${OS_RELEASE_FILE} | cut -d = -f 2)
+
+  case $distro in
+    *"Container-Optimized OS"*)
+      install_cos_linux_headers
+      HEADERS_TARGET=${SOURCES_DIR}
+      ;;
+    *)
+      echo "WARNING: ${distro} is not a supported distro, cannot install headers, ensure they are installed to /lib/modules"
+  esac
+}
+
+check_headers()
+{
+  modules_path=$1
+  utsname=$(uname -r)
+  arch=$(uname -m)
+  kdir="${modules_path}/${utsname}"
+
+  [ "${arch}" == "x86_64" ] && arch="x86"
+
+  [ ! -e ${kdir} ] && return 1
+  [ ! -e "${kdir}/source" ] && [ ! -e "${kdir}/build" ] && return 1
+
+  header_dir=$([ -e "${kdir}/source" ] && echo "${kdir}/source" || echo "${kdir}/build")
+
+  [ ! -e "${header_dir}/include/linux/kconfig.h" ] && return 1
+  [ ! -e "${header_dir}/include/generated/uapi" ] && return 1
+  [ ! -e "${header_dir}/arch/${arch}/include/generated/uapi" ] && return 1
+
+  return 0
+}
+
+if [ ! -e /lib/modules/.installed ];then
+  if ! check_headers ${HOST_MODULES_DIR}; then
+    install_headers
+  else
+    HEADERS_TARGET=${HOST_MODULES_DIR}/source
+  fi
+
+  mkdir -p "/lib/modules/$(uname -r)"
+  ln -sf ${HEADERS_TARGET} "/lib/modules/$(uname -r)/source"
+  ln -sf ${HEADERS_TARGET} "/lib/modules/$(uname -r)/build"
+  touch /lib/modules/.installed
+  exit 0
+else
+  echo "Headers already installed"
+  exit 0
+fi
diff --git a/pkg/tracejob/job.go b/pkg/tracejob/job.go
@@ -238,7 +238,7 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) {
 							},
 						},
 						apiv1.Volume{
-							Name: "modules",
+							Name: "modules-host",
 							VolumeSource: apiv1.VolumeSource{
 								HostPath: &apiv1.HostPathVolumeSource{
 									Path: "/lib/modules",
@@ -253,6 +253,69 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) {
 								},
 							},
 						},
+						apiv1.Volume{
+							Name: "lsb-release",
+							VolumeSource: apiv1.VolumeSource{
+								HostPath: &apiv1.HostPathVolumeSource{
+									Path: "/etc/lsb-release",
+								},
+							},
+						},
+						apiv1.Volume{
+							Name: "os-release",
+							VolumeSource: apiv1.VolumeSource{
+								HostPath: &apiv1.HostPathVolumeSource{
+									Path: "/etc/os-release",
+								},
+							},
+						},
+						apiv1.Volume{
+							Name: "modules-dir",
+							VolumeSource: apiv1.VolumeSource{
+								HostPath: &apiv1.HostPathVolumeSource{
+									Path: "/var/cache/linux-headers/modules_dir",
+								},
+							},
+						},
+						apiv1.Volume{
+							Name: "linux-headers-generated",
+							VolumeSource: apiv1.VolumeSource{
+								HostPath: &apiv1.HostPathVolumeSource{
+									Path: "/var/cache/linux-headers/generated",
+								},
+							},
+						},
+					},
+					InitContainers: []apiv1.Container{
+						apiv1.Container{
+							Name:    "kubectl-trace-init",
+							Image:   version.InitImageNameTag(),
+							VolumeMounts: []apiv1.VolumeMount{
+								apiv1.VolumeMount{
+									Name:      "lsb-release",
+									MountPath: "/etc/lsb-release.host",
+									ReadOnly:  true,
+								},
+								apiv1.VolumeMount{
+									Name:      "os-release",
+									MountPath: "/etc/os-release.host",
+									ReadOnly:  true,
+								},
+								apiv1.VolumeMount{
+									Name:      "modules-dir",
+									MountPath: "/lib/modules",
+								},
+								apiv1.VolumeMount{
+									Name:      "modules-host",
+									MountPath: "/lib/modules.host",
+									ReadOnly:  true,
+								},
+								apiv1.VolumeMount{
+									Name:      "linux-headers-generated",
+									MountPath: "/usr/src/",
+								},
+							},
+						},
 					},
 					Containers: []apiv1.Container{
 						apiv1.Container{
@@ -278,13 +341,23 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) {
 									ReadOnly:  true,
 								},
 								apiv1.VolumeMount{
-									Name:      "modules",
+									Name:      "sys",
+									MountPath: "/sys",
+									ReadOnly:  true,
+								},
+								apiv1.VolumeMount{
+									Name:      "modules-dir",
 									MountPath: "/lib/modules",
 									ReadOnly:  true,
 								},
 								apiv1.VolumeMount{
-									Name:      "sys",
-									MountPath: "/sys",
+									Name:      "modules-host",
+									MountPath: "/lib/modules.host",
+									ReadOnly:  true,
+								},
+								apiv1.VolumeMount{
+									Name:      "linux-headers-generated",
+									MountPath: "/usr/src/",
 									ReadOnly:  true,
 								},
 							},

diff --git a/pkg/version/version.go b/pkg/version/version.go
@@ -10,12 +10,38 @@ import (
 var gitCommit string
 var buildTime string
 var versionFormat = "git commit: %s\nbuild date: %s"
+var imageNameTagFormat = "%s:%s"
+var defaultImageName = "quay.io/fntlnz/kubectl-trace-bpftrace"
+var defaultImageTag = "latest"
+var defaultInitImageName = "quay.io/dalehamel/kubectl-trace-init"
+var defaultInitImageTag = "latest"
+
+// ImageName returns the container image name defined in Makefile
+func ImageName() string {
+	return imageName
+}
 
 // GitCommit returns the git commit
 func GitCommit() string {
 	return gitCommit
 }
 
+func ImageNameTag() string {
+	imageName := ImageName()
+	tag := GitCommit()
+	if len(tag) == 0 {
+		tag = defaultImageTag
+	}
+	if len(imageName) == 0 {
+		imageName = defaultImageName
+	}
+	return fmt.Sprintf(imageNameTagFormat, imageName, tag)
+}
+
+func InitImageNameTag() string {
+	return fmt.Sprintf(imageNameTagFormat, defaultInitImageName, defaultInitImageTag)
+}
+
 // Time returns the build time
 func Time() *time.Time {
 	if len(buildTime) == 0 {