Add basic systemd service to deb and rpm (#1335)

- Install `otel-collector` systemd service and run as `otel:otel` user:group
- Add simple installation test scripts using systemd-in-docker images

.circleci/config.yml

@@ -348,7 +348,8 @@ jobs:
           paths: project/bin
 
   build-package:
-    executor: golang
+    machine:
+      image: ubuntu-1604:202007-01
     parameters:
       package_type:
         type: enum
@@ -361,23 +362,21 @@ jobs:
           command: |
             sudo apt-get update
             sudo apt-get install -y ruby ruby-dev rubygems build-essential rpm
-            sudo gem install --no-document fpm -v 1.11.0
+            gem install --no-document fpm -v 1.11.0
       - run:
           name: Build << parameters.package_type >> amd64 package
           command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "amd64" "./bin/"
       - run:
           name: Build << parameters.package_type >> arm64 package
           command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "arm64" "./bin/"
       - run:
-          name: Test << parameters.package_type >> package installation
+          name: Test << parameters.package_type >> package
           command: |
             if [[ "<< parameters.package_type >>" = "deb" ]]; then
-                sudo dpkg -i bin/otel-collector*amd64.deb
+                ./internal/buildscripts/packaging/fpm/test.sh bin/otel-collector*amd64.deb
             else
-                sudo rpm -ivh bin/otel-collector*x86_64.rpm
+                ./internal/buildscripts/packaging/fpm/test.sh bin/otel-collector*x86_64.rpm
             fi
-            test -f /usr/bin/otelcol || (echo "/usr/bin/otelcol not found!" && exit 1)
-            test -f /etc/otel-collector/config.yaml || (echo "/etc/otel-collector/config.yaml not found" && exit 1)
       - persist_to_workspace:
           root: ~/
           paths: project/bin

internal/buildscripts/packaging/fpm/common.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+FPM_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
+
+PKG_NAME="otel-collector"
+PKG_VENDOR="OpenTelemetry Community"
+PKG_MAINTAINER="OpenTelemetry Community <[email protected]>"
+PKG_DESCRIPTION="OpenTelemetry Collector"
+PKG_LICENSE="Apache 2.0"
+PKG_URL="https://github.com/open-telemetry/opentelemetry-collector"
+PKG_USER="otel"
+PKG_GROUP="otel"
+
+SERVICE_NAME="otel-collector"
+PROCESS_NAME="otelcol"
+
+SERVICE_PATH="$FPM_DIR/$SERVICE_NAME.service"
+PREINSTALL_PATH="$FPM_DIR/preinstall.sh"
+POSTINSTALL_PATH="$FPM_DIR/postinstall.sh"
+PREUNINSTALL_PATH="$FPM_DIR/preuninstall.sh"
+
+install_pkg() {
+    local pkg_path="$1"
+    local pkg_base=$( basename "$pkg_path" )
+
+    echo "Installing $pkg_base ..."
+    docker cp "$pkg_path" $image_name:/tmp/$pkg_base
+    if [[ "${pkg_base##*.}" = "deb" ]]; then
+        $docker_exec dpkg -i /tmp/$pkg_base
+    else
+        $docker_exec rpm -ivh /tmp/$pkg_base
+    fi
+}
+
+uninstall_pkg() {
+    local pkg_type="$1"
+    local pkg_name="${2:-"$PKG_NAME"}"
+
+    echo "Uninstalling $pkg_name ..."
+    if [[ "$pkg_type" = "deb" ]]; then
+        $docker_exec dpkg -r $pkg_name
+    else
+        $docker_exec rpm -e $pkg_name
+    fi
+}

internal/buildscripts/packaging/fpm/deb/Dockerfile.test

@@ -0,0 +1,22 @@
+# A debian9 image with systemd enabled.  Must be run with:
+# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags
+FROM debian:9
+
+RUN apt-get update &&\
+    apt-get install -yq ca-certificates procps systemd
+
+ENV container docker
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
+    "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
+    rm -f /lib/systemd/system/multi-user.target.wants/*;\
+    rm -f /lib/systemd/system/local-fs.target.wants/*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+    rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+RUN systemctl set-default multi-user.target
+ENV init /lib/systemd/systemd
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+ENTRYPOINT ["/lib/systemd/systemd"]

internal/buildscripts/packaging/fpm/deb/README.md

@@ -7,4 +7,7 @@ To build the deb package, run `make deb-package` from the repo root directory. T
 
 By default, `<arch>` is `amd64` and `<version>` is the latest git tag with `-post` appended, e.g. `1.2.3-post`.
 To override these defaults, set the `ARCH` and `VERSION` environment variables, e.g.
-`ARCH=arm64 VERSION=4.5.6 make deb-package`.
+`ARCH=arm64 VERSION=4.5.6 make deb-package`.
+
+Run `./internal/buildscripts/packaging/fpm/test.sh PATH_TO_DEB_FILE` to run a basic installation test with the built
+package.

internal/buildscripts/packaging/fpm/deb/build.sh

@@ -7,22 +7,30 @@ REPO_DIR="$( cd "$SCRIPT_DIR/../../../../../" && pwd )"
 VERSION="${1:-}"
 ARCH="${2:-"amd64"}"
 OUTPUT_DIR="${3:-"$REPO_DIR/bin/"}"
-OTELCOL_PATH="$REPO_DIR/bin/otelcol_linux_amd64"
+OTELCOL_PATH="$REPO_DIR/bin/otelcol_linux_$ARCH"
 CONFIG_PATH="$REPO_DIR/examples/otel-local-config.yaml"
 
+. $SCRIPT_DIR/../common.sh
+
 if [[ -z "$VERSION" ]]; then
     latest_tag="$( git describe --abbrev=0 --match v[0-9]* )"
     VERSION="${latest_tag}-post"
 fi
 
-fpm -s dir -t deb -n otel-collector -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
-    --vendor "OpenTelemetry Community" \
-    --maintainer "OpenTelemetry Community <[email protected]>" \
-    --description "OpenTelemetry Collector" \
-    --license "Apache 2.0" \
-    --url "https://github.com/open-telemetry/opentelemetry-collector" \
+fpm -s dir -t deb -n $PKG_NAME -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
+    --vendor "$PKG_VENDOR" \
+    --maintainer "$PKG_MAINTAINER" \
+    --description "$PKG_DESCRIPTION" \
+    --license "$PKG_LICENSE" \
+    --url "$PKG_URL" \
     --architecture "$ARCH" \
     --config-files /etc/otel-collector/config.yaml \
     --deb-dist "stable" \
-    $OTELCOL_PATH=/usr/bin/otelcol \
+    --deb-user "$PKG_USER" \
+    --deb-group "$PKG_GROUP" \
+    --before-install "$PREINSTALL_PATH" \
+    --after-install "$POSTINSTALL_PATH" \
+    --pre-uninstall "$PREUNINSTALL_PATH" \
+    $OTELCOL_PATH=/usr/bin/$PROCESS_NAME \
+    $SERVICE_PATH=/lib/systemd/system/$SERVICE_NAME.service \
     $CONFIG_PATH=/etc/otel-collector/config.yaml

internal/buildscripts/packaging/fpm/otel-collector.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=OpenTelemety Collector
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/otelcol --config /etc/otel-collector/config.yaml
+KillMode=mixed
+Restart=on-failure
+Type=simple
+User=otel
+Group=otel
+
+[Install]
+WantedBy=multi-user.target

internal/buildscripts/packaging/fpm/postinstall.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if command -v systemctl >/dev/null 2>&1; then
+    systemctl enable otel-collector.service
+    if [ -f /etc/otel-collector/config.yaml ]; then
+        systemctl start otel-collector.service
+    fi
+fi

internal/buildscripts/packaging/fpm/preinstall.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+getent passwd otel >/dev/null || useradd --system --user-group --no-create-home --shell /sbin/nologin otel

internal/buildscripts/packaging/fpm/preuninstall.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if command -v systemctl >/dev/null 2>&1; then
+    systemctl stop otel-collector.service
+    systemctl disable otel-collector.service
+fi

internal/buildscripts/packaging/fpm/rpm/Dockerfile.test

@@ -0,0 +1,20 @@
+# A centos8 image with systemd enabled.  Must be run with:
+# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags
+FROM centos:8
+
+ENV container docker
+
+RUN dnf install -y initscripts
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
+    "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
+    rm -f /lib/systemd/system/multi-user.target.wants/*;\
+    rm -f /lib/systemd/system/local-fs.target.wants/*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+    rm -f /lib/systemd/system/basic.target.wants/*;\
+    rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+CMD ["/usr/sbin/init"]

internal/buildscripts/packaging/fpm/rpm/README.md

@@ -7,4 +7,7 @@ To build the rpm package, run `make rpm-package` from the repo root directory. T
 
 By default, `<arch>` is `amd64` and `<version>` is the latest git tag with `~post` appended, e.g. `1.2.3~post`.
 To override these defaults, set the `ARCH` and `VERSION` environment variables, e.g.
-`ARCH=arm64 VERSION=4.5.6 make rpm-package`.
+`ARCH=arm64 VERSION=4.5.6 make rpm-package`.
+
+Run `./internal/buildscripts/packaging/fpm/test.sh PATH_TO_RPM_FILE` to run a basic installation test with the built
+package.

internal/buildscripts/packaging/fpm/rpm/build.sh

@@ -7,22 +7,30 @@ REPO_DIR="$( cd "$SCRIPT_DIR/../../../../../" && pwd )"
 VERSION="${1:-}"
 ARCH="${2:-"amd64"}"
 OUTPUT_DIR="${3:-"$REPO_DIR/bin/"}"
-OTELCOL_PATH="$REPO_DIR/bin/otelcol_linux_amd64"
+OTELCOL_PATH="$REPO_DIR/bin/otelcol_linux_$ARCH"
 CONFIG_PATH="$REPO_DIR/examples/otel-local-config.yaml"
 
+. $SCRIPT_DIR/../common.sh
+
 if [[ -z "$VERSION" ]]; then
     latest_tag="$( git describe --abbrev=0 --match v[0-9]* )"
     VERSION="${latest_tag}~post"
 fi
 
-fpm -s dir -t rpm -n otel-collector -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
-    --vendor "OpenTelemetry Community" \
-    --maintainer "OpenTelemetry Community <[email protected]>" \
-    --description "OpenTelemetry Collector" \
-    --license "Apache 2.0" \
-    --rpm-summary "OpenTelemetry Collector" \
-    --url "https://github.com/open-telemetry/opentelemetry-collector" \
+fpm -s dir -t rpm -n $PKG_NAME -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
+    --vendor "$PKG_VENDOR" \
+    --maintainer "$PKG_MAINTAINER" \
+    --description "$PKG_DESCRIPTION" \
+    --license "$PKG_LICENSE" \
+    --url "$PKG_URL" \
     --architecture "$ARCH" \
     --config-files /etc/otel-collector/config.yaml \
-    $OTELCOL_PATH=/usr/bin/otelcol \
+    --rpm-summary "$PKG_DESCRIPTION" \
+    --rpm-user "$PKG_USER" \
+    --rpm-group "$PKG_GROUP" \
+    --before-install "$PREINSTALL_PATH" \
+    --after-install "$POSTINSTALL_PATH" \
+    --pre-uninstall "$PREUNINSTALL_PATH" \
+    $OTELCOL_PATH=/usr/bin/$PROCESS_NAME \
+    $SERVICE_PATH=/lib/systemd/system/$SERVICE_NAME.service \
     $CONFIG_PATH=/etc/otel-collector/config.yaml

internal/buildscripts/packaging/fpm/test.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
+REPO_DIR="$( cd "$SCRIPT_DIR/../../../../" && pwd )"
+PKG_PATH=${1:-}
+
+. $SCRIPT_DIR/common.sh
+
+if [[ -z "$PKG_PATH" ]]; then
+    echo "usage: ${BASH_SOURCE[0]} DEB_OR_RPM_PATH" >&2
+    exit 1
+fi
+
+if [[ ! -f "$PKG_PATH" ]]; then
+    echo "$PKG_PATH not found!" >&2
+    exit 1
+fi
+
+pkg_name="$( basename "$PKG_PATH" )"
+pkg_type="${pkg_name##*.}"
+if [[ ! "$pkg_type" =~ ^(deb|rpm)$ ]]; then
+    echo "$PKG_PATH not supported!" >&2
+    exit 1
+fi
+image_name="otelcol-$pkg_type-test"
+container_name="$image_name"
+docker_run="docker run --name $container_name -d -v /sys/fs/cgroup:/sys/fs/cgroup:ro --privileged $image_name"
+docker_exec="docker exec $container_name"
+
+trap "docker rm -fv $container_name >/dev/null 2>&1 || true" EXIT
+
+docker build -t $image_name -f "$SCRIPT_DIR/$pkg_type/Dockerfile.test" "$SCRIPT_DIR"
+docker rm -fv $container_name >/dev/null 2>&1 || true
+
+# test install
+echo
+$docker_run
+install_pkg "$PKG_PATH"
+
+# ensure service has started and still running after 5 seconds
+sleep 5
+echo "Checking $SERVICE_NAME service status ..."
+$docker_exec systemctl --no-pager status $SERVICE_NAME
+
+echo "Checking $PROCESS_NAME process ..."
+$docker_exec pgrep -a -u otel $PROCESS_NAME
+
+# test uninstall
+echo
+uninstall_pkg $pkg_type
+
+echo "Checking $SERVICE_NAME service status after uninstall ..."
+if $docker_exec systemctl --no-pager status $SERVICE_NAME; then
+    echo "$SERVICE_NAME service still running after uninstall!" >&2
+    exit 1
+fi
+echo "$SERVICE_NAME service successfully stopped after uninstall"
+
+echo "Checking $SERVICE_NAME service existence after uninstall ..."
+if $docker_exec systemctl list-unit-files --all | grep $SERVICE_NAME; then
+    echo "$SERVICE_NAME service still exists after uninstall!" >&2
+    exit 1
+fi
+echo "$SERVICE_NAME service successfully removed after uninstall"
+
+echo "Checking $PROCESS_NAME process after uninstall ..."
+if $docker_exec pgrep $PROCESS_NAME; then
+    echo "$PROCESS_NAME process still running after uninstall!" >&2
+    exit 1
+fi
+echo "$PROCESS_NAME process successfully killed after uninstall"