Initial deb and rpm packaging

.circleci/config.yml

@@ -23,6 +23,33 @@ executors:
       image: ubuntu-1604:201903-01
 
 commands:
+  verify_dist_files_exist:
+    parameters:
+      files:
+        type: string
+        default: |
+            bin/otelcontribcol_darwin_amd64
+            bin/otelcontribcol_linux_arm64
+            bin/otelcontribcol_linux_amd64
+            bin/otelcontribcol_windows_amd64.exe
+            dist/otel-contrib-collector-*.arm64.rpm
+            dist/otel-contrib-collector_*_amd64.deb
+            dist/otel-contrib-collector-*.x86_64.rpm
+            dist/otel-contrib-collector_*_arm64.deb
+            dist/opentelemetry-contrib-collector.msi
+    steps:
+      - run:
+          name: Check if files exist
+          command: |
+            files="<< parameters.files >>"
+            for f in $files; do
+              if [[ ! -f $f ]]
+              then
+                  echo "$f does not exist."
+                  exit 1
+              fi
+            done
+
   setup:
     steps:
       - checkout
@@ -160,13 +187,24 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - publish-check:
+          requires:
+            - lint
+            - unit-tests
+            - integration-tests
+            - cross-compile
+            - windows-msi
+            - deb-package
+            - rpm-package
       - publish-stable:
           requires:
             - lint
             - unit-tests
             - integration-tests
             - cross-compile
             - windows-msi
+            - deb-package
+            - rpm-package
           filters:
             branches:
               ignore: /.*/
@@ -187,6 +225,22 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - build-package:
+          name: deb-package
+          package_type: deb
+          requires:
+            - cross-compile
+          filters:
+            tags:
+              only: /^v([0-9])+.([0-9])+.([0-9])+.*/
+      - build-package:
+          name: rpm-package
+          package_type: rpm
+          requires:
+            - cross-compile
+          filters:
+            tags:
+              only: /^v([0-9])+.([0-9])+.([0-9])+.*/
 
 jobs:
   setup:
@@ -261,6 +315,8 @@ jobs:
       shell: powershell.exe
     steps:
       - attach_to_workspace
+      - run:
+          command: mkdir -p dist
       - run:
           name: Install Wix Toolset
           command: .\internal\buildscripts\packaging\msi\make.ps1 Install-Tools
@@ -274,28 +330,51 @@ jobs:
           command: .\internal\buildscripts\packaging\msi\make.ps1 Confirm-MSI
       - persist_to_workspace:
           root: ~/
-          paths: project/bin/*.msi
+          paths: project/dist/*.msi
+
+  publish-check:
+    docker:
+      - image: cimg/go:1.14
+    steps:
+      - attach_to_workspace
+      - setup_remote_docker
+      - verify_dist_files_exist
+      - run:
+          name: Check passed
+          command: echo "publish check passed meaning release CI jobs should work as expected"
+          when: on_success
+      - run:
+          name: Check failed
+          command: echo "publish check failed. This means release CI jobs will likely fail as well"
+          when: on_fail
 
   publish-stable:
     docker:
       - image: cimg/go:1.14
     steps:
       - restore_workspace
+      - verify_dist_files_exist
       - setup_remote_docker
       - publish_docker_images:
           repo: opentelemetry-collector-contrib
           tag: ${CIRCLE_TAG:1}
+      - run:
+          name: Prepare release artifacts
+          command: |
+            cp bin/* dist/
+            mv dist/opentelemetry-contrib-collector.msi dist/otel-contrib-collector.msi
       - run:
           name: Calculate checksums
-          command: cd bin && shasum -a 256 * > checksums.txt
+          command: cd dist && shasum -a 256 * > checksums.txt
       - run:
           name: Create Github release and upload artifacts
-          command: ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME --replace $CIRCLE_TAG bin/
+          command: ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME --replace $CIRCLE_TAG dist/
 
   publish-dev:
     executor: golang
     steps:
       - restore_workspace
+      - verify_dist_files_exist
       - setup_remote_docker
       - publish_docker_images:
           repo: opentelemetry-collector-contrib-dev
@@ -383,3 +462,37 @@ jobs:
           path: test-results/junit
       - store_artifacts:
           path: test-results
+
+  build-package:
+    machine:
+      image: ubuntu-1604:202007-01
+    parameters:
+      package_type:
+        type: enum
+        enum: ["deb", "rpm"]
+    steps:
+      - checkout
+      - attach_to_workspace
+      - run:
+          name: Install fpm and dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y ruby ruby-dev rubygems build-essential rpm
+            gem install --no-document fpm -v 1.11.0
+      - run:
+          name: Build << parameters.package_type >> amd64 package
+          command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "amd64" "./dist/"
+      - run:
+          name: Build << parameters.package_type >> arm64 package
+          command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "arm64" "./dist/"
+      - run:
+          name: Test << parameters.package_type >> package installation
+          command: |
+            if [[ "<< parameters.package_type >>" = "deb" ]]; then
+                ./internal/buildscripts/packaging/fpm/test.sh dist/otel-contrib-collector*amd64.deb examples/tracing/otel-collector-config.yml
+            else
+                ./internal/buildscripts/packaging/fpm/test.sh dist/otel-contrib-collector*x86_64.rpm examples/tracing/otel-collector-config.yml
+            fi
+      - persist_to_workspace:
+          root: ~/
+          paths: project/dist/*.<< parameters.package_type >>

.gitignore

@@ -14,6 +14,7 @@ local/
 *.so
 *.dylib
 bin/
+dist/
 
 # Emacs
 *~

CHANGELOG.md

@@ -22,7 +22,7 @@ The OpenTelemetry Collector Contrib contains everything in the [opentelemetry-co
 
 - `awsxray` exporter: Use `peer.service` as segment name when set. (#385)
 - `splunk` exporter: Add trace exports support (#359, #399)
-- Build and publish Windows MSI (#408)
+- Build and publish Windows MSI (#408) and DEB/RPM Linux packages (#405)
 
 ## 🧰 Bug fixes 🧰
 

Makefile

@@ -176,3 +176,10 @@ otel-from-lib:
 build-examples:
 	docker-compose -f examples/tracing/docker-compose.yml build
 	docker-compose -f exporter/splunkhecexporter/example/docker-compose.yml build
+
+.PHONY: deb-rpm-package
+%-package: ARCH ?= amd64
+%-package:
+	$(MAKE) otelcontribcol-linux_$(ARCH)
+	docker build -t otelcontribcol-fpm internal/buildscripts/packaging/fpm
+	docker run --rm -v $(CURDIR):/repo -e PACKAGE=$* -e VERSION=$(VERSION) -e ARCH=$(ARCH) otelcontribcol-fpm

internal/buildscripts/packaging/fpm/Dockerfile

@@ -0,0 +1,16 @@
+FROM debian:9
+
+RUN apt-get update && \
+    apt-get install -y ruby ruby-dev rubygems build-essential git rpm
+
+RUN gem install --no-document fpm -v 1.11.0
+
+VOLUME /repo
+WORKDIR /repo
+
+ENV PACKAGE="deb"
+ENV VERSION=""
+ENV ARCH="amd64"
+ENV OUTPUT_DIR="/repo/dist/"
+
+CMD ./internal/buildscripts/packaging/fpm/$PACKAGE/build.sh "$VERSION" "$ARCH" "$OUTPUT_DIR"

internal/buildscripts/packaging/fpm/common.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+FPM_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
+
+PKG_NAME="otel-contrib-collector"
+PKG_VENDOR="OpenTelemetry Community"
+PKG_MAINTAINER="OpenTelemetry Community <[email protected]>"
+PKG_DESCRIPTION="OpenTelemetry Contrib Collector"
+PKG_LICENSE="Apache 2.0"
+PKG_URL="https://github.com/open-telemetry/opentelemetry-collector-contrib"
+PKG_USER="otel"
+PKG_GROUP="otel"
+
+SERVICE_NAME="otel-contrib-collector"
+PROCESS_NAME="otelcontribcol"
+
+SERVICE_PATH="$FPM_DIR/$SERVICE_NAME.service"
+PREINSTALL_PATH="$FPM_DIR/preinstall.sh"
+POSTINSTALL_PATH="$FPM_DIR/postinstall.sh"
+PREUNINSTALL_PATH="$FPM_DIR/preuninstall.sh"
+
+docker_cp() {
+    local container="$1"
+    local src="$2"
+    local dest="$3"
+    local dest_dir="$( dirname "$dest" )"
+
+    echo "Copying $src to $container:$dest ..."
+    docker exec $container mkdir -p "$dest_dir"
+    docker cp "$src" $container:"$dest"
+}
+
+install_pkg() {
+    local container="$1"
+    local pkg_path="$2"
+    local pkg_base=$( basename "$pkg_path" )
+
+    echo "Installing $pkg_base ..."
+    docker_cp $container "$pkg_path" /tmp/$pkg_base
+    if [[ "${pkg_base##*.}" = "deb" ]]; then
+        docker exec $container dpkg -i /tmp/$pkg_base
+    else
+        docker exec $container rpm -ivh /tmp/$pkg_base
+    fi
+}
+
+uninstall_pkg() {
+    local container="$1"
+    local pkg_type="$2"
+    local pkg_name="${3:-"$PKG_NAME"}"
+
+    echo "Uninstalling $pkg_name ..."
+    if [[ "$pkg_type" = "deb" ]]; then
+        docker exec $container dpkg -r $pkg_name
+    else
+        docker exec $container rpm -e $pkg_name
+    fi
+}

internal/buildscripts/packaging/fpm/deb/Dockerfile.test

@@ -0,0 +1,22 @@
+# A debian9 image with systemd enabled.  Must be run with:
+# `-d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro` flags
+FROM debian:9
+
+RUN apt-get update &&\
+    apt-get install -yq ca-certificates procps systemd
+
+ENV container docker
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
+    "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
+    rm -f /lib/systemd/system/multi-user.target.wants/*;\
+    rm -f /lib/systemd/system/local-fs.target.wants/*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+    rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+RUN systemctl set-default multi-user.target
+ENV init /lib/systemd/systemd
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+ENTRYPOINT ["/lib/systemd/systemd"]

internal/buildscripts/packaging/fpm/deb/README.md

@@ -0,0 +1,14 @@
+# Build otel-contrib-collector deb package
+
+Build the otel-contrib-collector deb package with [fpm](https://github.com/jordansissel/fpm).
+
+To build the deb package, run `make deb-package` from the repo root directory. The deb package will be written to
+`dist/otel-contrib-collector_<version>_<arch>.deb`.
+
+By default, `<arch>` is `amd64` and `<version>` is the latest git tag with `-post` appended, e.g. `1.2.3-post`.
+To override these defaults, set the `ARCH` and `VERSION` environment variables, e.g.
+`ARCH=arm64 VERSION=4.5.6 make deb-package`.
+
+Run `./internal/buildscripts/packaging/fpm/test.sh PATH_TO_DEB_FILE [PATH_TO_CONFIG_FILE]` to run a basic installation
+test with the built package. `PATH_TO_CONFIG_FILE` defaults to `examples/tracing/otel-collector-config.yml` if one is
+not specified.

internal/buildscripts/packaging/fpm/deb/build.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
+REPO_DIR="$( cd "$SCRIPT_DIR/../../../../../" && pwd )"
+VERSION="${1:-}"
+ARCH="${2:-"amd64"}"
+OUTPUT_DIR="${3:-"$REPO_DIR/dist/"}"
+OTELCONTRIBCOL_PATH="$REPO_DIR/bin/otelcontribcol_linux_$ARCH"
+
+. $SCRIPT_DIR/../common.sh
+
+if [[ -z "$VERSION" ]]; then
+    latest_tag="$( git describe --abbrev=0 --match v[0-9]* )"
+    VERSION="${latest_tag}-post"
+fi
+
+mkdir -p "$OUTPUT_DIR"
+
+fpm -s dir -t deb -n $PKG_NAME -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
+    --vendor "$PKG_VENDOR" \
+    --maintainer "$PKG_MAINTAINER" \
+    --description "$PKG_DESCRIPTION" \
+    --license "$PKG_LICENSE" \
+    --url "$PKG_URL" \
+    --architecture "$ARCH" \
+    --deb-dist "stable" \
+    --deb-user "$PKG_USER" \
+    --deb-group "$PKG_GROUP" \
+    --before-install "$PREINSTALL_PATH" \
+    --after-install "$POSTINSTALL_PATH" \
+    --pre-uninstall "$PREUNINSTALL_PATH" \
+    $SERVICE_PATH=/lib/systemd/system/$SERVICE_NAME.service \
+    $OTELCONTRIBCOL_PATH=/usr/bin/otelcontribcol

internal/buildscripts/packaging/fpm/otel-contrib-collector.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=OpenTelemety Contrib Collector
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/otelcontribcol --config /etc/otel-contrib-collector/config.yaml
+KillMode=mixed
+Restart=on-failure
+Type=simple
+User=otel
+Group=otel
+
+[Install]
+WantedBy=multi-user.target

internal/buildscripts/packaging/fpm/postinstall.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if command -v systemctl >/dev/null 2>&1; then
+    systemctl enable otel-contrib-collector.service
+    if [ -f /etc/otel-contrib-collector/config.yaml ]; then
+        systemctl start otel-contrib-collector.service
+    fi
+fi

internal/buildscripts/packaging/fpm/preinstall.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+getent passwd otel >/dev/null || useradd --system --user-group --no-create-home --shell /sbin/nologin otel