Scala Stream Collector 0.13.0 images (closes #41)

.travis.yml

@@ -10,7 +10,10 @@ script:
 env:
   matrix:
   - IMG=base
-  - IMG=scala-stream-collector/0.12.0
+  - IMG=scala-stream-collector/0.13.0/google-pubsub
+  - IMG=scala-stream-collector/0.13.0/kafka
+  - IMG=scala-stream-collector/0.13.0/kinesis
+  - IMG=scala-stream-collector/0.13.0/nsq
   - IMG=stream-enrich/0.13.0
   - IMG=s3-loader/0.6.0
   - IMG=elasticsearch-loader/0.10.1
@@ -27,7 +30,7 @@ deploy:
     tags: true
     condition: '"$(.travis/is_release_tag.sh base $TRAVIS_TAG)" == "" && $? == 0 && $IMG = base'
 - provider: script
-  script: ./.travis/deploy.sh scala-stream-collector $TRAVIS_TAG
+  script: ./.travis/all_platforms_deploy.sh scala-stream-collector $TRAVIS_TAG
   skip_cleanup: true
   on:
     tags: true

.travis/all_platforms_deploy.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+project=$1
+tag=$2
+
+slashed="${project}/"
+slashed_len=${#slashed}
+
+cicd=${tag:0:${slashed_len}}
+release=${tag:${slashed_len}}
+
+if [ "${cicd}" == "${slashed}" ]; then
+    if [ "${release}" == "" ]; then
+        echo "Warning! No release specified! Ignoring."
+        exit 2
+    fi
+else
+    echo "Warning! This can't be deployed, there's no ${project} tag! Is the travis condition set?"
+    exit 1
+fi
+
+file="${HOME}/.dockercfg"
+docker_repo="snowplow-docker-registry.bintray.io"
+curl -X GET \
+    -u${BINTRAY_SNOWPLOW_DOCKER_USER}:${BINTRAY_SNOWPLOW_DOCKER_API_KEY} \
+    https://${docker_repo}/v2/auth > $file
+
+cd $project
+if [ -d "${release}" ]; then
+    cd $release
+else
+    echo "Warning! Release ${project} ${release} doesn't have an associated folder"
+    exit 1
+fi
+
+for platform in 'google-pubsub' 'kafka' 'kinesis' 'nsq'; do
+    if [ -d "${platform}" ]; then
+        cd $platform
+    else
+        echo "Warning! Release ${project} ${release} ${platform} doesn't have an associated folder"
+        exit 1
+    fi
+
+    img_tag="snowplow/${project}-${platform}:${release}"
+    final_tag="${docker_repo}/${img_tag}"
+    docker build -t $final_tag .
+    docker push $final_tag
+    cd ..
+done

README.md

@@ -19,8 +19,8 @@ They are published in the [`snowplow-docker-registry.bintray.io`][registry] dock
 You can pull the images from the registry directly:
 
 ```bash
-# Scala Stream Collector image
-docker pull snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector:0.12.0
+# NSQ Scala Stream Collector image, there are others available for Kinesis, Kafka and Google PubSub
+docker pull snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:0.13.0
 
 # Stream Enrich image
 docker pull snowplow-docker-registry.bintray.io/snowplow/stream-enrich:0.13.0
@@ -40,8 +40,8 @@ Alternatively, you can build them yourself:
 # All images are based on the base image
 docker pull snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
 
-# Scala Stream Collector image
-docker build -t snowplow/scala-stream-collector:0.12.0 scala-stream-collector/0.12.0
+# NSQ Scala Stream Collector image, there are others available for Kinesis, Kafka and Google PubSub
+docker build -t snowplow/scala-stream-collector-nsq:0.13.0 scala-stream-collector/0.13.0/nsq
 
 # Stream Enrich image
 docker build -t snowplow/stream-enrich:0.13.0 stream-enrich/0.13.0
@@ -66,11 +66,11 @@ following locations:
 Next, you can run a container for each component by mounting your configuration directory:
 
 ```bash
-# Scala Stream Collector container
+# NSQ Scala Stream Collector container, there are others available for Kinesis, Kafka and Google PubSub
 docker run \
   -v $PWD/scala-stream-collector-config:/snowplow/config \
-  snowplow/scala-stream-collector:0.12.0 \ # if you have built the image
-  # snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector:0.12.0 if you have pulled the image
+  snowplow/scala-stream-collector-nsq:0.13.0 \ # if you have built the image
+  # snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:0.13.0 if you have pulled the image
   --config /snowplow/config/config.hocon
 
 # Stream Enrich

scala-stream-collector/0.13.0/google-pubsub/Dockerfile

@@ -0,0 +1,29 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of the collector to download.
+ENV COLLECTOR_VERSION="0.13.0"
+
+# The targeted platform
+ENV PLATFORM="google_pubsub"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_scala_stream_collector_${PLATFORM}_${COLLECTOR_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Port used by the collector.
+EXPOSE 80
+
+# Defines an entrypoint script delegating the lauching of the collector to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

scala-stream-collector/0.13.0/google-pubsub/docker-entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/dumb-init /bin/sh
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Make sure we run the collector as the snowplow user
+exec su-exec snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-collector-${PLATFORM//_/-}-${COLLECTOR_VERSION}.jar "$@"

scala-stream-collector/0.13.0/kafka/Dockerfile

@@ -0,0 +1,29 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of the collector to download.
+ENV COLLECTOR_VERSION="0.13.0"
+
+# The targeted platform
+ENV PLATFORM="kafka"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_scala_stream_collector_${PLATFORM}_${COLLECTOR_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Port used by the collector.
+EXPOSE 80
+
+# Defines an entrypoint script delegating the lauching of the collector to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

scala-stream-collector/0.13.0/kafka/docker-entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/dumb-init /bin/sh
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Make sure we run the collector as the snowplow user
+exec su-exec snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-collector-${PLATFORM}-${COLLECTOR_VERSION}.jar "$@"

scala-stream-collector/0.13.0/kinesis/Dockerfile

@@ -0,0 +1,29 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of the collector to download.
+ENV COLLECTOR_VERSION="0.13.0"
+
+# The targeted platform
+ENV PLATFORM="kinesis"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_scala_stream_collector_${PLATFORM}_${COLLECTOR_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Port used by the collector.
+EXPOSE 80
+
+# Defines an entrypoint script delegating the lauching of the collector to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

scala-stream-collector/0.13.0/kinesis/docker-entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/dumb-init /bin/sh
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Make sure we run the collector as the snowplow user
+exec su-exec snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-collector-${PLATFORM}-${COLLECTOR_VERSION}.jar "$@"

scala-stream-collector/0.13.0/nsq/Dockerfile

@@ -0,0 +1,29 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of the collector to download.
+ENV COLLECTOR_VERSION="0.13.0"
+
+# The targeted platform
+ENV PLATFORM="nsq"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_scala_stream_collector_${PLATFORM}_${COLLECTOR_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Port used by the collector.
+EXPOSE 80
+
+# Defines an entrypoint script delegating the lauching of the collector to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

scala-stream-collector/0.13.0/nsq/docker-entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/dumb-init /bin/sh
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Make sure we run the collector as the snowplow user
+exec su-exec snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-collector-${PLATFORM}-${COLLECTOR_VERSION}.jar "$@"

scala-stream-collector/README.md

@@ -1,6 +1,13 @@
 # Scala Stream Collector
 
-This folder contains the Docker image for [the Snowplow Scala Stream Collector][ssc]
+This folder contains the Docker images for [the Snowplow Scala Stream Collector][ssc].
+
+There is one image per targeted platform:
+
+- `google-pubsub`
+- `kafka`
+- `kinesis`
+- `nsq`
 
 ## Introduction
 
@@ -27,9 +34,10 @@ Additional JVM options can be set through the `SP_JAVA_OPTS` environment variabl
 Running the container without arguments will print out its usage:
 
 ```bash
-$ docker run snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector:0.x.0
+$ VERSION=0.13.0
+$ docker run snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:${VERSION}
 
-snowplow-stream-collector 0.x.0
+snowplow-stream-collector $VERSION
 Usage: snowplow-stream-collector [options]
 
   --help
@@ -44,7 +52,7 @@ $ docker run \
   -d \
   -v ${PWD}/config:/snowplow/config \
   -p 80:80 \
-  snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector:0.x.0 \
+  snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:${VERSION} \
   --config /snowplow/config/config.hocon
 ```
 
@@ -56,7 +64,7 @@ $ docker run \
   -v ${PWD}/config:/snowplow/config \
   -p 80:80 \
   -e 'SP_JAVA_OPTS=-Xms512m -Xmx512m' \
-  snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector:0.x.0 \
+  snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:${VERSION} \
   --config /snowplow/config/config.hocon
 ```