Merge pull request #1179 from amazeeio/elastic-7

publish elastic toolkit versions 7, still use 6 inside lagoon for now.

Makefile

@@ -85,6 +85,8 @@ docker_build = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(
 # 3. Path of Docker Build context
 docker_build_python = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(LAGOON_VERSION) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) --build-arg PYTHON_VERSION=$(1) -t $(CI_BUILD_TAG)/python:$(2) -f $(3) $(4)
 
+docker_build_elastic = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(LAGOON_VERSION) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) -t $(CI_BUILD_TAG)/$(2):$(1) -f $(3) $(4)
+
 # Build a PHP docker image. Expects as arguments:
 # 1. PHP version
 # 2. PHP version and type of image (ie 7.0-fpm, 7.0-cli etc)
@@ -129,9 +131,6 @@ images :=     oc \
 							rabbitmq \
 							rabbitmq-cluster \
 							mongo \
-							elasticsearch \
-							kibana \
-							logstash \
 							athenapdf-service \
 							curator \
 							docker-host
@@ -177,16 +176,41 @@ build/redis-persistent: build/redis images/redis-persistent/Dockerfile
 build/rabbitmq: build/commons images/rabbitmq/Dockerfile
 build/rabbitmq-cluster: build/rabbitmq images/rabbitmq-cluster/Dockerfile
 build/mongo: build/commons images/mongo/Dockerfile
-build/elasticsearch: build/commons images/elasticsearch/Dockerfile
-build/logstash: build/commons images/logstash/Dockerfile
-build/kibana: build/commons images/kibana/Dockerfile
 build/docker-host: build/commons images/docker-host/Dockerfile
 build/oc: build/commons images/oc/Dockerfile
 build/curator: build/commons images/curator/Dockerfile
 build/oc-build-deploy-dind: build/oc images/oc-build-deploy-dind
 build/athenapdf-service: images/athenapdf-service/Dockerfile
 
 
+#######
+####### Elastic Images
+#######
+
+elasticimages :=  elasticsearch__6 \
+								  elasticsearch__7 \
+									kibana__6 \
+									kibana__7 \
+									logstash__6 \
+									logstash__7
+
+build-elasticimages = $(foreach image,$(elasticimages),build/$(image))
+
+# Define the make recepie for all base images
+$(build-elasticimages): build/commons
+	$(eval clean = $(subst build/,,$@))
+	$(eval tool = $(word 1,$(subst __, ,$(clean))))
+	$(eval version = $(word 2,$(subst __, ,$(clean))))
+# Call the docker build
+	$(call docker_build_elastic,$(version),$(tool),images/$(tool)/Dockerfile$(version),images/$(tool))
+# Touch an empty file which make itself is using to understand when the image has been last build
+	touch $@
+
+base-images-with-versions += $(elasticimages)
+s3-images += elasticimages
+
+build/elasticsearch__6 build/elasticsearch__7 build/kibana__6 build/kibana__7 build/logstash__6 build/logstash__7: images/commons
+
 #######
 ####### Python Images
 #######
@@ -420,9 +444,9 @@ $(build-services-galera):
 
 # Dependencies of Service Images
 build/auth-server build/logs2slack build/logs2rocketchat build/openshiftbuilddeploy build/openshiftbuilddeploymonitor build/openshiftjobs build/openshiftjobsmonitor build/openshiftmisc build/openshiftremove build/rest2tasks build/webhook-handler build/webhooks2tasks build/api build/cli build/ui: build/yarn-workspace-builder
-build/logs2logs-db: build/logstash
-build/logs-db: build/elasticsearch
-build/logs-db-ui: build/kibana
+build/logs2logs-db: build/logstash__6
+build/logs-db: build/elasticsearch__6
+build/logs-db-ui: build/kibana__6
 build/logs-db-curator: build/curator
 build/auto-idler: build/oc
 build/storage-calculator: build/oc

images/elasticsearch/Dockerfile6

@@ -0,0 +1,54 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/commons as commons
+FROM docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+
+LABEL maintainer="amazee.io"
+ENV LAGOON=elasticsearch
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
+COPY --from=commons /home /home
+
+RUN curl -sL https://github.com/krallin/tini/releases/download/v0.18.0/tini -o /sbin/tini && chmod a+x /sbin/tini
+
+COPY peer-finder /bin/
+COPY configure-es.sh /lagoon/
+COPY docker-entrypoint.sh /lagoon/entrypoints/90-elasticsearch.sh
+
+RUN chmod g+w /etc/passwd \
+    && mkdir -p /home
+
+# Reproduce behavior of Alpine: Run Bash as sh
+RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+RUN sed -i 's/discovery.zen.minimum_master_nodes: 1//' config/elasticsearch.yml
+
+RUN echo $'xpack.security.enabled: false\n\
+\n\
+node.name: "${HOSTNAME}"\n\
+node.master: "${NODE_MASTER}"\n\
+discovery.zen.minimum_master_nodes: "${DISCOVERY_ZEN_MINIMUM_MASTER_NODES}"' >> config/elasticsearch.yml
+
+RUN fix-permissions config
+
+ENV ES_JAVA_OPTS="-Xms200m -Xmx200m" \
+    DISCOVERY_ZEN_MINIMUM_MASTER_NODES=1 \
+    NODE_MASTER=true
+
+VOLUME [ "/usr/share/elasticsearch/data" ]
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.bash"]
+
+CMD ["/usr/local/bin/docker-entrypoint.sh"]

images/elasticsearch/Dockerfile → images/elasticsearch/Dockerfile7

@@ -1,6 +1,6 @@
 ARG IMAGE_REPO
 FROM ${IMAGE_REPO:-lagoon}/commons as commons
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.6.1
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.3.0
 
 LABEL maintainer="amazee.io"
 ENV LAGOON=elasticsearch

images/kibana/Dockerfile → images/kibana/Dockerfile6

@@ -1,6 +1,6 @@
 ARG IMAGE_REPO
 FROM ${IMAGE_REPO:-lagoon}/commons as commons
-FROM docker.elastic.co/kibana/kibana:6.6.1
+FROM docker.elastic.co/kibana/kibana:6.8.2
 
 LABEL maintainer="amazee.io"
 ENV LAGOON=kibana

images/kibana/Dockerfile7

@@ -0,0 +1,42 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/commons as commons
+FROM docker.elastic.co/kibana/kibana:7.3.0
+
+LABEL maintainer="amazee.io"
+ENV LAGOON=kibana
+
+USER root
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
+COPY --from=commons /home /home
+
+RUN curl -sL https://github.com/krallin/tini/releases/download/v0.18.0/tini -o /sbin/tini && chmod a+x /sbin/tini
+
+RUN chmod g+w /etc/passwd \
+    && mkdir -p /home
+
+# Reproduce behavior of Alpine: Run Bash as sh
+RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+RUN fix-permissions /usr/share/kibana
+
+# tells the local development environment on which port we are running
+# ENV LAGOON_LOCALDEV_HTTP_PORT=5601
+
+ENV NODE_OPTIONS="--max-old-space-size=200"
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.bash"]
+CMD ["/bin/bash", "/usr/local/bin/kibana-docker"]

images/logstash/Dockerfile6

@@ -0,0 +1,41 @@
+
+
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/commons as commons
+FROM docker.elastic.co/logstash/logstash:6.8.2
+
+LABEL maintainer="amazee.io"
+ENV LAGOON=logstash
+
+USER root
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
+COPY --from=commons /home /home
+
+RUN curl -sL https://github.com/krallin/tini/releases/download/v0.18.0/tini -o /sbin/tini && chmod a+x /sbin/tini
+
+RUN chmod g+w /etc/passwd \
+    && mkdir -p /home
+
+# Reproduce behavior of Alpine: Run Bash as sh
+RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+RUN fix-permissions /usr/share/logstash/data \
+    && fix-permissions /usr/share/logstash/config
+
+ENV LS_JAVA_OPTS "-Xms400m -Xmx400m"
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.bash", "/usr/local/bin/docker-entrypoint"]

images/logstash/Dockerfile → images/logstash/Dockerfile7

@@ -1,6 +1,6 @@
 ARG IMAGE_REPO
 FROM ${IMAGE_REPO:-lagoon}/commons as commons
-FROM docker.elastic.co/logstash/logstash:6.6.1
+FROM docker.elastic.co/logstash/logstash:7.3.0
 
 LABEL maintainer="amazee.io"
 ENV LAGOON=logstash
@@ -34,6 +34,6 @@ ENV TMPDIR=/tmp \
 RUN fix-permissions /usr/share/logstash/data \
     && fix-permissions /usr/share/logstash/config
 
-ENV LS_JAVA_OPTS "-Xms200m -Xmx200m"
+ENV LS_JAVA_OPTS "-Xms400m -Xmx400m"
 
 ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.bash", "/usr/local/bin/docker-entrypoint"]

services/logs-db-ui/Dockerfile

@@ -1,5 +1,5 @@
 ARG IMAGE_REPO
-FROM ${IMAGE_REPO:-lagoon}/kibana
+FROM ${IMAGE_REPO:-lagoon}/kibana:6
 
 ENV NODE_OPTIONS="--max-old-space-size=2048" \
     LOGSDB_KIBANASERVER_PASSWORD=kibanaserver \
@@ -9,7 +9,8 @@ ENV NODE_OPTIONS="--max-old-space-size=2048" \
     KEYCLOAK_ADMIN_USER=admin \
     KEYCLOAK_ADMIN_PASSWORD=admin
 
-RUN echo $'xpack.security.enabled: false\n\
+RUN echo $'\n\
+xpack.security.enabled: false\n\
 xpack.reporting.encryptionKey: "${LOGSDB_UI_XPACK_REPORTING_ENCRYPTIONKEY:-lagoonrocksyou}"\n\
 \n\
 # Configure the Kibana internal server user\n\
@@ -51,7 +52,7 @@ searchguard.cookie.password: "${SEARCHGUARD_COOKIE_PASSWORD:-aaaaaaaaaaaaaaaaaaa
 xpack.spaces.enabled: false\n\
 ' >> config/kibana.yml
 
-RUN bin/kibana-plugin install https://search.maven.org/remotecontent?filepath=com/floragunn/search-guard-kibana-plugin/6.6.1-18.1/search-guard-kibana-plugin-6.6.1-18.1.zip
+RUN bin/kibana-plugin install https://search.maven.org/remotecontent?filepath=com/floragunn/search-guard-kibana-plugin/6.8.2-18.5/search-guard-kibana-plugin-6.8.2-18.5.zip
 
 COPY entrypoints/80-keycloak-url.bash /lagoon/entrypoints/
 COPY entrypoints/81-logs-db-ui-url.bash /lagoon/entrypoints/

services/logs-db/Dockerfile

@@ -1,7 +1,7 @@
 ARG IMAGE_REPO
-FROM ${IMAGE_REPO:-lagoon}/elasticsearch
+FROM ${IMAGE_REPO:-lagoon}/elasticsearch:6
 
-RUN bin/elasticsearch-plugin install -b com.floragunn:search-guard-6:6.6.1-24.1 \
+RUN bin/elasticsearch-plugin install -b com.floragunn:search-guard-6:6.8.2-25.4 \
     && chmod a+x plugins/search-guard-6/tools/install_demo_configuration.sh \
     && plugins/search-guard-6/tools/install_demo_configuration.sh -y \
     && sed -i 's/searchguard.ssl.http.*//' config/elasticsearch.yml \

services/logs2logs-db/Dockerfile

@@ -1,5 +1,5 @@
 ARG IMAGE_REPO
-FROM ${IMAGE_REPO:-lagoon}/logstash
+FROM ${IMAGE_REPO:-lagoon}/logstash:6
 
 RUN bin/logstash-plugin install logstash-input-lumberjack
 

tests/files/elasticsearch-cluster/elasticsearch.dockerfile

@@ -1,2 +1,2 @@
 ARG IMAGE_REPO
-FROM ${IMAGE_REPO:-amazeeio}/elasticsearch
+FROM ${IMAGE_REPO:-amazeeio}/elasticsearch:6

tests/files/elasticsearch/elasticsearch.dockerfile

@@ -1,2 +1,2 @@
 ARG IMAGE_REPO
-FROM ${IMAGE_REPO:-amazeeio}/elasticsearch
+FROM ${IMAGE_REPO:-amazeeio}/elasticsearch:6