Merge pull request #2109 from jcantrill/gen_mid_stream

LOG-1281: Add linting and Dockerfile generation from midstream Dockerfile.in
openshift · May 3, 2021 · 53b271b · 53b271b
2 parents 5d3e9c2 + 91524e2
commit 53b271b
Show file tree

Hide file tree

Showing 18 changed files with 741 additions and 151 deletions.
diff --git a/Makefile b/Makefile
@@ -56,3 +56,9 @@ deploy-all-images: deploy-fluentd-image deploy-kibana-image deploy-elasticsearch
 lint:
 	@hack/run-linter
 .PHONY: lint
+
+gen-dockerfiles:
+	@for d in "curator" "elasticsearch" "fluentd" "kibana"; do  \
+		./hack/generate-dockerfile-from-midstream "$$d/Dockerfile.in" > "$$d/Dockerfile" ; \
+	done
+.PHONY: gen-dockerfiles
diff --git a/curator/Dockerfile b/curator/Dockerfile
@@ -1,5 +1,29 @@
+### This is a generated file from Dockerfile.in ###
+
+#@follow_tag(openshift-ose-base:ubi8.python.36)
 FROM registry.ci.openshift.org/ocp/builder:ubi8.python.36
 
+ENV BUILD_VERSION=${CI_CONTAINER_VERSION:-}
+ENV OS_GIT_MAJOR=${CI_X_VERSION:-}
+ENV OS_GIT_MINOR=${CI_Y_VERSION:-}
+ENV OS_GIT_PATCH=${CI_Z_VERSION:-}
+ENV SOURCE_GIT_COMMIT=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT:-}
+ENV SOURCE_GIT_URL=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL:-}
+
+LABEL \
+        License="GPLv2+" \
+        io.k8s.description="Curator elasticsearch container for elasticsearch deletion/archival" \
+        io.k8s.display-name="Curator 5" \
+        io.openshift.tags="logging,elk,elasticsearch,curator" \
+        vendor="Red Hat" \
+        name="openshift/ose-logging-curator5" \
+        com.redhat.component="logging-curator5-container" \
+        io.openshift.maintainer.product="OpenShift Container Platform" \
+        io.openshift.build.commit.id=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT} \
+        io.openshift.build.source-location=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL} \
+        io.openshift.build.commit.url=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL}/commit/${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL} \
+        version=${CI_CONTAINER_VERSION}
+
 MAINTAINER OpenShift Development <[email protected]>
 
 ENV HOME=/opt/app-root/src \
@@ -17,9 +41,10 @@ ENV HOME=/opt/app-root/src \
     CURATOR_VER=5.8.1 \
     container=oci \
     LC_ALL=en_US.UTF-8
+ENV upstream_code=${upstream_code:-"."}
 
 USER 0
-COPY . ${HOME}
+COPY  ${upstream_code}/ ${HOME}
 
 RUN mkdir -p $(dirname "$CURATOR_CONF_LOCATION") && \
     touch ${CURATOR_CONF_LOCATION} && \

diff --git a/curator/Dockerfile.in b/curator/Dockerfile.in
@@ -0,0 +1,75 @@
+## EXCLUDE BEGIN ##
+#@follow_tag(ubi8:8-released)
+FROM ubi8:8.2-347 AS builder
+
+ARG upstream_code
+ENV upstream_code=upstream_code
+ARG upstream_tarball=${upstream_code}.tar.gz
+
+COPY ${CI_ARCHIVE_ORIGIN_AGGREGATED_LOGGING} ${upstream_tarball}
+RUN mkdir -p ${upstream_code} \
+    && tar -xzf ${upstream_tarball} \
+        --strip-components 1 \
+        -C ${upstream_code}
+## EXCLUDE END ##
+
+#@follow_tag(openshift-ose-base:ubi8.python.36)
+FROM openshift-ose-base:v4.0-202009120102.31989
+
+ENV BUILD_VERSION=${CI_CONTAINER_VERSION:-}
+ENV OS_GIT_MAJOR=${CI_X_VERSION:-}
+ENV OS_GIT_MINOR=${CI_Y_VERSION:-}
+ENV OS_GIT_PATCH=${CI_Z_VERSION:-}
+ENV SOURCE_GIT_COMMIT=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT:-}
+ENV SOURCE_GIT_URL=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL:-}
+
+LABEL \
+        License="GPLv2+" \
+        io.k8s.description="Curator elasticsearch container for elasticsearch deletion/archival" \
+        io.k8s.display-name="Curator 5" \
+        io.openshift.tags="logging,elk,elasticsearch,curator" \
+        vendor="Red Hat" \
+        name="openshift/ose-logging-curator5" \
+        com.redhat.component="logging-curator5-container" \
+        io.openshift.maintainer.product="OpenShift Container Platform" \
+        io.openshift.build.commit.id=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT} \
+        io.openshift.build.source-location=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL} \
+        io.openshift.build.commit.url=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL}/commit/${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL} \
+        version=${CI_CONTAINER_VERSION}
+
+MAINTAINER OpenShift Development <[email protected]>
+
+ENV HOME=/opt/app-root/src \
+    ES_HOST=localhost \
+    ES_PORT=9200 \
+    ES_CA=/etc/curator/keys/ca \
+    ES_CLIENT_CERT=/etc/curator/keys/cert \
+    ES_CLIENT_KEY=/etc/curator/keys/key \
+    CURATOR_CONF_LOCATION=/etc/curator/settings/config.yaml \
+    CURATOR_CONF_FILE=/etc/curator/settings/curator5.yaml \
+    CURATOR_ACTIONS_FILE=/etc/curator/settings/actions.yaml \
+    CURATOR_LOG_LEVEL=ERROR \
+    CURATOR_SCRIPT_LOG_LEVEL=INFO \
+    CURATOR_TIMEOUT=300 \
+    CURATOR_VER=5.8.1 \
+    container=oci \
+    LC_ALL=en_US.UTF-8
+ENV upstream_code=${upstream_code:-"."}
+
+USER 0
+COPY --from=builder ${upstream_code}/ ${HOME}
+
+RUN mkdir -p $(dirname "$CURATOR_CONF_LOCATION") && \
+    touch ${CURATOR_CONF_LOCATION} && \
+    chmod -R u+x ${HOME} && \
+    chgrp -R 0 ${HOME} && \
+    chmod -R g=u ${HOME}
+
+WORKDIR ${HOME}/vendor
+RUN pip install $(ls . | grep -v curator) -q --no-index --find-links . && \
+    pip install elasticsearch-curator* --no-index -q && \
+    rm -rf $HOME/vendor
+
+WORKDIR ${HOME}
+USER 1001
+CMD ["sh", "run.sh"]
diff --git a/curator/origin-meta.yaml b/curator/origin-meta.yaml
@@ -0,0 +1,3 @@
+from:
+- source: openshift-ose-base\:v(?:[\.0-9\-]*)
+  target: registry.ci.openshift.org/ocp/builder:ubi8.python.36 
diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile
diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile
@@ -0,0 +1,99 @@
+### This is a generated file from Dockerfile.in ###
+
+#@follow_tag(openshift-ose-base:ubi8)
+FROM registry.ci.openshift.org/ocp/builder:rhel-8-base-openshift-4.7
+
+ENV BUILD_VERSION=${CI_CONTAINER_VERSION:-}
+ENV OS_GIT_MAJOR=${CI_X_VERSION:-}
+ENV OS_GIT_MINOR=${CI_Y_VERSION:-}
+ENV SOURCE_GIT_COMMIT=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT:-}
+ENV SOURCE_GIT_URL=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL:-}
+
+LABEL \
+        License="GPLv2+" \
+        io.k8s.description="Elasticsearch container for EFK aggregated logging storage" \
+        io.k8s.display-name="Elasticsearch 6" \
+        io.openshift.tags="logging,elk,elasticsearch" \
+        vendor="Red Hat" \
+        name="openshift-logging/elasticsearch6-rhel8" \
+        com.redhat.component="logging-elasticsearch6-container" \
+        io.openshift.maintainer.product="OpenShift Container Platform" \
+        io.openshift.build.commit.id=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT} \
+        io.openshift.build.source-location=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL} \
+        io.openshift.build.commit.url=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL}/commit/${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT} \
+        version=${CI_CONTAINER_VERSION}
+
+MAINTAINER OpenShift Development <[email protected]>
+
+EXPOSE 9200
+EXPOSE 9300
+USER 0
+
+ARG ES_ARCHIVE_URL
+ARG PROMETHEUS_EXPORTER_URL
+ARG OPENDISTRO_URL
+ARG OPENSHIFT_CI
+
+
+ENV ES_PATH_CONF=/etc/elasticsearch/ \
+    ES_HOME=/usr/share/elasticsearch \
+    ES_VER=6.8.1.redhat-00006 \
+    HOME=/opt/app-root/src \
+    INSTANCE_RAM=512G \
+    JAVA_VER=11 \
+    JAVA_HOME=/usr/lib/jvm/jre \
+    NODE_QUORUM=1 \
+    PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-00001 \
+    OPENDISTRO_VER=0.10.0.4-redhat-00001 \
+    PLUGIN_LOGLEVEL=INFO \
+    RECOVER_AFTER_NODES=1 \
+    RECOVER_EXPECTED_NODES=1 \
+    RECOVER_AFTER_TIME=5m \
+    DHE_TMP_KEY_SIZE=2048 \
+    RELEASE_STREAM=prod \
+    OPENSHIFT_CI=${OPENSHIFT_CI:-false}
+
+
+# COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR
+# WORKDIR $REMOTE_SOURCE_DIR/app/elasticsearch
+
+RUN packages="java-${JAVA_VER}-openjdk-headless \
+              python36 \
+              python3-pyyaml \
+              hostname \
+              openssl \
+              zip \
+              unzip" && \
+    yum install -y ${packages} && \
+    rpm -V ${packages} && \
+    alternatives --set python /usr/bin/python3 && \
+    yum clean all
+
+ARG MAVEN_REPO_URL=${MAVEN_REPO_URL:-file:///artifacts/}
+ENV upstream_code=${upstream_code:-"."}
+
+COPY  ${upstream_code}/extra-jvm.options /var/tmp
+COPY  ${upstream_code}/ci-env.sh /var/tmp
+COPY  ${upstream_code}/install-es.sh /var/tmp
+COPY artifacts /artifacts
+RUN /var/tmp/install-es.sh
+
+COPY  ${upstream_code}/sgconfig/ ${HOME}/sgconfig/
+COPY  ${upstream_code}/index_templates/ ${ES_HOME}/index_templates/
+COPY  ${upstream_code}/index_patterns/ ${ES_HOME}/index_patterns/
+COPY  ${upstream_code}/init/ ${ES_HOME}/init/
+COPY  ${upstream_code}/probe/ ${ES_HOME}/probe/
+COPY  ${upstream_code}/init.sh ${HOME}/
+COPY  ${upstream_code}/run.sh ${HOME}/
+COPY  ${upstream_code}/ci-env.sh ${HOME}/
+COPY  ${upstream_code}/install.sh ${HOME}/
+COPY  ${upstream_code}/utils/** /usr/local/bin/
+
+RUN ${HOME}/install.sh && rm -rf /artifacts && \
+  mv ${ES_HOME}/lib/elasticsearch-${ES_VER}.jar ${ES_HOME}/lib/elasticsearch-$(echo $ES_VER | cut -d'.' -f1-3).jar
+
+WORKDIR ${HOME}
+USER 1000
+CMD ["sh", "/opt/app-root/src/run.sh"]
+
+
diff --git a/elasticsearch/Dockerfile.in b/elasticsearch/Dockerfile.in
@@ -0,0 +1,114 @@
+## EXCLUDE BEGIN ##
+#@follow_tag(openshift-ose-base:ubi8)
+FROM openshift-ose-base:v4.0-202104100031.20055 AS builder
+
+ARG upstream_code
+ENV upstream_code=upstream_code    
+ARG upstream_tarball=${upstream_code}.tar.gz    
+
+COPY ${CI_ARCHIVE_ORIGIN_AGGREGATED_LOGGING} ${upstream_tarball}    
+RUN mkdir -p ${upstream_code} \    
+    && tar -xzf ${upstream_tarball} \    
+        --strip-components 1 \    
+        -C ${upstream_code}  
+## EXCLUDE END ##
+
+#@follow_tag(openshift-ose-base:ubi8)
+FROM openshift-ose-base:v4.0-202104100031.20055
+
+ENV BUILD_VERSION=${CI_CONTAINER_VERSION:-}
+ENV OS_GIT_MAJOR=${CI_X_VERSION:-}
+ENV OS_GIT_MINOR=${CI_Y_VERSION:-}
+ENV SOURCE_GIT_COMMIT=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT:-}
+ENV SOURCE_GIT_URL=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL:-}
+
+LABEL \
+        License="GPLv2+" \
+        io.k8s.description="Elasticsearch container for EFK aggregated logging storage" \
+        io.k8s.display-name="Elasticsearch 6" \
+        io.openshift.tags="logging,elk,elasticsearch" \
+        vendor="Red Hat" \
+        name="openshift-logging/elasticsearch6-rhel8" \
+        com.redhat.component="logging-elasticsearch6-container" \
+        io.openshift.maintainer.product="OpenShift Container Platform" \
+        io.openshift.build.commit.id=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT} \
+        io.openshift.build.source-location=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL} \
+        io.openshift.build.commit.url=${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_URL}/commit/${CI_ORIGIN_AGGREGATED_LOGGING_UPSTREAM_COMMIT} \
+        version=${CI_CONTAINER_VERSION}
+
+MAINTAINER OpenShift Development <[email protected]>
+
+EXPOSE 9200
+EXPOSE 9300
+USER 0
+
+ARG ES_ARCHIVE_URL
+ARG PROMETHEUS_EXPORTER_URL
+ARG OPENDISTRO_URL
+ARG OPENSHIFT_CI
+
+
+ENV ES_PATH_CONF=/etc/elasticsearch/ \
+    ES_HOME=/usr/share/elasticsearch \
+    ES_VER=6.8.1.redhat-00006 \
+    HOME=/opt/app-root/src \
+    INSTANCE_RAM=512G \
+    JAVA_VER=11 \
+    JAVA_HOME=/usr/lib/jvm/jre \
+    NODE_QUORUM=1 \
+    PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-00001 \
+    OPENDISTRO_VER=0.10.0.4-redhat-00001 \
+    PLUGIN_LOGLEVEL=INFO \
+    RECOVER_AFTER_NODES=1 \
+    RECOVER_EXPECTED_NODES=1 \
+    RECOVER_AFTER_TIME=5m \
+    DHE_TMP_KEY_SIZE=2048 \
+    RELEASE_STREAM=prod \
+    OPENSHIFT_CI=${OPENSHIFT_CI:-false}
+
+
+# COPY $REMOTE_SOURCE $REMOTE_SOURCE_DIR
+# WORKDIR $REMOTE_SOURCE_DIR/app/elasticsearch
+
+RUN packages="java-${JAVA_VER}-openjdk-headless \
+              python36 \
+              python3-pyyaml \
+              hostname \
+              openssl \
+              zip \
+              unzip" && \
+    yum install -y ${packages} && \
+    rpm -V ${packages} && \
+    alternatives --set python /usr/bin/python3 && \
+    yum clean all
+
+ARG MAVEN_REPO_URL=${MAVEN_REPO_URL:-file:///artifacts/}
+## EXCLUDE BEGIN ##
+ENV upstream_code=upstream_code/elasticsearch
+## EXCLUDE END ##
+ENV upstream_code=${upstream_code:-"."}
+
+COPY --from=builder ${upstream_code}/extra-jvm.options /var/tmp
+COPY --from=builder ${upstream_code}/ci-env.sh /var/tmp
+COPY --from=builder ${upstream_code}/install-es.sh /var/tmp
+COPY artifacts /artifacts
+RUN /var/tmp/install-es.sh
+
+COPY --from=builder ${upstream_code}/sgconfig/ ${HOME}/sgconfig/
+COPY --from=builder ${upstream_code}/index_templates/ ${ES_HOME}/index_templates/
+COPY --from=builder ${upstream_code}/index_patterns/ ${ES_HOME}/index_patterns/
+COPY --from=builder ${upstream_code}/init/ ${ES_HOME}/init/
+COPY --from=builder ${upstream_code}/probe/ ${ES_HOME}/probe/
+COPY --from=builder ${upstream_code}/init.sh ${HOME}/
+COPY --from=builder ${upstream_code}/run.sh ${HOME}/
+COPY --from=builder ${upstream_code}/ci-env.sh ${HOME}/
+COPY --from=builder ${upstream_code}/install.sh ${HOME}/
+COPY --from=builder ${upstream_code}/utils/** /usr/local/bin/
+
+RUN ${HOME}/install.sh && rm -rf /artifacts && \
+  mv ${ES_HOME}/lib/elasticsearch-${ES_VER}.jar ${ES_HOME}/lib/elasticsearch-$(echo $ES_VER | cut -d'.' -f1-3).jar
+
+WORKDIR ${HOME}
+USER 1000
+CMD ["sh", "/opt/app-root/src/run.sh"]
+
diff --git a/elasticsearch/Dockerfile.origin b/elasticsearch/Dockerfile.origin
@@ -1 +1 @@
-Dockerfile.rhel8
+Dockerfile