diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 3cec6573d..d820993cd 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -31,6 +31,7 @@ USER 0 ARG ES_ARCHIVE_URL ARG PROMETHEUS_EXPORTER_URL +ARG INGEST_PLUGIN_URL ARG OPENDISTRO_URL ARG OPENSHIFT_CI @@ -44,6 +45,7 @@ ENV ES_PATH_CONF=/etc/elasticsearch/ \ JAVA_HOME=/usr/lib/jvm/jre \ NODE_QUORUM=1 \ PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-00001 \ + INGEST_PLUGIN_VER=6.8.1.0-redhat-00002 \ OPENDISTRO_VER=0.10.0.4-redhat-00001 \ PLUGIN_LOGLEVEL=INFO \ RECOVER_AFTER_NODES=1 \ @@ -80,6 +82,7 @@ RUN /var/tmp/install-es.sh COPY ${upstream_code}/sgconfig/ ${HOME}/sgconfig/ COPY ${upstream_code}/index_templates/ ${ES_HOME}/index_templates/ +COPY ${upstream_code}/ingest_pipelines/ ${ES_HOME}/ingest_pipelines/ COPY ${upstream_code}/index_patterns/ ${ES_HOME}/index_patterns/ COPY ${upstream_code}/init/ ${ES_HOME}/init/ COPY ${upstream_code}/probe/ ${ES_HOME}/probe/ diff --git a/elasticsearch/Dockerfile.in b/elasticsearch/Dockerfile.in index 999bb266c..366119d2c 100644 --- a/elasticsearch/Dockerfile.in +++ b/elasticsearch/Dockerfile.in @@ -44,6 +44,7 @@ USER 0 ARG ES_ARCHIVE_URL ARG PROMETHEUS_EXPORTER_URL +ARG INGEST_PLUGIN_URL ARG OPENDISTRO_URL ARG OPENSHIFT_CI @@ -57,6 +58,7 @@ ENV ES_PATH_CONF=/etc/elasticsearch/ \ JAVA_HOME=/usr/lib/jvm/jre \ NODE_QUORUM=1 \ PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-00001 \ + INGEST_PLUGIN_VER=6.8.1.0-redhat-00002 \ OPENDISTRO_VER=0.10.0.4-redhat-00001 \ PLUGIN_LOGLEVEL=INFO \ RECOVER_AFTER_NODES=1 \ @@ -96,6 +98,7 @@ RUN /var/tmp/install-es.sh COPY --from=builder ${upstream_code}/sgconfig/ ${HOME}/sgconfig/ COPY --from=builder ${upstream_code}/index_templates/ ${ES_HOME}/index_templates/ +COPY --from=builder ${upstream_code}/ingest_pipelines/ ${ES_HOME}/ingest_pipelines/ COPY --from=builder ${upstream_code}/index_patterns/ ${ES_HOME}/index_patterns/ COPY --from=builder ${upstream_code}/init/ ${ES_HOME}/init/ COPY --from=builder ${upstream_code}/probe/ ${ES_HOME}/probe/ diff --git a/elasticsearch/ci-env.sh b/elasticsearch/ci-env.sh index cd62469fc..b32a29a9a 100755 --- a/elasticsearch/ci-env.sh +++ b/elasticsearch/ci-env.sh @@ -4,6 +4,7 @@ set -o xtrace PROMETHEUS_EXPORTER_URL=${PROMETHEUS_EXPORTER_URL:-$MAVEN_REPO_URL/org/elasticsearch/plugin/prometheus/prometheus-exporter/$PROMETHEUS_EXPORTER_VER/prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip} OPENDISTRO_URL=${OPENDISTRO_URL:-$MAVEN_REPO_URL/com/amazon/opendistroforelasticsearch/opendistro_security/$OPENDISTRO_VER/opendistro_security-$OPENDISTRO_VER.zip} +INGEST_PLUGIN_URL=${INGEST_PLUGIN_URL:-https://github.com/ViaQ/elasticsearch-openshift-ingest-plugin/releases/download/$INGEST_PLUGIN_VER/openshift-ingest-plugin-$INGEST_PLUGIN_VER.zip} if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then # This flag is set during CI runs. If no ARG was passed in, @@ -11,7 +12,12 @@ if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then export ES_ARCHIVE_URL=https://github.com/openshift/origin-aggregated-logging/releases/download/elasticsearch-oss-$ES_VER/elasticsearch-oss-$ES_VER.zip export OPENDISTRO_URL=https://github.com/openshift/origin-aggregated-logging/releases/download/opendistro_security-$OPENDISTRO_VER/opendistro_security-$OPENDISTRO_VER.zip + INGEST_PLUGIN_VER=$(echo $INGEST_PLUGIN_VER | cut -d'-' -f1) + export INGEST_PLUGIN_URL=https://github.com/ViaQ/elasticsearch-openshift-ingest-plugin/releases/download/$INGEST_PLUGIN_VER/openshift-ingest-plugin-$INGEST_PLUGIN_VER.zip + # Modify this if you need to CI-test the plugin sourcing from different repo + # export INGEST_PLUGIN_URL=https://github.com/lukas-vlcek/elasticsearch-openshift-ingest-plugin/releases/download/v$INGEST_PLUGIN_VER/openshift-ingest-plugin-$INGEST_PLUGIN_VER.zip + PROMETHEUS_EXPORTER_VER=$(echo $PROMETHEUS_EXPORTER_VER | cut -d'-' -f1) export PROMETHEUS_EXPORTER_URL=https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/$PROMETHEUS_EXPORTER_VER/prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip fi -es_plugins=($OPENDISTRO_URL $PROMETHEUS_EXPORTER_URL) +es_plugins=($OPENDISTRO_URL $PROMETHEUS_EXPORTER_URL $INGEST_PLUGIN_URL) diff --git a/elasticsearch/fetch-artifacts-koji.yaml b/elasticsearch/fetch-artifacts-koji.yaml index 48e0ebf79..e5d7e428b 100644 --- a/elasticsearch/fetch-artifacts-koji.yaml +++ b/elasticsearch/fetch-artifacts-koji.yaml @@ -1,3 +1,4 @@ - nvr: org.elasticsearch-elasticsearch-6.8.1.redhat_00006-1 - nvr: com.amazon.opendistroforelasticsearch-opendistro_security-0.10.0.4_redhat_00001-1 - nvr: org.elasticsearch.plugin.prometheus-prometheus-exporter-6.8.1.0_redhat_00001-1 +- nvr: org.elasticsearch.plugin.ingest-openshift-ingest-plugin-6.8.1.0_redhat_00002-1 \ No newline at end of file diff --git a/elasticsearch/index_templates/com.redhat.viaq-openshift-operations.template.json b/elasticsearch/index_templates/com.redhat.viaq-openshift-operations.template.json index 85e7509ad..1d4633cc7 100644 --- a/elasticsearch/index_templates/com.redhat.viaq-openshift-operations.template.json +++ b/elasticsearch/index_templates/com.redhat.viaq-openshift-operations.template.json @@ -973,6 +973,7 @@ }, "order": 10, "settings": { - "index.refresh_interval": "5s" + "index.refresh_interval": "5s", + "index.default_pipeline": "openshift_schema.json" } } diff --git a/elasticsearch/index_templates/com.redhat.viaq-openshift-project.template.json b/elasticsearch/index_templates/com.redhat.viaq-openshift-project.template.json index a426bb281..581754da6 100644 --- a/elasticsearch/index_templates/com.redhat.viaq-openshift-project.template.json +++ b/elasticsearch/index_templates/com.redhat.viaq-openshift-project.template.json @@ -972,6 +972,7 @@ }, "order": 10, "settings": { - "index.refresh_interval": "5s" + "index.refresh_interval": "5s", + "index.default_pipeline": "openshift_schema.json" } } diff --git a/elasticsearch/ingest_pipelines/openshift_schema.json b/elasticsearch/ingest_pipelines/openshift_schema.json new file mode 100644 index 000000000..972bd5ec7 --- /dev/null +++ b/elasticsearch/ingest_pipelines/openshift_schema.json @@ -0,0 +1,6 @@ +{ + "description": "Openshift ingest pipeline for custom schema", + "processors": [ + { "openshift-ingestion-processor": {} } + ] +} diff --git a/elasticsearch/init/0100-seed-pipelines b/elasticsearch/init/0100-seed-pipelines new file mode 100755 index 000000000..bfa3e7a06 --- /dev/null +++ b/elasticsearch/init/0100-seed-pipelines @@ -0,0 +1,42 @@ +#!/bin/bash -e +# +# Copyright 2018 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "logging" + +info Adding ingest pipelines +shopt -s failglob +for ingest_file in ${ES_HOME}/ingest_pipelines/*.json +do + + template=`basename $ingest_file` + # Check if ingest template already exists + response_code=$(es_util --query=_ingest/pipeline/$template \ + ${DEBUG:+-v} -s \ + --request HEAD --head --output /dev/null \ + -w '%{response_code}') + if [ "${response_code}" == "200" ]; then + info "Ingest pipeline '$template' found in the cluster, overriding it" + else + info "Create ingest pipeline '$template'" + fi + es_util --query=_ingest/pipeline/$template \ + ${DEBUG:+-v} -s -X PUT \ + -d@${ingest_file} + +done +shopt -u failglob +info Finished adding ingest pipelines diff --git a/elasticsearch/init/0100-seed-index-templates b/elasticsearch/init/0200-seed-index-templates similarity index 100% rename from elasticsearch/init/0100-seed-index-templates rename to elasticsearch/init/0200-seed-index-templates diff --git a/elasticsearch/install.sh b/elasticsearch/install.sh index 333197af7..30662644e 100755 --- a/elasticsearch/install.sh +++ b/elasticsearch/install.sh @@ -23,6 +23,7 @@ done #fix location from config if [[ "${ES_HOME}" != "/usr/share/elasticsearch" ]]; then ln -s ${ES_HOME}/index_templates /usr/share/elasticsearch/index_templates + ln -s ${ES_HOME}/ingest_pipelines /usr/share/elasticsearch/ingest_pipelines ln -s ${ES_HOME}/index_patterns /usr/share/elasticsearch/index_patterns fi