Merge branch 'develop' into feature/long-lasting-certificates-openssl

.dockerignore

@@ -1,15 +1,13 @@
-# Ignore all unwanted files from the docker build process
-
-.git
-.git*
-*.log
-certs/
-**/data/*
-
-*.md
-
-tests/
-Dockerfile
-
-core/data/
-core/build/
+# Exclude unnecessary files from the build context
+# This file is meant to be used as a whitelist
+
+# Exclude everything
+*
+
+# Include the following
+!README.md
+!LICENSE
+!core/src/epicli/.devcontainer/requirements.txt
+!core/src/epicli/cli/
+!core/src/epicli/data/
+!core/src/epicli/setup.py

CHANGELOG-0.8.md

@@ -5,8 +5,10 @@
 ### Added
 
 - [#1302](https://github.com/epiphany-platform/epiphany/issues/1302) - Ability to update control plane certificates expiration date
+- [#1324](https://github.com/epiphany-platform/epiphany/issues/1324) - Added Logstash to export data from Elasticsearch to csv format
 - [#1300](https://github.com/epiphany-platform/epiphany/issues/1300) - Configure OpenSSH according to Mozilla Infosec guidance
 - [#1543](https://github.com/epiphany-platform/epiphany/issues/1543) - Add support for Azure availability sets
+- [#1609](https://github.com/epiphany-platform/epiphany/issues/1609) - Build epicli image using Dockerfile only (without shell script)
 
 ### Updated
 

Dockerfile

@@ -0,0 +1,47 @@
+# === Build epicli wheel file ===
+
+FROM python:3.7-slim AS build-epicli-wheel
+
+COPY . /src
+WORKDIR /src/core/src/epicli
+
+RUN python setup.py bdist_wheel
+
+# === Build final image ===
+
+FROM python:3.7-slim
+
+ARG USERNAME=epiuser
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+ENV EPICLI_DOCKER_SHARED_DIR=/shared
+
+COPY --from=build-epicli-wheel /src/core/src/epicli/dist/ /epicli/
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        gcc libffi-dev make musl-dev openssh-client ruby-full sudo tar unzip vim \
+    && gem install \
+        rake rspec_junit_formatter serverspec \
+    && pip install --disable-pip-version-check --no-cache-dir \
+        /epicli/epicli-*-py3-none-any.whl \
+\
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* \
+\
+    && groupadd --gid $USER_GID $USERNAME \
+    && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME \
+\
+    && mkdir -p $EPICLI_DOCKER_SHARED_DIR \
+    && chown $USERNAME $EPICLI_DOCKER_SHARED_DIR \
+    && chmod g+w $EPICLI_DOCKER_SHARED_DIR
+
+WORKDIR $EPICLI_DOCKER_SHARED_DIR
+
+USER $USERNAME
+
+ENTRYPOINT ["/bin/bash"]

core/src/epicli/.devcontainer/Dockerfile

@@ -14,31 +14,32 @@ COPY requirements.txt /
 
 RUN chmod +x /config-pre.sh \
     && /bin/bash -c /config-pre.sh \
-
+\
     && apt-get update \
-    && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \
-
-    && apt-get -y install git git-lfs procps lsb-release gcc make musl-dev libffi-dev tar unzip vim \
-
-    && apt-get -y install ruby-full \
-
-    && gem install serverspec rake rspec_junit_formatter \
-
-    && pip --disable-pip-version-check --no-cache-dir install -r /requirements.txt --default-timeout=100 \
-
-    && pip --disable-pip-version-check --no-cache-dir install pipenv pylint pytest wheel setuptools twine --default-timeout=100 \
-
+    && apt-get install --no-install-recommends -y \
+        apt-utils dialog 2>&1 \
+    && apt-get install --no-install-recommends -y \
+        gcc libffi-dev make musl-dev openssh-client ruby-full sudo tar unzip vim \
+        \
+        git git-lfs lsb-release procps \
+\
+    && gem install \
+        rake rspec_junit_formatter serverspec \
+\
+    && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
+        --requirement /requirements.txt \
+    && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
+        pipenv pylint pytest setuptools twine wheel \
+\
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* \
+\
     && groupadd --gid $USER_GID $USERNAME \
     && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \
-
-    && apt-get install -y sudo \
     && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
     && chmod 0440 /etc/sudoers.d/$USERNAME \
-
-    && apt-get autoremove -y \
-    && apt-get clean -y \
-    && rm -rf /var/lib/apt/lists/* \
-
+\
     && chmod +x /config-post.sh \
     && /bin/bash -c /config-post.sh
 

core/src/epicli/data/common/ansible/playbooks/logstash.yml

@@ -0,0 +1,9 @@
+---
+- hosts: all
+  tasks: []
+
+- hosts: logstash
+  become: true
+  become_method: sudo
+  roles:
+    - logstash

core/src/epicli/data/common/ansible/playbooks/roles/common/templates/sshd_config.j2

@@ -3,7 +3,7 @@
 # Supported HostKey algorithms by order of preference.
 HostKey /etc/ssh/ssh_host_ed25519_key
 HostKey /etc/ssh/ssh_host_rsa_key
-HostKey /etc/ssh/ssh_host_ecdsa_key
+#HostKey /etc/ssh/ssh_host_ecdsa_key
 
 KexAlgorithms [email protected],ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256
 

core/src/epicli/data/common/ansible/playbooks/roles/logstash/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+versions:
+  RedHat: "7.8.1"
+  Debian: "1:7.8.1*"

core/src/epicli/data/common/ansible/playbooks/roles/logstash/files/logstash-export.conf.template

@@ -0,0 +1,23 @@
+input {
+ elasticsearch {
+    hosts => "HOST_IP:ES_HTTP_PORT"
+    ssl => "BOOLEAN"
+    ca_file => "CA_FILE.pem"
+    user => "USER"
+    password => "PASSWORD"
+    index => "INDEX_NAME"
+    query => '
+    {
+	    "query": {
+		    "match_all": {}
+	    }
+    }
+  '
+  }
+}
+output {
+  csv {
+    fields => ["FIELDS_TO_EXPORT"]
+    path => "PATH"
+  }
+}

core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/Debian.yml

@@ -0,0 +1,7 @@
+---
+- name: Install Logstash package
+  apt:
+    name:
+      - logstash-oss={{ versions[ansible_os_family] }}
+    update_cache: yes
+    state: present

core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/RedHat.yml

@@ -0,0 +1,7 @@
+---
+- name: Install Logstash package
+  yum:
+    name:
+      - logstash-oss-{{ versions[ansible_os_family] }}
+    update_cache: yes
+    state: present

core/src/epicli/data/common/ansible/playbooks/roles/logstash/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+- name: Install logstash-oss
+  include_tasks: "{{ ansible_os_family }}.yml"
+
+- name: Ensure Logstash is disabled
+  systemd:
+    name: logstash
+    enabled: no
+    state: stopped
+
+- name: Provide logstash-export.conf template file
+  copy:
+    src: logstash-export.conf.template
+    dest: /etc/logstash/logstash-export.conf.template

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -57,6 +57,7 @@ libxcb # for grafana
 libXcursor # for grafana
 libXt # for grafana
 logrotate
+logstash-oss-7.8.1
 net-tools
 nfs-utils
 nmap-ncat

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt

@@ -56,6 +56,7 @@ libxcb # for grafana
 libXcursor # for grafana
 libXt # for grafana
 logrotate
+logstash-oss-7.8.1
 net-tools
 nfs-utils
 nmap-ncat

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt

@@ -29,6 +29,7 @@ init-system-helpers
 jq
 libfontconfig1
 logrotate
+logstash-oss 1:7.8.1
 netcat
 net-tools
 nfs-common

core/src/epicli/data/common/defaults/configuration/feature-mapping.yml

@@ -27,6 +27,8 @@ specification:
       enabled: yes
     - name: filebeat
       enabled: yes
+    - name: logstash
+      enabled: yes
     - name: prometheus
       enabled: yes
     - name: grafana

core/src/epicli/data/common/defaults/configuration/logstash.yml

@@ -0,0 +1,4 @@
+kind: configuration/logstash
+title: "Logstash"
+name: default
+specification: {}

core/src/epicli/data/common/validation/configuration/logstash.yml

@@ -0,0 +1 @@
+$ref: '#/definitions/unvalidated_specification'

docs/home/COMPONENTS.md

@@ -21,7 +21,8 @@ Note that versions are default versions and can be changed in certain cases thro
 | Kibana                    | 6.5.4   | https://github.com/elastic/kibana                     | https://github.com/elastic/kibana/blob/master/LICENSE.txt         |
 | Opendistro for Elasticsearch  | 1.9.0   | https://opendistro.github.io/for-elasticsearch/  | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)  |
 | Opendistro for Elasticsearch Kibana  | 1.9.0   | https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/  | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)  |
-| Filebeat                  | 7.8.1   | https://github.com/elastic/beats/tree/6.8             | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
+| Filebeat                  | 7.8.1   | https://github.com/elastic/beats                      | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
+| Logstash OSS              | 7.8.1   | https://github.com/elastic/logstash                   | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Prometheus                | 2.10.0  | https://github.com/prometheus/prometheus              | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Grafana                   | 6.2.5   | https://github.com/grafana/grafana                    | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | node_exporter             | 0.16.0  | https://github.com/prometheus/node_exporter           | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |

docs/home/HOWTO.md

@@ -60,11 +60,11 @@
   - [How to use TLS/SSL certificate with RabbitMQ](./howto/SECURITY.md#how-to-use-tlsssl-certificate-with-rabbitmq)
   - [How to use TLS/SSL with Kafka](./howto/SECURITY.md#how-to-use-tlsssl-with-kafka)
   - [How to use TLS/SSL certificates for Kafka authentication](./howto/SECURITY.md#how-to-use-tlsssl-certificates-for-kafka-authentication)
-  - [How to use TLS/SSL certificates for Kafka authorization](./howto/SECURITY.md#how-to-use-tlsssl-certificates-for-kafka-authorization)  
+  - [How to use TLS/SSL certificates for Kafka authorization](./howto/SECURITY.md#how-to-use-tlsssl-certificates-for-kafka-authorization)
   - [How to enable Azure disk encryption](./howto/SECURITY.md#how-to-enable-azure-disk-encryption)
   - [How to enable AWS disk encryption](./howto/SECURITY.md#how-to-enable-AWS-disk-encryption)
   - [How to use Kubernetes Secrets](./howto/SECURITY.md#how-to-use-kubernetes-secrets)
-  - [How to authenticate to Azure AD app](./howto/SECURITY.md#how-to-authenticate-to-azure-ad-app)  
+  - [How to authenticate to Azure AD app](./howto/SECURITY.md#how-to-authenticate-to-azure-ad-app)
   - [How to run epicli with password](./howto/SECURITY.md#how-to-run-epicli-with-password)
   - [How to enable kubectl on Kubernetes master](./howto/SECURITY.md#how-to-enable-kubectl-on-Kubernetes-master)
   - [How to turn on Hashicorp Vault functionality](./howto/SECURITY.md#how-to-turn-on-hashicorp-vault-functionality)
@@ -80,6 +80,7 @@
   - [How to set up PGBouncer, PgPool and PostgreSQL parameters](./howto/DATABASES.md#how-to-set-up-pgbouncer-pgpool-and-postgresql-parameters)
   - [How to set up PostgreSQL audit logging](./howto/DATABASES.md#how-to-set-up-postgresql-audit-logging)
   - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch)
+  - [How to export Elasticsearch data to csv format](./howto/LOGGING.md##how-to-export-elasticsearch-data-to-csv-format)
   - [How to start working with Apache Ignite Stateful setup](./howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateful-setup)
   - [How to start working with Apache Ignite Stateless setup](./howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateless-setup)
 
@@ -101,3 +102,4 @@
 
 - [Centralized Logging](./howto/LOGGING.md)
   - [Centralized logging setup](./howto/LOGGING.md#centralized-logging-setup)
+  - [How to export Elasticsearch data to csv format](./howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format)

docs/home/howto/LOGGING.md

@@ -30,6 +30,9 @@ specification:
       - firewall
     ...
 ```
+>Optional feature (role) available for logging: **logstash**
+>more details here: [link](https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/LOGGING.md#how-to-export-elasticsearch-data-to-csv-format)
+
 The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0.
 
 Default configuration of `logging` and `opendistro_for_elasticsearch` roles is identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify configuration of centralized logging adjust and use the following defaults in your manifest:
@@ -46,3 +49,28 @@ specification:
     repo: /var/lib/elasticsearch-snapshots
     logs: /var/log/elasticsearch
 ```
+
+### How to export Elasticsearch data to csv format
+Since v0.8 Epiphany provide posibility to export data from Elasticsearch to CSV using Logstash *(logstash-oss v7.8.1*) along with *logstash-input-elasticsearch (v4.6.2)* and *logstash-output-csv (v3.0.8)* plugin.
+
+To install Logstash in your cluster add **logstash** to feature mapping for *logging, opendistro_for_elasticsearch* or *elasticsearch* group.
+
+Epiphany provides a basic configuration file `(logstash-export.conf.template)` as template for your data export.
+This file has to be modified according to your Elasticsearch configuration and data you want to export.
+
+`Note: Exporting data is not automated. It has to be invoked manually. Logstash daemon is disabled by default after installation.`
+
+Run Logstash to export data:  
+`/usr/share/logstash/bin/logstash -f /etc/logstash/logstash-export.conf`
+
+More details about configuration of input plugin:  
+https://www.elastic.co/guide/en/logstash/current/plugins-inputs-elasticsearch.html
+
+More details about configuration of output plugin:  
+https://www.elastic.co/guide/en/logstash/current/plugins-outputs-csv.html
+
+Note: Currently input plugin doesn't officialy support skipping certificate validation for secure connection to Elasticsearch.
+
+For non-production environment you can easly disable it by adding new line:  
+`ssl_options[:verify] = false` right after other ssl_options definitions in file:  
+`/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.6.2/lib/logstash/inputs/elasticsearch.rb`