From 5de9ae5853e00bd9dc52c6c4bba49d4c828f723e Mon Sep 17 00:00:00 2001
From: rafzei <13080132+rafzei@users.noreply.github.com>
Date: Wed, 22 Jul 2020 16:30:22 +0200
Subject: [PATCH 1/3] Add missing opendistro_elasticsearch host in kibana
 config

---
 .../playbooks/roles/kibana/tasks/main.yml      |  8 +++++---
 .../roles/kibana/tasks/setup-logging.yml       |  2 +-
 .../roles/kibana/templates/kibana.yml.j2       | 18 ++++++++++++++----
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml
index eb426707f2..0ae91e1b22 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml
@@ -1,5 +1,6 @@
 ---
-- include_tasks: "{{ ansible_os_family }}.yml"
+- name: Include task for providing Kibana package
+  include_tasks: "{{ ansible_os_family }}.yml"
 
 - name: Update Kibana configuration file
   template:
@@ -10,7 +11,8 @@
     mode: 0644
   register: change_config
 
-- include_tasks: setup-logging.yml
+- name: Include tasks for configure logging
+  include_tasks: setup-logging.yml
 
 - name: Restart Kibana service
   systemd:
@@ -29,7 +31,7 @@
     host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
     port: 5601
     delay: 5
-  
+
 - name: Wait for kibana to be ready
   uri:
     url: http://{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}:5601/api/kibana/settings
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/setup-logging.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/setup-logging.yml
index 4863f46100..8d1b4c6375 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/setup-logging.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/setup-logging.yml
@@ -28,4 +28,4 @@
     owner: root
     group: root
     mode: 0644
-    src: logrotate.conf.j2
\ No newline at end of file
+    src: logrotate.conf.j2
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
index f3d4903731..c086398d3c 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
@@ -14,12 +14,22 @@
 # permissions and limitations under the License.
 
 # Description:
-# Default Kibana configuration for Open Distro.
+# https://www.elastic.co/guide/en/kibana/current/settings.html
+# Default Kibana configuration
+
 server.host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
-elasticsearch.hosts: 
-{% for host in groups['logging'] %}
+elasticsearch.hosts:
+{% if groups['logging'] is defined %}
+  {%- for host in groups['logging'] %}
+  - "https://{{hostvars[host]['ansible_hostname']}}:9200"
+  {% endfor %}
+{% endif %}
+{% if groups['opendistro_for_elasticsearch'] is defined %}
+  {%- for host in groups['opendistro_for_elasticsearch'] -%}
   - "https://{{hostvars[host]['ansible_hostname']}}:9200"
-{% endfor %}
+  {% endfor %}
+{% endif %}
+
 elasticsearch.ssl.verificationMode: none
 elasticsearch.username: kibanaserver
 elasticsearch.password: kibanaserver

From 22a78c71d83bd61894318d26167935a08bfb0e06 Mon Sep 17 00:00:00 2001
From: rafzei <13080132+rafzei@users.noreply.github.com>
Date: Thu, 23 Jul 2020 14:49:30 +0200
Subject: [PATCH 2/3] Add missing opendistro_elasticsearch host in filebeat
 config

---
 CHANGELOG-0.7.md                              |  3 +-
 .../filebeat/tasks/configure-filebeat.yml     | 23 +++++------
 .../playbooks/roles/filebeat/tasks/main.yml   | 28 ++++---------
 .../roles/filebeat/templates/filebeat.yml.j2  | 18 +++++----
 .../roles/kibana/templates/kibana.yml.j2      | 11 +++---
 docs/home/howto/DATABASES.md                  | 20 +++++++++-
 docs/home/howto/LOGGING.md                    | 39 +++++++++++++++----
 7 files changed, 87 insertions(+), 55 deletions(-)

diff --git a/CHANGELOG-0.7.md b/CHANGELOG-0.7.md
index 0b1893f948..0063f70920 100644
--- a/CHANGELOG-0.7.md
+++ b/CHANGELOG-0.7.md
@@ -16,7 +16,8 @@
 - [#1408](https://github.com/epiphany-platform/epiphany/issues/1408) - Epiphany does not support upgrades for Kubernetes in HA mode
 - [#1482](https://github.com/epiphany-platform/epiphany/issues/1482) - Epicli upgrade: flannel CNI plugin is not upgraded to v0.12.0
 - [#1462](https://github.com/epiphany-platform/epiphany/issues/1461) - [BUG] Error upgrading AWS/RHEL/calico cluster from 0.6 to 0.7
-- [#802](https://github.com/epiphany-platform/epiphany/issues/802) - Docker version is hard-coded in installation tasks 
+- [#802](https://github.com/epiphany-platform/epiphany/issues/802) - Docker version is hard-coded in installation tasks
+- [#1347](https://github.com/epiphany-platform/epiphany/issues/1347) - [BUG] Kibana config always points its elasticsearch.hosts to a "logging" VM
 
 ### Added
 
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml
index f8d68a5643..0445f586cf 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml
@@ -7,12 +7,6 @@
     src: filebeat.yml.j2
   register: modify_filebeat_yml
 
-- name: Enable auditd module
-  shell: >-
-    filebeat modules enable auditd 
-  register: enable_module
-  changed_when: "'Enabled auditd' in enable_module.stdout"
-
 - name: Set Filebeat to be started after Docker
   block:
     - name: Create directory (filebeat.service.d)
@@ -37,6 +31,11 @@
 
 - name: Start/restart and enable filebeat service
   block:
+    - name: Enable auditd module
+      command: filebeat modules enable auditd
+      register: enable_module
+      changed_when: "'Enabled auditd' in enable_module.stdout"
+
     - name: Restart filebeat service
       systemd:
         name: filebeat
@@ -51,16 +50,18 @@
         state: started
         enabled: true
 
-    - name: Verify Filebeat is running
+    - name: Verify filebeat is running
       command: systemctl is-active filebeat
       changed_when: false
-  when:
-    - groups['logging'][0] is defined
+      retries: 10
+      delay: 1
+      register: result
+      until: result is succeeded
+  when: groups['logging'][0] is defined
 
 - name: Stop and disable filebeat service
   systemd:
     name: filebeat
     state: stopped
     enabled: false
-  when:
-    - groups['logging'][0] is undefined
\ No newline at end of file
+  when: groups['logging'][0] is undefined
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml
index 8934855b1a..020e45342d 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml
@@ -2,29 +2,15 @@
 - name: Include installation tasks
   include_tasks: install-filebeat.yml
 
-- name: Set facts for filebeat.yml template
-  block: 
-    
-    - name: Set flag if logging output exists
-      set_fact:
-        logging_output_exists: False 
-
-    - name: Set flag if logging output exists
-      set_fact:
-        logging_output_exists: True
-      when:
-        - groups["logging"] is defined
-        - groups["logging"] | length > 0
-
-    - name: Set value for setup.kibana.host
-      set_fact:
-        setup_kibana_host: >-
-          {{ hostvars[groups['kibana'][0]].ansible_default_ipv4.address + ':5601' }}
-      when:
-        - groups['kibana'][0] is defined
+- name: Set value for setup.kibana.host
+  set_fact:
+    setup_kibana_host: >-
+      {{ hostvars[groups['kibana'][0]].ansible_default_ipv4.address + ':5601' }}
+  when:
+    - groups['kibana'][0] is defined
 
 - name: Include auditd configuration tasks
   include_tasks: configure-auditd.yml
 
 - name: Include filebeat configuration tasks
-  include_tasks: configure-filebeat.yml
\ No newline at end of file
+  include_tasks: configure-filebeat.yml
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2
index dc781da291..357905a5f4 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2
@@ -198,27 +198,29 @@ setup.template.settings:
 #================================ Outputs =====================================
 
 # Configure what output to use when sending the data collected by the beat.
+# Default Elasticsearch output is array of 'logging' hosts. If not exist, 'hosts' remains empty to manual configuration.
 
 #-------------------------- Elasticsearch output ------------------------------
 output.elasticsearch:
   # Array of hosts to connect to.
-{% if logging_output_exists %}
-  hosts: 
-{% for host in groups['logging'] %}
-  - "https://{{hostvars[host]['ansible_hostname']}}:9200"
-{% endfor %}
+{% if groups['logging'] is defined %}
+  hosts:
+    # Logging hosts:
+    {% for host in groups['logging'] -%}
+    - "https://{{hostvars[host]['ansible_hostname']}}:9200"
+    {% endfor %}
+
   protocol: "https"
   ssl.verification_mode: none
   username: logstash
   password: logstash
 {% else %}
   hosts: []
-{% endif %}
-
-  # Optional protocol and basic auth credentials.
   #protocol: "https"
+  #ssl.verification_mode: none
   #username: "elastic"
   #password: "changeme"
+{% endif %}
 
 #----------------------------- Logstash output --------------------------------
 #output.logstash:
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
index c086398d3c..410741063c 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
@@ -19,13 +19,14 @@
 
 server.host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
 elasticsearch.hosts:
-{% if groups['logging'] is defined %}
-  {%- for host in groups['logging'] %}
+{% if 'logging' in group_names %}
+  # Logging hosts:
+  {% for host in groups['logging'] %}
   - "https://{{hostvars[host]['ansible_hostname']}}:9200"
   {% endfor %}
-{% endif %}
-{% if groups['opendistro_for_elasticsearch'] is defined %}
-  {%- for host in groups['opendistro_for_elasticsearch'] -%}
+{% elif 'opendistro_for_elasticsearch' in group_names %}
+  # Opendistro hosts:
+  {% for host in groups['opendistro_for_elasticsearch'] %}
   - "https://{{hostvars[host]['ansible_hostname']}}:9200"
   {% endfor %}
 {% endif %}
diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md
index bae586aad5..c5a10989f1 100644
--- a/docs/home/howto/DATABASES.md
+++ b/docs/home/howto/DATABASES.md
@@ -543,7 +543,8 @@ DROP EXTENSION IF EXISTS pgaudit;
 
 ## How to start working with OpenDistro for Elasticsearch
 
-OpenDistro for Elasticsearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration:
+OpenDistro for Elasticsearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/).
+In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration:
 
 ```yaml
 kind: epiphany-cluster
@@ -576,6 +577,23 @@ specification:
 
 Result of this configuration will be one or more independent nodes of OpenDistro.
 
+By default Kibana is deployed only for `logging` component. If you want to deploy kibana for `opendistro_for_elasticsearch` you have to modify feature mapping. Use below configuration in your initial manifest.
+```yaml
+kind: configuration/feature-mapping
+title: "Feature mapping to roles"
+name: default
+specification:
+  roles_mapping:
+    opendistro_for_elasticsearch:
+      - opendistro-for-elasticsearch
+      - node-exporter
+      - filebeat
+      - firewall
+      - kibana
+```
+
+Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md). You can change it manually in /etc/filebeat/filebeat.yml if needed.
+
 ## How to start working with Apache Ignite Stateful setup
 
 Apache Ignite can be installed in Epiphany if `count` property for `ignite` feature is greater than 0.
diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md
index 0cb4f10f78..2bad5072ea 100644
--- a/docs/home/howto/LOGGING.md
+++ b/docs/home/howto/LOGGING.md
@@ -1,32 +1,55 @@
 ## Centralized logging setup
 
-For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/). In order to enable centralized logging, there is required to use `logging` role on feature mapping level - as of 0.5.0 this is a default configuration.
+For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/).
+In order to enable centralized logging, be sure that `count` property for `logging` feature is greater than 0 in your initial manifest.
 
+```yaml
+kind: epiphany-cluster
+...
+specification:
+  ...
+  components:
+    kubernetes_master:
+      count: 1
+    kubernetes_node:
+      count: 0
+    ...
+    logging:
+      count: 1
+    ...
+```
+
+### Default feature mapping for logging:
 ```yaml
     ...
     logging:
       - logging
       - kibana
+      - node-exporter
       - filebeat
       - firewall
     ...
 ```
+The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0.
 
-The `logging` role replaced `elasticsearch` role in logging feature. This change was done to enable Elasticsearch usage for data storage - not only for logs as it was till 0.5.0.
-
-Default configuration of `logging` role is the same as [opendistro_for_elasticsearch](./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) which is used for logs storage. In order to modify centralized logging configuration adjust and use following defaults:
+Default configuration of `logging` and `opendistro_for_elasticsearch` roles are identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify centralized logging configuration adjust and use following defaults in your manifest:
 
 ```yaml
 kind: configuration/logging
 title: Logging Config
 name: default
 specification:
-  opendistro_version_redhat: "1.3.0"
-  elasticsearch_oss_version_debian: "7.3.2"
-  opendistro_version_debian: "1.3.0*"
+  versions:
+    RedHat:
+      elasticsearch_oss: "7.3.2"
+      opendistro: "1.3.0*"
+    Debian:
+      elasticsearch_oss: "7.3.2"
+      opendistro: "1.3.0*"
   cluster_name: EpiphanyElastic
   clustered: True
   paths:
     data: /var/lib/elasticsearch
+    repo: /var/lib/elasticsearch-snapshots
     logs: /var/log/elasticsearch
-```
\ No newline at end of file
+```

From 4c68ba1de17f9f1ab01e34df1a5f3ce60113803a Mon Sep 17 00:00:00 2001
From: rafzei <13080132+rafzei@users.noreply.github.com>
Date: Wed, 29 Jul 2020 16:15:29 +0200
Subject: [PATCH 3/3] Changes after review

---
 .../ansible/playbooks/roles/kibana/tasks/main.yml     |  4 ++--
 .../playbooks/roles/kibana/templates/kibana.yml.j2    |  5 ++---
 docs/home/howto/DATABASES.md                          |  4 ++--
 docs/home/howto/LOGGING.md                            | 11 ++---------
 4 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml
index 0ae91e1b22..fce9dc138c 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/tasks/main.yml
@@ -1,5 +1,5 @@
 ---
-- name: Include task for providing Kibana package
+- name: Include Kibana package installation task
   include_tasks: "{{ ansible_os_family }}.yml"
 
 - name: Update Kibana configuration file
@@ -11,7 +11,7 @@
     mode: 0644
   register: change_config
 
-- name: Include tasks for configure logging
+- name: Include logging configuration tasks
   include_tasks: setup-logging.yml
 
 - name: Restart Kibana service
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
index 410741063c..feec5a7157 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2
@@ -14,8 +14,7 @@
 # permissions and limitations under the License.
 
 # Description:
-# https://www.elastic.co/guide/en/kibana/current/settings.html
-# Default Kibana configuration
+# Default Kibana configuration for Open Distro.
 
 server.host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
 elasticsearch.hosts:
@@ -25,7 +24,7 @@ elasticsearch.hosts:
   - "https://{{hostvars[host]['ansible_hostname']}}:9200"
   {% endfor %}
 {% elif 'opendistro_for_elasticsearch' in group_names %}
-  # Opendistro hosts:
+  # Opendistro (data) hosts:
   {% for host in groups['opendistro_for_elasticsearch'] %}
   - "https://{{hostvars[host]['ansible_hostname']}}:9200"
   {% endfor %}
diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md
index c5a10989f1..e3e7abc854 100644
--- a/docs/home/howto/DATABASES.md
+++ b/docs/home/howto/DATABASES.md
@@ -577,7 +577,7 @@ specification:
 
 Result of this configuration will be one or more independent nodes of OpenDistro.
 
-By default Kibana is deployed only for `logging` component. If you want to deploy kibana for `opendistro_for_elasticsearch` you have to modify feature mapping. Use below configuration in your initial manifest.
+By default Kibana is deployed only for `logging` component. If you want to deploy Kibana for `opendistro_for_elasticsearch` you have to modify feature mapping. Use below configuration in your manifest.
 ```yaml
 kind: configuration/feature-mapping
 title: "Feature mapping to roles"
@@ -592,7 +592,7 @@ specification:
       - kibana
 ```
 
-Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md). You can change it manually in /etc/filebeat/filebeat.yml if needed.
+Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md).
 
 ## How to start working with Apache Ignite Stateful setup
 
diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md
index 2bad5072ea..a30434e189 100644
--- a/docs/home/howto/LOGGING.md
+++ b/docs/home/howto/LOGGING.md
@@ -1,7 +1,7 @@
 ## Centralized logging setup
 
 For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/).
-In order to enable centralized logging, be sure that `count` property for `logging` feature is greater than 0 in your initial manifest.
+In order to enable centralized logging, be sure that `count` property for `logging` feature is greater than 0 in your configuration manifest.
 
 ```yaml
 kind: epiphany-cluster
@@ -32,20 +32,13 @@ specification:
 ```
 The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0.
 
-Default configuration of `logging` and `opendistro_for_elasticsearch` roles are identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify centralized logging configuration adjust and use following defaults in your manifest:
+Default configuration of `logging` and `opendistro_for_elasticsearch` roles is identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify configuration of centralized logging adjust and use the following defaults in your manifest:
 
 ```yaml
 kind: configuration/logging
 title: Logging Config
 name: default
 specification:
-  versions:
-    RedHat:
-      elasticsearch_oss: "7.3.2"
-      opendistro: "1.3.0*"
-    Debian:
-      elasticsearch_oss: "7.3.2"
-      opendistro: "1.3.0*"
   cluster_name: EpiphanyElastic
   clustered: True
   paths: