diff --git a/CHANGELOG-0.7.md b/CHANGELOG-0.7.md index 4f95624080..a79b82fd71 100644 --- a/CHANGELOG-0.7.md +++ b/CHANGELOG-0.7.md @@ -14,6 +14,7 @@ - [#1398](https://github.com/epiphany-platform/epiphany/issues/1398) - Vault installation fails when using canal/calico network plugin - [#1412](https://github.com/epiphany-platform/epiphany/issues/1412) - Certificate in Vault is also generated or copied even if flag in configuration tls_disable is set to true - [#1408](https://github.com/epiphany-platform/epiphany/issues/1408) - Epiphany does not support upgrades for Kubernetes in HA mode +- [#1347](https://github.com/epiphany-platform/epiphany/issues/1347) - [BUG] Kibana config always points its elasticsearch.hosts to a "logging" VM ### Added diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml index f8d68a5643..0445f586cf 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml @@ -7,12 +7,6 @@ src: filebeat.yml.j2 register: modify_filebeat_yml -- name: Enable auditd module - shell: >- - filebeat modules enable auditd - register: enable_module - changed_when: "'Enabled auditd' in enable_module.stdout" - - name: Set Filebeat to be started after Docker block: - name: Create directory (filebeat.service.d) @@ -37,6 +31,11 @@ - name: Start/restart and enable filebeat service block: + - name: Enable auditd module + command: filebeat modules enable auditd + register: enable_module + changed_when: "'Enabled auditd' in enable_module.stdout" + - name: Restart filebeat service systemd: name: filebeat @@ -51,16 +50,18 @@ state: started enabled: true - - name: Verify Filebeat is running + - name: Verify filebeat is running command: systemctl is-active filebeat changed_when: false - when: - - groups['logging'][0] is defined + retries: 10 + delay: 1 + register: result + until: result is succeeded + when: groups['logging'][0] is defined - name: Stop and disable filebeat service systemd: name: filebeat state: stopped enabled: false - when: - - groups['logging'][0] is undefined \ No newline at end of file + when: groups['logging'][0] is undefined \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml index 8934855b1a..020e45342d 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/main.yml @@ -2,29 +2,15 @@ - name: Include installation tasks include_tasks: install-filebeat.yml -- name: Set facts for filebeat.yml template - block: - - - name: Set flag if logging output exists - set_fact: - logging_output_exists: False - - - name: Set flag if logging output exists - set_fact: - logging_output_exists: True - when: - - groups["logging"] is defined - - groups["logging"] | length > 0 - - - name: Set value for setup.kibana.host - set_fact: - setup_kibana_host: >- - {{ hostvars[groups['kibana'][0]].ansible_default_ipv4.address + ':5601' }} - when: - - groups['kibana'][0] is defined +- name: Set value for setup.kibana.host + set_fact: + setup_kibana_host: >- + {{ hostvars[groups['kibana'][0]].ansible_default_ipv4.address + ':5601' }} + when: + - groups['kibana'][0] is defined - name: Include auditd configuration tasks include_tasks: configure-auditd.yml - name: Include filebeat configuration tasks - include_tasks: configure-filebeat.yml \ No newline at end of file + include_tasks: configure-filebeat.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index dc781da291..357905a5f4 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -198,27 +198,29 @@ setup.template.settings: #================================ Outputs ===================================== # Configure what output to use when sending the data collected by the beat. +# Default Elasticsearch output is array of 'logging' hosts. If not exist, 'hosts' remains empty to manual configuration. #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: # Array of hosts to connect to. -{% if logging_output_exists %} - hosts: -{% for host in groups['logging'] %} - - "https://{{hostvars[host]['ansible_hostname']}}:9200" -{% endfor %} +{% if groups['logging'] is defined %} + hosts: + # Logging hosts: + {% for host in groups['logging'] -%} + - "https://{{hostvars[host]['ansible_hostname']}}:9200" + {% endfor %} + protocol: "https" ssl.verification_mode: none username: logstash password: logstash {% else %} hosts: [] -{% endif %} - - # Optional protocol and basic auth credentials. #protocol: "https" + #ssl.verification_mode: none #username: "elastic" #password: "changeme" +{% endif %} #----------------------------- Logstash output -------------------------------- #output.logstash: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 index c086398d3c..410741063c 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 @@ -19,13 +19,14 @@ server.host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" elasticsearch.hosts: -{% if groups['logging'] is defined %} - {%- for host in groups['logging'] %} +{% if 'logging' in group_names %} + # Logging hosts: + {% for host in groups['logging'] %} - "https://{{hostvars[host]['ansible_hostname']}}:9200" {% endfor %} -{% endif %} -{% if groups['opendistro_for_elasticsearch'] is defined %} - {%- for host in groups['opendistro_for_elasticsearch'] -%} +{% elif 'opendistro_for_elasticsearch' in group_names %} + # Opendistro hosts: + {% for host in groups['opendistro_for_elasticsearch'] %} - "https://{{hostvars[host]['ansible_hostname']}}:9200" {% endfor %} {% endif %} diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index bae586aad5..c5a10989f1 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -543,7 +543,8 @@ DROP EXTENSION IF EXISTS pgaudit; ## How to start working with OpenDistro for Elasticsearch -OpenDistro for Elasticsearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration: +OpenDistro for Elasticsearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). +In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration: ```yaml kind: epiphany-cluster @@ -576,6 +577,23 @@ specification: Result of this configuration will be one or more independent nodes of OpenDistro. +By default Kibana is deployed only for `logging` component. If you want to deploy kibana for `opendistro_for_elasticsearch` you have to modify feature mapping. Use below configuration in your initial manifest. +```yaml +kind: configuration/feature-mapping +title: "Feature mapping to roles" +name: default +specification: + roles_mapping: + opendistro_for_elasticsearch: + - opendistro-for-elasticsearch + - node-exporter + - filebeat + - firewall + - kibana +``` + +Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md). You can change it manually in /etc/filebeat/filebeat.yml if needed. + ## How to start working with Apache Ignite Stateful setup Apache Ignite can be installed in Epiphany if `count` property for `ignite` feature is greater than 0. diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 0cb4f10f78..2bad5072ea 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -1,32 +1,55 @@ ## Centralized logging setup -For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/). In order to enable centralized logging, there is required to use `logging` role on feature mapping level - as of 0.5.0 this is a default configuration. +For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/). +In order to enable centralized logging, be sure that `count` property for `logging` feature is greater than 0 in your initial manifest. +```yaml +kind: epiphany-cluster +... +specification: + ... + components: + kubernetes_master: + count: 1 + kubernetes_node: + count: 0 + ... + logging: + count: 1 + ... +``` + +### Default feature mapping for logging: ```yaml ... logging: - logging - kibana + - node-exporter - filebeat - firewall ... ``` +The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0. -The `logging` role replaced `elasticsearch` role in logging feature. This change was done to enable Elasticsearch usage for data storage - not only for logs as it was till 0.5.0. - -Default configuration of `logging` role is the same as [opendistro_for_elasticsearch](./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) which is used for logs storage. In order to modify centralized logging configuration adjust and use following defaults: +Default configuration of `logging` and `opendistro_for_elasticsearch` roles are identical (./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify centralized logging configuration adjust and use following defaults in your manifest: ```yaml kind: configuration/logging title: Logging Config name: default specification: - opendistro_version_redhat: "1.3.0" - elasticsearch_oss_version_debian: "7.3.2" - opendistro_version_debian: "1.3.0*" + versions: + RedHat: + elasticsearch_oss: "7.3.2" + opendistro: "1.3.0*" + Debian: + elasticsearch_oss: "7.3.2" + opendistro: "1.3.0*" cluster_name: EpiphanyElastic clustered: True paths: data: /var/lib/elasticsearch + repo: /var/lib/elasticsearch-snapshots logs: /var/log/elasticsearch -``` \ No newline at end of file +```