diff --git a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index 70dc3f92dc..f886e51a75 100644 --- a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -314,10 +314,11 @@ output.elasticsearch: {% endfor %} # Authentication credentials - either API key or username/password. - username: logstash {% if not is_upgrade_run %} - password: {{ "'%s'" % logging_vars.specification.logstash_password | replace("'","''") }} + username: filebeatservice + password: {{ "'%s'" % logging_vars.specification.filebeatservice_password | replace("'","''") }} {% else %} + username: logstash password: {{ "'%s'" % existing_output_es_password | replace("'","''") }} {% endif %} diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml index c75697d47d..205275397e 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml @@ -195,7 +195,9 @@ method: GET status_code: [200, 404] register: kibanaserver_check_response - when: specification.kibanaserver_user_active + when: + - groups.opensearch_dashboards[0] is defined + - inventory_hostname in groups.opensearch_dashboards - name: Create default kibanaserver user uri: @@ -203,7 +205,7 @@ url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/kibanaserver" method: PUT status_code: [200] - body: + body: &kibanaserver_data password: "{{ specification.kibanaserver_password }}" reserved: "true" description: "Demo OpenSearch Dashboards user" @@ -213,7 +215,7 @@ delay: 1 run_once: true when: - - kibanaserver_check_response is defined + - kibanaserver_check_response.status is defined - kibanaserver_check_response.status == 404 - name: Set kibanaserver user password @@ -226,47 +228,49 @@ - op: "replace" path: "/kibanaserver" value: - password: "{{ specification.kibanaserver_password }}" - reserved: "true" - description: "Demo OpenSearch Dashboards user" + <<: *kibanaserver_data register: uri_response until: uri_response is success retries: 15 delay: 1 run_once: true when: - - kibanaserver_check_response is defined + - kibanaserver_check_response.status is defined - kibanaserver_check_response.status == 200 - - name: Check if default logstash user exists + - name: Check if filebeatservice user exists # for re-apply scenario uri: <<: *uri - url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash" + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/filebeatservice" method: GET status_code: [200, 404] - register: logstash_check_response - when: specification.logstash_user_active + register: filebeatservice_check_response + when: + - groups.logging[0] is defined + - inventory_hostname in groups.logging - - name: Create default logstash user + - name: Create dedicated filebeatservice user uri: <<: *uri - url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash" + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/filebeatservice" method: PUT status_code: [200] - body: - password: "{{ specification.logstash_password }}" + body: &filebeatservice_data + password: "{{ specification.filebeatservice_password }}" reserved: "true" - description: "OpenSearch logstash user" + backend_roles: + - "logstash" + description: "Epiphany user for Filebeat service" register: uri_response until: uri_response is success retries: 5 delay: 1 run_once: true when: - - logstash_check_response is defined - - logstash_check_response.status == 404 + - filebeatservice_check_response.status is defined + - filebeatservice_check_response.status == 404 - - name: Set OpenSearch logstash user password + - name: Set filebeatservice user password uri: <<: *uri url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" @@ -274,21 +278,17 @@ status_code: [200] body: - op: "replace" - path: "/logstash" + path: "/filebeatservice" value: - password: "{{ specification.logstash_password }}" - reserved: "true" - backend_roles: - - "logstash" - description: "OpenSearch logstash user" + <<: *filebeatservice_data register: uri_response until: uri_response is success - retries: 3 - delay: 5 + retries: 15 + delay: 1 run_once: true when: - - logstash_check_response is defined - - kibanaserver_check_response.status == 200 + - filebeatservice_check_response.status is defined + - filebeatservice_check_response.status == 200 - name: Remove OpenSearch demo users uri: diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index d05f8c27e1..a59541c278 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -28,12 +28,16 @@ extra_opts: - --strip-components=1 +# if opensearch-dashboards is enabled for groups 'logging' and 'opensearch', form dashboards cluster +# on the basis of belonging to a given group - name: Set opensearch dashboards hosts as fact set_fact: opensearch_nodes_dashboards: |- - {% for item in groups['opensearch_dashboards'] -%} - https://{{ item }}:{{ opensearch_api_port }}{% if not loop.last %}","{% endif %} - {%- endfor %} + {%- set current_host_group = groups[(group_names | intersect(['logging', 'opensearch'])) | first] -%} + {%- set hosts = groups['opensearch_dashboards'] | intersect(current_host_group) -%} + {%- for item in hosts -%} + https://{{ item }}:{{ opensearch_api_port }}{%- if not loop.last -%}","{%- endif -%} + {%- endfor -%} - name: Copy configuration file template: diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index a8ec7aa340..d5b476a92c 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -14,6 +14,7 @@ - [#3106](https://github.com/epiphany-platform/epiphany/issues/3106) - Add image-registry configuration reading - [#3140](https://github.com/epiphany-platform/epiphany/issues/3140) - Allow to disable OpenSearch audit logs - [#3218](https://github.com/epiphany-platform/epiphany/issues/3218) - Add support for original output coloring +- [#3079](https://github.com/epiphany-platform/epiphany/issues/3079) - OpenSearch improvement - add dedicated user for Filebeat ### Fixed diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index c0df337025..bd03185fdd 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -486,8 +486,10 @@ specification: cluster_name: EpiphanyOpenSearch ``` -By default, OpenSearch Dashboards ( previously Kibana component ) is deployed only for `logging` component. If you want to deploy it -for `opensearch` component you have to modify feature mapping. Use below configuration in your manifest: +By default, OpenSearch Dashboards (previously Kibana) is deployed only for `logging` component. If you want to deploy it +for `opensearch` component you have to: +- modify feature mapping by adding `opensearch-dashboards` under `opensearch` component (see configuration below) +- set up `kibanaserver` user and its password in `configuration/opensearch`, see [Opensearch user and password configuration](./MONITORING.md#opensearch-component) ```yaml kind: configuration/feature-mappings diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index 4516657a35..a5181bfd75 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -250,21 +250,29 @@ This filter pattern can now be used to query the OpenSsearch indices. By default OpenSearch Dashoboards adjusts the UTC time in `@timestamp` to the browser's local timezone. This can be changed in `Stack Management` > `Advanced Settings` > `Timezone for date formatting`. -## How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat +## How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat + +Epiphany provides two componenets that include OpenSearch installation: `logging` (by default includes OpenSearch-Dashboards as well) and `opensearch`. +In order to learn more about both components, please look through documentation: +- [logging](./LOGGING.md#centralized-logging-setup) +- [opensearch](./DATABASES.md#how-to-start-working-with-opensearch) To configure admin password for OpenSearch Dashoboards ( previously Kibana ) and OpenSearch you need to follow the procedure below. -There are separate procedures for `logging` and `opensearch` roles since for most of the time `opensearch`, `kibanaserver` and `logstash` users are not required to be present. ### Logging component #### Logging role -By default Epiphany removes users that are listed in `demo_users_to_remove` section of `configuration/logging` manifest document. -Additionally, `kibanaserver`[1] user (needed by default Epiphany installation of Dashboards) and `logstash` user (needed by default Epiphany installation of Filebeat) are not removed. If you want to perform configuration by Epiphany, set `kibanaserver_user_active` to `true` -for `kibanaserver` user and/or `logstash_user_active` to `true` for `logstash` user. For `logging` role, those settings are already set to `true` by default. -We strongly advice to set different password for each user. +Default users configured by Epiphany for `logging` role are: +- `kibanaserver`[1] - needed by default Epiphany installation of Dashboards +- `filebeatservice` - needed by default Epiphany installation of Filebeat +Note that `logstash` user from earlier versions of Epiphany, has been replaced by dedicated `filebeatservice` user. + +**We strongly advice to set different password for each user.** + +Additionally, Epiphany removes users that are listed in `demo_users_to_remove` section of `configuration/logging` manifest document. -To change `admin` user's password, you need to change the value for `admin_password` key ( see the example below ). For `kibanaserver` and `logstash`, you need to change values for `kibanaserver_password` and `logstash_password` keys respectively. Changes from logging role will be propagated to OpenSearch Dashboards and Filebeat configuration accordingly. +To change `admin` user's password, you need to change the value for `admin_password` key ( see the example below ). For `kibanaserver` and `filebeatservice`, you need to change values for `kibanaserver_password` and `filebeatservice_password` keys respectively. Changes from logging role will be propagated to OpenSearch Dashboards and Filebeat configuration accordingly. ```yaml kind: configuration/logging @@ -274,29 +282,23 @@ specification: [...] admin_password: YOUR_PASSWORD kibanaserver_password: YOUR_PASSWORD - kibanaserver_user_active: true - logstash_password: YOUR_PASSWORD - logstash_user_active: true + filebeatservice_password: PASSWORD_TO_CHANGE demo_users_to_remove: - kibanaro - readall + - logstash - snapshotrestore ``` -#### OpenSearch Dashboards ( Kibana ) role - -To set password for `kibanaserver` user, which is used by Dashboards for communication with OpenSearch Dashboards backend follow the procedure described in [Logging role](#logging-role). - -#### Filebeat role - -To set password of `logstash` user, which is used by Filebeat for communication with OpenSearch Dashboards backend follow the procedure described in [Logging role](#-logging-role). - ### OpenSearch component -By default Epiphany removes all demo users except `admin` user. Those users are listed in `demo_users_to_remove` section of `configuration/opensearch` manifest doc ( see example below ). If you want to keep `kibanaserver` user (needed by default Epiphany installation of OpenSearch Dashboards), you need to exclude it from `demo_users_to_remove` list and set `kibanaserver_user_active` to `true` in order to change the default password. -We strongly advice to set different password for each user. +Default user provided by Epiphany for OpenSearch role is `admin`. Additionally, Epiphany removes all demo users except `admin` user. +Those users are listed in `demo_users_to_remove` section of `configuration/opensearch` manifest doc ( see example below ). +To change `admin` user's password, change value for the `admin_password` key. -To change `admin` user's password, change value for the `admin_password` key. For `kibanaserver` and `logstash`, change values for `kibanaserver_password` and `logstash_password` keys respectively. +**We strongly advice to set different password for admin user.** + +Note that adding `opensearch-dashboards` mapping in `configuration/feature-mappings` under `opensearch` component requires commenting out `kibanaserver` user in `demo_users_to_remove` section (as presented in configuration below). This step should be followed by changing default password for `kibanaserver` user by modifying value for `kibanaserver_password` key. ```yaml kind: configuration/opensearch @@ -305,16 +307,13 @@ name: default specification: [...] admin_password: YOUR_PASSWORD - kibanaserver_password: YOUR_PASSWORD - kibanaserver_user_active: false - logstash_password: YOUR_PASSWORD - logstash_user_active: false + kibanaserver_password: YOUR_PASSWPRD demo_users_to_remove: - kibanaro - readall - snapshotrestore - logstash - - kibanaserver + # - kibanaserver ``` ### Upgrade of OpenSearch, OpenSearch Dashboards and Filebeat diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 0842a0883f..5adfa60107 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -8,15 +8,12 @@ specification: opensearch_os_group: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE - kibanaserver_user_active: true - logstash_password: PASSWORD_TO_CHANGE - logstash_user_active: true + filebeatservice_password: PASSWORD_TO_CHANGE demo_users_to_remove: - kibanaro - readall - snapshotrestore - # - logstash - # - kibanaserver + - logstash paths: opensearch_home: /usr/share/opensearch opensearch_conf_dir: /usr/share/opensearch/config diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 86daf5a804..7ebaf1390c 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -8,15 +8,12 @@ specification: opensearch_os_group: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE - kibanaserver_user_active: true - logstash_password: PASSWORD_TO_CHANGE - logstash_user_active: true demo_users_to_remove: - kibanaro - readall - snapshotrestore - # - logstash - # - kibanaserver + - logstash + - kibanaserver paths: opensearch_home: /usr/share/opensearch opensearch_conf_dir: /usr/share/opensearch/config diff --git a/schema/common/validation/configuration/logging.yml b/schema/common/validation/configuration/logging.yml index cd793a50d2..72d1c92d4e 100644 --- a/schema/common/validation/configuration/logging.yml +++ b/schema/common/validation/configuration/logging.yml @@ -14,12 +14,8 @@ properties: type: string kibanaserver_password: type: string - kibanaserver_user_active: - type: boolean - logstash_password: + filebeatservice_password: type: string - logstash_user_active: - type: boolean demo_users_to_remove: type: array items: diff --git a/schema/common/validation/configuration/opensearch.yml b/schema/common/validation/configuration/opensearch.yml index dce0ba1036..ad3ebcd0b8 100644 --- a/schema/common/validation/configuration/opensearch.yml +++ b/schema/common/validation/configuration/opensearch.yml @@ -14,12 +14,6 @@ properties: type: string kibanaserver_password: type: string - kibanaserver_user_active: - type: boolean - logstash_password: - type: string - logstash_user_active: - type: boolean demo_users_to_remove: type: array items: diff --git a/tests/spec/spec/filebeat/filebeat_spec.rb b/tests/spec/spec/filebeat/filebeat_spec.rb index d4783ae5f2..31db49c081 100644 --- a/tests/spec/spec/filebeat/filebeat_spec.rb +++ b/tests/spec/spec/filebeat/filebeat_spec.rb @@ -8,13 +8,13 @@ # Configurable passwords for ES users were introduced in v0.10.0. # For testing upgrades, we use default passwords for now but they should be read from filebeat.yml (remote host). -es_logstash_user_password = readDataYaml('configuration/logging')['specification']['logstash_password'] || 'logstash' -es_logstash_user_is_active = readDataYaml('configuration/logging')['specification']['logstash_user_active'] -es_logstash_user_is_active = true if es_logstash_user_is_active.nil? +es_filebeat_user_password = readDataYaml('configuration/logging')['specification']['filebeatservice_password'] || 'PASSWORD_TO_CHANGE' +es_filebeat_user_is_active = !listInventoryHosts('logging').empty? + +filebeat_user = upgradeRun? ? 'logstash' : 'filebeatservice' es_kibanaserver_user_password = readDataYaml('configuration/logging')['specification']['kibanaserver_password'] || 'kibanaserver' -es_kibanaserver_user_is_active = readDataYaml('configuration/logging')['specification']['kibanaserver_user_active'] -es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil? +es_kibanaserver_user_is_active = !listInventoryHosts('logging').empty? es_api_port = 9200 kibana_api_port = 5601 @@ -44,11 +44,11 @@ end end -if es_logstash_user_is_active +if es_filebeat_user_is_active listInventoryHosts('logging').each do |val| describe 'Check the connection to the Elasticsearch hosts' do let(:disable_sudo) { false } - describe command("curl -k -u logstash:#{es_logstash_user_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_api_port}") do + describe command("curl -k -u #{filebeat_user}:#{es_filebeat_user_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_api_port}") do it 'is expected to be equal' do expect(subject.stdout.to_i).to eq 200 end diff --git a/tests/spec/spec/logging/logging_spec.rb b/tests/spec/spec/logging/logging_spec.rb index 54fa551981..da2b9997e6 100644 --- a/tests/spec/spec/logging/logging_spec.rb +++ b/tests/spec/spec/logging/logging_spec.rb @@ -1,9 +1,7 @@ require 'spec_helper' # Configurable passwords for ES users were introduced in v0.10.0. # For testing upgrades, we use the default password for now but we're going to switch to TLS auth. -es_kibanaserver_user_password = readDataYaml('configuration/logging')['specification']['kibanaserver_password'] || 'kibanaserver' -es_kibanaserver_user_is_active = readDataYaml('configuration/logging')['specification']['kibanaserver_user_active'] -es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil? +es_kibanaserver_user_password = readDataYaml('configuration/logging')['specification']['kibanaserver_password'] || 'kibanaserver' es_admin_password = readDataYaml('configuration/logging')['specification']['admin_password'] || 'admin' es_rest_api_port = 9200 es_transport_port = 9300 diff --git a/tests/spec/spec/opensearch/opensearch_spec.rb b/tests/spec/spec/opensearch/opensearch_spec.rb index 8138c69389..41ea711aa0 100644 --- a/tests/spec/spec/opensearch/opensearch_spec.rb +++ b/tests/spec/spec/opensearch/opensearch_spec.rb @@ -1,9 +1,6 @@ require 'spec_helper' # Configurable passwords for ES users were introduced in v0.10.0. # For testing upgrades, we use the default password for now but we're going to switch to TLS auth. -es_kibanaserver_user_password = readDataYaml('configuration/opensearch')['specification']['kibanaserver_password'] || 'kibanaserver' -es_kibanaserver_user_is_active = readDataYaml('configuration/opensearch')['specification']['kibanaserver_user_active'] -es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil? es_admin_password = readDataYaml('configuration/opensearch')['specification']['admin_password'] || 'admin' es_rest_api_port = 9200 es_transport_port = 9300