OpenSearch: add dedicated user for Filebeat (#3079) (#3221)

* removes previously used `logstash` user from filebeat configuration
* removes `logstash` user from demo users configured by opensearch
* enables creation of dedicated filebeat user - by default name set to
  `filebeatservice`
* add user detection in case of re-apply
* set user `kibanaserver` and `filebeatservice` installation
  dependent on inventory groups rather than user_active
  flag (previously configured by users)
* simplify documentation
* set dashboards hosts list based on their group
* use yaml anchors in user manipulation tasks

ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2

@@ -314,10 +314,11 @@ output.elasticsearch:
   {% endfor %}
 
   # Authentication credentials - either API key or username/password.
-  username: logstash
   {% if not is_upgrade_run %}
-  password: {{ "'%s'" % logging_vars.specification.logstash_password | replace("'","''") }}
+  username: filebeatservice
+  password: {{ "'%s'" % logging_vars.specification.filebeatservice_password | replace("'","''") }}
   {% else %}
+  username: logstash
   password: {{ "'%s'" % existing_output_es_password | replace("'","''") }}
   {% endif %}
 

ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml

@@ -195,15 +195,17 @@
         method: GET
         status_code: [200, 404]
       register: kibanaserver_check_response
-      when: specification.kibanaserver_user_active
+      when:
+        - groups.opensearch_dashboards[0] is defined
+        - inventory_hostname in groups.opensearch_dashboards
 
     - name: Create default kibanaserver user
       uri:
         <<: *uri
         url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/kibanaserver"
         method: PUT
         status_code: [200]
-        body:
+        body: &kibanaserver_data
           password: "{{ specification.kibanaserver_password }}"
           reserved: "true"
           description: "Demo OpenSearch Dashboards user"
@@ -213,7 +215,7 @@
       delay: 1
       run_once: true
       when:
-        - kibanaserver_check_response is defined
+        - kibanaserver_check_response.status is defined
         - kibanaserver_check_response.status == 404
 
     - name: Set kibanaserver user password
@@ -226,69 +228,67 @@
           - op: "replace"
             path: "/kibanaserver"
             value:
-              password: "{{ specification.kibanaserver_password }}"
-              reserved: "true"
-              description: "Demo OpenSearch Dashboards user"
+              <<: *kibanaserver_data
       register: uri_response
       until: uri_response is success
       retries: 15
       delay: 1
       run_once: true
       when:
-        - kibanaserver_check_response is defined
+        - kibanaserver_check_response.status is defined
         - kibanaserver_check_response.status == 200
 
-    - name: Check if default logstash user exists
+    - name: Check if filebeatservice user exists  # for re-apply scenario
       uri:
         <<: *uri
-        url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash"
+        url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/filebeatservice"
         method: GET
         status_code: [200, 404]
-      register: logstash_check_response
-      when: specification.logstash_user_active
+      register: filebeatservice_check_response
+      when:
+        - groups.logging[0] is defined
+        - inventory_hostname in groups.logging
 
-    - name: Create default logstash user
+    - name: Create dedicated filebeatservice user
       uri:
         <<: *uri
-        url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash"
+        url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/filebeatservice"
         method: PUT
         status_code: [200]
-        body:
-          password: "{{ specification.logstash_password }}"
+        body: &filebeatservice_data
+          password: "{{ specification.filebeatservice_password }}"
           reserved: "true"
-          description: "OpenSearch logstash user"
+          backend_roles:
+            - "logstash"
+          description: "Epiphany user for Filebeat service"
       register: uri_response
       until: uri_response is success
       retries: 5
       delay: 1
       run_once: true
       when:
-        - logstash_check_response is defined
-        - logstash_check_response.status == 404
+        - filebeatservice_check_response.status is defined
+        - filebeatservice_check_response.status == 404
 
-    - name: Set OpenSearch logstash user password
+    - name: Set filebeatservice user password
       uri:
         <<: *uri
         url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/"
         method: PATCH
         status_code: [200]
         body:
           - op: "replace"
-            path: "/logstash"
+            path: "/filebeatservice"
             value:
-              password: "{{ specification.logstash_password }}"
-              reserved: "true"
-              backend_roles:
-                - "logstash"
-              description: "OpenSearch logstash user"
+              <<: *filebeatservice_data
       register: uri_response
       until: uri_response is success
-      retries: 3
-      delay: 5
+      retries: 15
+      delay: 1
       run_once: true
       when:
-        - logstash_check_response is defined
-        - kibanaserver_check_response.status == 200
+        - filebeatservice_check_response.status is defined
+        - filebeatservice_check_response.status == 200
 
     - name: Remove OpenSearch demo users
       uri:

ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml

@@ -28,12 +28,16 @@
     extra_opts:
       - --strip-components=1
 
+# if opensearch-dashboards is enabled for groups 'logging' and 'opensearch', form dashboards cluster
+# on the basis of belonging to a given group
 - name: Set opensearch dashboards hosts as fact
   set_fact:
     opensearch_nodes_dashboards: |-
-        {% for item in groups['opensearch_dashboards'] -%}
-          https://{{ item }}:{{ opensearch_api_port }}{% if not loop.last %}","{% endif %}
-        {%- endfor %}
+      {%- set current_host_group = groups[(group_names | intersect(['logging', 'opensearch'])) | first] -%}
+      {%- set hosts = groups['opensearch_dashboards'] | intersect(current_host_group) -%}
+      {%- for item in hosts -%}
+        https://{{ item }}:{{ opensearch_api_port }}{%- if not loop.last -%}","{%- endif -%}
+      {%- endfor -%}
 
 - name: Copy configuration file
   template:

docs/changelogs/CHANGELOG-2.0.md

@@ -14,6 +14,7 @@
 - [#3106](https://github.com/epiphany-platform/epiphany/issues/3106) - Add image-registry configuration reading
 - [#3140](https://github.com/epiphany-platform/epiphany/issues/3140) - Allow to disable OpenSearch audit logs
 - [#3218](https://github.com/epiphany-platform/epiphany/issues/3218) - Add support for original output coloring
+- [#3079](https://github.com/epiphany-platform/epiphany/issues/3079) - OpenSearch improvement - add dedicated user for Filebeat
 
 ### Fixed
 

docs/home/howto/DATABASES.md

@@ -486,8 +486,10 @@ specification:
   cluster_name: EpiphanyOpenSearch
 ```
 
-By default, OpenSearch Dashboards ( previously Kibana component )  is deployed only for `logging` component. If you want to deploy it
-for `opensearch` component you have to modify feature mapping. Use below configuration in your manifest:
+By default, OpenSearch Dashboards (previously Kibana) is deployed only for `logging` component. If you want to deploy it
+for `opensearch` component you have to:
+- modify feature mapping by adding `opensearch-dashboards` under `opensearch` component (see configuration below)
+- set up `kibanaserver` user and its password in `configuration/opensearch`, see [Opensearch user and password configuration](./MONITORING.md#opensearch-component)
 
 ```yaml
 kind: configuration/feature-mappings

docs/home/howto/MONITORING.md

@@ -250,21 +250,29 @@ This filter pattern can now be used to query the OpenSsearch indices.
 
 By default OpenSearch Dashoboards adjusts the UTC time in `@timestamp` to the browser's local timezone. This can be changed in `Stack Management` > `Advanced Settings` > `Timezone for date formatting`.
 
-## How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat 
+## How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat
+
+Epiphany provides two componenets that include OpenSearch installation: `logging` (by default includes OpenSearch-Dashboards as well) and `opensearch`.
+In order to learn more about both components, please look through documentation:
+- [logging](./LOGGING.md#centralized-logging-setup)
+- [opensearch](./DATABASES.md#how-to-start-working-with-opensearch)
 
 To configure admin password for OpenSearch Dashoboards ( previously Kibana ) and OpenSearch you need to follow the procedure below.
-There are separate procedures for `logging` and `opensearch` roles since for most of the time `opensearch`, `kibanaserver` and `logstash` users are not required to be present.
 
 ### Logging component
 
 #### Logging role
 
-By default Epiphany removes users that are listed in `demo_users_to_remove` section of `configuration/logging` manifest document.
-Additionally, `kibanaserver`<sup>[1]</sup> user (needed by default Epiphany installation of Dashboards) and `logstash` user (needed by default Epiphany installation of Filebeat) are not removed. If you want to perform configuration by Epiphany, set `kibanaserver_user_active` to `true`
-for `kibanaserver` user and/or `logstash_user_active` to `true` for `logstash` user. For `logging` role, those settings are already set to `true` by default.
-We strongly advice to set different password for each user.
+Default users configured by Epiphany for `logging` role are:
+- `kibanaserver`<sup>[1]</sup> - needed by default Epiphany installation of Dashboards
+- `filebeatservice` - needed by default Epiphany installation of Filebeat
+Note that `logstash` user from earlier versions of Epiphany, has been replaced by dedicated `filebeatservice` user.
+
+**We strongly advice to set different password for each user.**
+
+Additionally, Epiphany removes users that are listed in `demo_users_to_remove` section of `configuration/logging` manifest document.
 
-To change `admin` user's password, you need to change the value for `admin_password` key ( see the example below ). For `kibanaserver` and `logstash`, you need to change values for `kibanaserver_password` and `logstash_password` keys respectively. Changes from logging role will be propagated to OpenSearch Dashboards and Filebeat configuration accordingly.
+To change `admin` user's password, you need to change the value for `admin_password` key ( see the example below ). For `kibanaserver` and `filebeatservice`, you need to change values for `kibanaserver_password` and `filebeatservice_password` keys respectively. Changes from logging role will be propagated to OpenSearch Dashboards and Filebeat configuration accordingly.
 
 ```yaml
 kind: configuration/logging
@@ -274,29 +282,23 @@ specification:
   [...]
   admin_password: YOUR_PASSWORD
   kibanaserver_password: YOUR_PASSWORD
-  kibanaserver_user_active: true
-  logstash_password: YOUR_PASSWORD
-  logstash_user_active: true
+  filebeatservice_password: PASSWORD_TO_CHANGE
   demo_users_to_remove:
   - kibanaro
   - readall
+  - logstash
   - snapshotrestore
 ```
 
-#### OpenSearch Dashboards ( Kibana ) role
-
-To set password for `kibanaserver` user, which is used by Dashboards for communication with OpenSearch Dashboards backend follow the procedure described in [Logging role](#logging-role).
-
-#### Filebeat role
-
-To set password of `logstash` user, which is used by Filebeat for communication with OpenSearch Dashboards backend follow the procedure described in  [Logging role](#-logging-role).
-
 ### OpenSearch component
 
-By default Epiphany removes all demo users except `admin` user. Those users are listed in `demo_users_to_remove` section of `configuration/opensearch` manifest doc ( see example below ). If you want to keep `kibanaserver` user (needed by default Epiphany installation of OpenSearch Dashboards), you need to exclude it from `demo_users_to_remove` list and set `kibanaserver_user_active` to `true` in order to change the default password.
-We strongly advice to set different password for each user.
+Default user provided by Epiphany for OpenSearch role is `admin`. Additionally, Epiphany removes all demo users except `admin` user.
+Those users are listed in `demo_users_to_remove` section of `configuration/opensearch` manifest doc ( see example below ).
+To change `admin` user's password, change value for the `admin_password` key.
 
-To change `admin` user's password, change value for the `admin_password` key. For `kibanaserver` and `logstash`, change values for `kibanaserver_password` and `logstash_password` keys respectively.
+**We strongly advice to set different password for admin user.**
+
+Note that adding `opensearch-dashboards` mapping in `configuration/feature-mappings` under `opensearch` component requires commenting out `kibanaserver` user in `demo_users_to_remove` section (as presented in configuration below). This step should be followed by changing default password for `kibanaserver` user by modifying value for `kibanaserver_password` key.
 
 ```yaml
 kind: configuration/opensearch
@@ -305,16 +307,13 @@ name: default
 specification:
   [...]
   admin_password: YOUR_PASSWORD
-  kibanaserver_password: YOUR_PASSWORD
-  kibanaserver_user_active: false
-  logstash_password: YOUR_PASSWORD
-  logstash_user_active: false
+  kibanaserver_password: YOUR_PASSWPRD
   demo_users_to_remove:
   - kibanaro
   - readall
   - snapshotrestore
   - logstash
-  - kibanaserver
+  # - kibanaserver
 ```
 
 ### Upgrade of OpenSearch, OpenSearch Dashboards and Filebeat

schema/common/defaults/configuration/logging.yml

@@ -8,15 +8,12 @@ specification:
   opensearch_os_group: opensearch
   admin_password: PASSWORD_TO_CHANGE
   kibanaserver_password: PASSWORD_TO_CHANGE
-  kibanaserver_user_active: true
-  logstash_password: PASSWORD_TO_CHANGE
-  logstash_user_active: true
+  filebeatservice_password: PASSWORD_TO_CHANGE
   demo_users_to_remove:
     - kibanaro
     - readall
     - snapshotrestore
-    # - logstash
-    # - kibanaserver
+    - logstash
   paths:
     opensearch_home: /usr/share/opensearch
     opensearch_conf_dir: /usr/share/opensearch/config

schema/common/defaults/configuration/opensearch.yml

@@ -8,15 +8,12 @@ specification:
   opensearch_os_group: opensearch
   admin_password: PASSWORD_TO_CHANGE
   kibanaserver_password: PASSWORD_TO_CHANGE
-  kibanaserver_user_active: true
-  logstash_password: PASSWORD_TO_CHANGE
-  logstash_user_active: true
   demo_users_to_remove:
     - kibanaro
     - readall
     - snapshotrestore
-    # - logstash
-    # - kibanaserver
+    - logstash
+    - kibanaserver
   paths:
     opensearch_home: /usr/share/opensearch
     opensearch_conf_dir: /usr/share/opensearch/config

schema/common/validation/configuration/logging.yml

@@ -14,12 +14,8 @@ properties:
     type: string
   kibanaserver_password:
     type: string
-  kibanaserver_user_active:
-    type: boolean
-  logstash_password:
+  filebeatservice_password:
     type: string
-  logstash_user_active:
-    type: boolean
   demo_users_to_remove:
     type: array
     items:

schema/common/validation/configuration/opensearch.yml

@@ -14,12 +14,6 @@ properties:
     type: string
   kibanaserver_password:
     type: string
-  kibanaserver_user_active:
-    type: boolean
-  logstash_password:
-    type: string
-  logstash_user_active:
-    type: boolean
   demo_users_to_remove:
     type: array
     items:

tests/spec/spec/filebeat/filebeat_spec.rb

@@ -8,13 +8,13 @@
 
 # Configurable passwords for ES users were introduced in v0.10.0.
 # For testing upgrades, we use default passwords for now but they should be read from filebeat.yml (remote host).
-es_logstash_user_password  = readDataYaml('configuration/logging')['specification']['logstash_password'] || 'logstash'
-es_logstash_user_is_active = readDataYaml('configuration/logging')['specification']['logstash_user_active']
-es_logstash_user_is_active = true if es_logstash_user_is_active.nil?
+es_filebeat_user_password  = readDataYaml('configuration/logging')['specification']['filebeatservice_password'] || 'PASSWORD_TO_CHANGE'
+es_filebeat_user_is_active = !listInventoryHosts('logging').empty?
+
+filebeat_user = upgradeRun? ? 'logstash' : 'filebeatservice'
 
 es_kibanaserver_user_password  = readDataYaml('configuration/logging')['specification']['kibanaserver_password'] || 'kibanaserver'
-es_kibanaserver_user_is_active = readDataYaml('configuration/logging')['specification']['kibanaserver_user_active']
-es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil?
+es_kibanaserver_user_is_active = !listInventoryHosts('logging').empty?
 
 es_api_port     = 9200
 kibana_api_port = 5601
@@ -44,11 +44,11 @@
   end
 end
 
-if es_logstash_user_is_active
+if es_filebeat_user_is_active
   listInventoryHosts('logging').each do |val|
     describe 'Check the connection to the Elasticsearch hosts' do
       let(:disable_sudo) { false }
-      describe command("curl -k -u logstash:#{es_logstash_user_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_api_port}") do
+      describe command("curl -k -u #{filebeat_user}:#{es_filebeat_user_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_api_port}") do
         it 'is expected to be equal' do
           expect(subject.stdout.to_i).to eq 200
         end

tests/spec/spec/logging/logging_spec.rb

@@ -1,9 +1,7 @@
 require 'spec_helper'
 # Configurable passwords for ES users were introduced in v0.10.0.
 # For testing upgrades, we use the default password for now but we're going to switch to TLS auth.
-es_kibanaserver_user_password  = readDataYaml('configuration/logging')['specification']['kibanaserver_password'] || 'kibanaserver'
-es_kibanaserver_user_is_active = readDataYaml('configuration/logging')['specification']['kibanaserver_user_active']
-es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil?
+es_kibanaserver_user_password = readDataYaml('configuration/logging')['specification']['kibanaserver_password'] || 'kibanaserver'
 es_admin_password = readDataYaml('configuration/logging')['specification']['admin_password'] || 'admin'
 es_rest_api_port  = 9200
 es_transport_port = 9300

tests/spec/spec/opensearch/opensearch_spec.rb

@@ -1,9 +1,6 @@
 require 'spec_helper'
 # Configurable passwords for ES users were introduced in v0.10.0.
 # For testing upgrades, we use the default password for now but we're going to switch to TLS auth.
-es_kibanaserver_user_password  = readDataYaml('configuration/opensearch')['specification']['kibanaserver_password'] || 'kibanaserver'
-es_kibanaserver_user_is_active = readDataYaml('configuration/opensearch')['specification']['kibanaserver_user_active']
-es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil?
 es_admin_password = readDataYaml('configuration/opensearch')['specification']['admin_password'] || 'admin'
 es_rest_api_port  = 9200
 es_transport_port = 9300