From dbff14bd0fe47c5f783cd579af682de01cd8240a Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 20 Jan 2022 16:23:31 +0100 Subject: [PATCH 001/157] Move to Opensearch --- ansible/playbooks/filebeat.yml | 2 +- .../opendistro_for_elasticsearch.yml | 6 +-- .../tasks/logging_elasticsearch_snapshot.yml | 4 +- .../elasticsearch_curator/tasks/main.yml | 2 +- .../filebeat/tasks/configure-filebeat.yml | 2 +- ansible/playbooks/roles/kibana/tasks/main.yml | 8 ++-- .../roles/kibana/templates/kibana.yml.j2 | 6 +-- .../playbooks/roles/logging/tasks/main.yml | 4 +- .../tasks/install-es.yml | 14 ------ .../tasks/install-opendistro.yml | 25 ---------- .../defaults/main.yml | 7 ++- .../meta/main.yml | 0 .../tasks/configure-ops.yml} | 2 +- .../tasks/generate-certs.yml | 0 .../opensearch/tasks/install-ops-plugins.yml | 25 ++++++++++ .../roles/opensearch/tasks/install-ops.yml | 48 +++++++++++++++++++ .../tasks/main.yml | 6 +-- .../tasks/patch-log4j.yml | 14 +++--- .../tasks/remove-demo-certs.yml | 0 .../tasks/remove-known-demo-certs.yml | 0 .../templates/jvm.options.j2 | 0 .../templates/opensearch.yml.j2} | 0 .../roles/preflight/defaults/main.yml | 4 +- .../tasks/logging_elasticsearch_snapshot.yml | 4 +- .../centos-7/requirements.aarch64.txt | 12 ++--- .../centos-7/requirements.x86_64.txt | 12 ++--- .../redhat-7/requirements.x86_64.txt | 12 ++--- .../ubuntu-20.04/requirements.x86_64.txt | 12 ++--- .../playbooks/roles/upgrade/defaults/main.yml | 2 +- .../upgrade/tasks/elasticsearch-curator.yml | 2 +- .../tasks/opendistro_for_elasticsearch-01.yml | 10 ++-- .../tasks/opendistro_for_elasticsearch-02.yml | 4 +- .../upgrade-plugins.yml | 18 ------- .../migrate-from-demo-certs-01.yml | 10 ++-- .../migrate-from-demo-certs-02.yml | 0 .../migrate-from-demo-certs-non-clustered.yml | 0 .../upgrade-opensearch-01.yml} | 4 +- .../upgrade-opensearch-02.yml} | 38 +++++++-------- .../tasks/opensearch/upgrade-plugins.yml | 18 +++++++ .../utils/assert-api-access.yml | 0 .../utils/assert-cert-files-exist.yml | 0 .../utils/create-dual-cert-file.yml | 0 .../utils/enable-shard-allocation.yml | 0 .../utils/get-cluster-health.yml | 0 .../utils/get-config-from-files.yml | 0 .../prepare-cluster-for-node-restart.yml | 0 .../utils/restart-node.yml | 0 .../utils/save-initial-cluster-status.yml | 4 +- .../utils/test-api-access.yml | 0 .../utils/wait-for-cluster-status.yml | 0 .../utils/wait-for-node-to-join.yml | 0 .../utils/wait-for-shard-allocation.yml | 0 ansible/playbooks/upgrade.yml | 22 ++++----- cli/epicli.py | 2 +- docs/changelogs/CHANGELOG-0.10.md | 2 +- docs/changelogs/CHANGELOG-0.5.md | 2 +- docs/changelogs/CHANGELOG-0.8.md | 2 +- docs/changelogs/CHANGELOG-0.9.md | 2 +- docs/design-docs/arm/centos-arm-analysis.md | 12 ++--- docs/design-docs/arm/redhat-arm-analysis.md | 12 ++--- docs/design-docs/arm/ubuntu-arm-analysis.md | 12 ++--- docs/home/ARM.md | 14 +++--- docs/home/HOWTO.md | 4 +- docs/home/howto/CLUSTER.md | 2 +- docs/home/howto/DATABASES.md | 12 ++--- docs/home/howto/LOGGING.md | 4 +- docs/home/howto/MONITORING.md | 6 +-- docs/home/howto/SECURITY_GROUPS.md | 2 +- docs/home/howto/UPGRADE.md | 8 ++-- schema/any/defaults/epiphany-cluster.yml | 2 +- schema/aws/defaults/epiphany-cluster.yml | 2 +- schema/azure/defaults/epiphany-cluster.yml | 2 +- .../configuration/feature-mapping.yml | 6 +-- .../defaults/configuration/firewall.yml | 2 +- .../opendistro-for-elasticsearch.yml | 2 +- .../configuration/feature-mapping.yml | 2 +- .../validation/configuration/firewall.yml | 2 +- .../opendistro-for-elasticsearch.yml | 4 +- 78 files changed, 259 insertions(+), 226 deletions(-) delete mode 100644 ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-es.yml delete mode 100644 ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro.yml rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/defaults/main.yml (94%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/meta/main.yml (100%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch/tasks/configure-es.yml => opensearch/tasks/configure-ops.yml} (99%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/tasks/generate-certs.yml (100%) create mode 100644 ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml create mode 100644 ansible/playbooks/roles/opensearch/tasks/install-ops.yml rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/tasks/main.yml (86%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/tasks/patch-log4j.yml (86%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/tasks/remove-demo-certs.yml (100%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/tasks/remove-known-demo-certs.yml (100%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch => opensearch}/templates/jvm.options.j2 (100%) rename ansible/playbooks/roles/{opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 => opensearch/templates/opensearch.yml.j2} (100%) delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-plugins.yml rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/migrate-from-demo-certs-01.yml (89%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/migrate-from-demo-certs-02.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/migrate-from-demo-certs-non-clustered.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch/upgrade-elasticsearch-01.yml => opensearch/upgrade-opensearch-01.yml} (98%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch/upgrade-elasticsearch-02.yml => opensearch/upgrade-opensearch-02.yml} (71%) create mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/assert-api-access.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/assert-cert-files-exist.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/create-dual-cert-file.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/enable-shard-allocation.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/get-cluster-health.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/get-config-from-files.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/prepare-cluster-for-node-restart.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/restart-node.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/save-initial-cluster-status.yml (76%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/test-api-access.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/wait-for-cluster-status.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/wait-for-node-to-join.yml (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch => opensearch}/utils/wait-for-shard-allocation.yml (100%) diff --git a/ansible/playbooks/filebeat.yml b/ansible/playbooks/filebeat.yml index d2295b29c3..94233b3970 100644 --- a/ansible/playbooks/filebeat.yml +++ b/ansible/playbooks/filebeat.yml @@ -1,7 +1,7 @@ --- # Ansible playbook that installs and configures Filebeat -- hosts: opendistro_for_elasticsearch:logging:kibana # to gather facts +- hosts: opensearch:logging:kibana # to gather facts tasks: [] - hosts: filebeat diff --git a/ansible/playbooks/opendistro_for_elasticsearch.yml b/ansible/playbooks/opendistro_for_elasticsearch.yml index 9ec9a72ed6..4779a1efbe 100644 --- a/ansible/playbooks/opendistro_for_elasticsearch.yml +++ b/ansible/playbooks/opendistro_for_elasticsearch.yml @@ -1,10 +1,10 @@ --- # Ansible playbook for installing Elasticsearch -- hosts: opendistro_for_elasticsearch +- hosts: opensearch become: true become_method: sudo roles: - - opendistro_for_elasticsearch + - opensearch vars: - current_group_name: "opendistro_for_elasticsearch" + current_group_name: "opensearch" diff --git a/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml index 67e55ed34d..352fd9858c 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml @@ -1,7 +1,7 @@ --- -- name: Include default vars from opendistro_for_elasticsearch role +- name: Include default vars from opensearch role include_vars: - file: roles/opendistro_for_elasticsearch/defaults/main.yml + file: roles/opensearch/defaults/main.yml name: odfe - name: Set helper facts diff --git a/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml b/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml index 8df02d6c8a..1d03db3046 100644 --- a/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml +++ b/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Include installation task - include_tasks: install-es-curator-{{ ansible_os_family }}.yml + include_tasks: install-ops-curator-{{ ansible_os_family }}.yml - name: Include configuration tasks include_tasks: configure-cron-jobs.yml \ No newline at end of file diff --git a/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml b/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml index 47f622cca5..6951e11c1f 100644 --- a/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml +++ b/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml @@ -7,7 +7,7 @@ name: postgresql_defaults when: "'postgresql' in group_names" -# Do not select Kibana configured to use ES deployed by 'opendistro_for_elasticsearch' role +# Do not select Kibana configured to use ES deployed by 'opensearch' role - name: Set value for setup.kibana.host set_fact: setup_kibana_host: >- diff --git a/ansible/playbooks/roles/kibana/tasks/main.yml b/ansible/playbooks/roles/kibana/tasks/main.yml index be53a44420..1e68011a4a 100644 --- a/ansible/playbooks/roles/kibana/tasks/main.yml +++ b/ansible/playbooks/roles/kibana/tasks/main.yml @@ -15,7 +15,7 @@ - name: Include logging configuration tasks include_tasks: setup-logging.yml -- name: Load variables from logging/opendistro_for_elasticsearch role +- name: Load variables from logging/opensearch role when: context is undefined or context != "upgrade" block: - name: Load variables from logging role @@ -24,11 +24,11 @@ name: opendistro_for_logging_vars when: "'logging' in group_names" - - name: Load variables from opendistro_for_elasticsearch role + - name: Load variables from opensearch role include_vars: - file: roles/opendistro_for_elasticsearch/vars/main.yml + file: roles/opensearch/vars/main.yml name: opendistro_for_data_vars - when: "'opendistro_for_elasticsearch' in group_names" + when: "'opensearch' in group_names" - name: Update Kibana configuration file template: diff --git a/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 index e27bf5112d..ca8c03bf1e 100644 --- a/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 +++ b/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 @@ -23,9 +23,9 @@ elasticsearch.hosts: {% for host in groups['logging'] %} - "https://{{hostvars[host]['ansible_hostname']}}:9200" {% endfor %} -{% elif 'opendistro_for_elasticsearch' in group_names %} +{% elif 'opensearch' in group_names %} # Data hosts: - {% for host in groups['opendistro_for_elasticsearch'] %} + {% for host in groups['opensearch'] %} - "https://{{hostvars[host]['ansible_hostname']}}:9200" {% endfor %} {% endif %} @@ -37,7 +37,7 @@ elasticsearch.username: kibanaserver {# mode: apply -#} {% if 'logging' in group_names -%} {% set password = opendistro_for_logging_vars.specification.kibanaserver_password -%} - {% elif 'opendistro_for_elasticsearch' in group_names -%} + {% elif 'opensearch' in group_names -%} {% set password = opendistro_for_data_vars.specification.kibanaserver_password -%} {% endif %} {% else -%} diff --git a/ansible/playbooks/roles/logging/tasks/main.yml b/ansible/playbooks/roles/logging/tasks/main.yml index 5671e42791..9262896c4b 100644 --- a/ansible/playbooks/roles/logging/tasks/main.yml +++ b/ansible/playbooks/roles/logging/tasks/main.yml @@ -12,6 +12,6 @@ - name: Install and configure OpenDistro for Elasticsearch import_role: - name: opendistro_for_elasticsearch + name: opensearch vars: - specification: "{{ logging_vars.specification }}" # to override opendistro_for_elasticsearch specification + specification: "{{ logging_vars.specification }}" # to override opensearch specification diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-es.yml b/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-es.yml deleted file mode 100644 index 4bed42d55f..0000000000 --- a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-es.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Install elasticsearch-oss packages - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: - - elasticsearch-oss={{ versions[ansible_os_family].elasticsearch_oss }} - RedHat: - - elasticsearch-oss-{{ versions[ansible_os_family].elasticsearch_oss }} - register: install_elasticsearch_package - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro.yml b/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro.yml deleted file mode 100644 index d38b2ebcd3..0000000000 --- a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# NOTE: Keep in mind compatibility matrix for Open Distro https://opendistro.github.io/for-elasticsearch-docs/docs/install/plugins/#plugin-compatibility -- name: Install opendistro-* packages - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: - - opendistro-alerting={{ versions[ansible_os_family].opendistro }} - - opendistro-index-management={{ versions[ansible_os_family].opendistro }} - - opendistro-job-scheduler={{ versions[ansible_os_family].opendistro }} - - opendistro-performance-analyzer={{ versions[ansible_os_family].opendistro }} - - opendistro-security={{ versions[ansible_os_family].opendistro }} - - opendistro-sql={{ versions[ansible_os_family].opendistro }} - RedHat: - - opendistro-alerting-{{ versions[ansible_os_family].opendistro }} - - opendistro-index-management-{{ versions[ansible_os_family].opendistro }} - - opendistro-job-scheduler-{{ versions[ansible_os_family].opendistro }} - - opendistro-performance-analyzer-{{ versions[ansible_os_family].opendistro }} - - opendistro-security-{{ versions[ansible_os_family].opendistro }} - - opendistro-sql-{{ versions[ansible_os_family].opendistro }} - register: install_opendistro_packages - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml similarity index 94% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml rename to ansible/playbooks/roles/opensearch/defaults/main.yml index cbde5b2a67..b0f5b1186d 100644 --- a/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -3,11 +3,9 @@ versions: RedHat: - elasticsearch_oss: "7.10.2" - opendistro: "1.13.*" + ops_version: "1.2.3" Debian: - elasticsearch_oss: "7.10.2" - opendistro: "1.13.*" + ops_version: "1.2.3" certificates: dirs: certs: /etc/elasticsearch @@ -61,3 +59,4 @@ ports: http: 9200 # defaults to range but we want static port transport: 9300 # defaults to range but we want static port log4j_file_name: apache-log4j-2.17.1-bin.tar.gz +ops_user: opensearch \ No newline at end of file diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/meta/main.yml b/ansible/playbooks/roles/opensearch/meta/main.yml similarity index 100% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/meta/main.yml rename to ansible/playbooks/roles/opensearch/meta/main.yml diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml similarity index 99% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml rename to ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index f60cf05e27..cbd2c5436a 100644 --- a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -34,7 +34,7 @@ - name: Provide Elasticsearch configuration file template: backup: yes - src: elasticsearch.yml.j2 + src: opensearch.yml.j2 dest: /etc/elasticsearch/elasticsearch.yml owner: root group: elasticsearch diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/generate-certs.yml b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml similarity index 100% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/generate-certs.yml rename to ansible/playbooks/roles/opensearch/tasks/generate-certs.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml new file mode 100644 index 0000000000..c5802ac6c6 --- /dev/null +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml @@ -0,0 +1,25 @@ +--- +# NOTE: Keep in mind compatibility matrix for Open Distro https://opendistro.github.io/for-elasticsearch-docs/docs/install/plugins/#plugin-compatibility +- name: Install opensearch-* packages + package: + name: "{{ _packages[ansible_os_family] }}" + state: present + vars: + _packages: + Debian: + - opensearch-alerting={{ versions[ansible_os_family].opendistro }} + - opensearch-index-management={{ versions[ansible_os_family].opendistro }} + - opensearch-job-scheduler={{ versions[ansible_os_family].opendistro }} + - opensearch-performance-analyzer={{ versions[ansible_os_family].opendistro }} + - opensearch-security={{ versions[ansible_os_family].opendistro }} + - opensearch-sql={{ versions[ansible_os_family].opendistro }} + RedHat: + - opensearch-alerting-{{ versions[ansible_os_family].opendistro }} + - opensearch-index-management-{{ versions[ansible_os_family].opendistro }} + - opensearch-job-scheduler-{{ versions[ansible_os_family].opendistro }} + - opensearch-performance-analyzer-{{ versions[ansible_os_family].opendistro }} + - opensearch-security-{{ versions[ansible_os_family].opendistro }} + - opensearch-sql-{{ versions[ansible_os_family].opendistro }} + register: install_opendistro_packages + module_defaults: + yum: { lock_timeout: "{{ yum_lock_timeout }}" } diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml new file mode 100644 index 0000000000..f27786714a --- /dev/null +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -0,0 +1,48 @@ +--- +- name: OpenSearch Install | Download opensearch {{ os_version }} + get_url: + url: "{{ os_download_url }}/{{ os_version }}/opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" + dest: "/tmp/opensearch.tar.gz" + register: download + +- name: OpenSearch Install | Create opensearch user + user: + name: "{{ ops_user }}" + state: present + shell: /bin/bash + when: download.changed + +- name: OpenSearch Install | Create home directory + file: + path: "{{ os_home }}" + state: directory + owner: "{{ ops_user }}" + group: "{{ ops_user }}" + when: download.changed + +- name: OpenSearch Install | Extract the tar file + command: chdir=/tmp/ tar -xvzf opensearch.tar.gz -C "{{ os_home }}" --strip-components=1 + when: download.changed + +- name: OpenSearch Install | Copy Configuration File + template: + src: opensearch.yml + dest: "{{os_conf_dir}}/opensearch.yml" + owner: "{{ ops_user }}" + group: "{{ ops_user }}" + mode: 0644 + backup: yes + +- name: OpenSearch Install | Copy jvm.options File for Instance + template: + src: jvm.options + dest: "{{os_conf_dir}}/jvm.options" + owner: "{{ ops_user }}" + group: "{{ ops_user }}" + mode: 0644 + force: yes + +- name: OpenSearch Install | create systemd service + template: + src: opensearch.service + dest: "{{ systemctl_path }}/opensearch.service" diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml similarity index 86% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml rename to ansible/playbooks/roles/opensearch/tasks/main.yml index 6860c69c17..c04c7d3c4b 100644 --- a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -13,9 +13,9 @@ - { limit_type: 'soft', limit_item: 'memlock', value: unlimited } - { limit_type: 'hard', limit_item: 'memlock', value: unlimited } -- include_tasks: install-es.yml +- include_tasks: install-ops.yml -- include_tasks: install-opendistro.yml +- include_tasks: install-ops-plugins.yml - name: Include configuration tasks - include_tasks: configure-es.yml + include_tasks: configure-ops.yml diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/patch-log4j.yml b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml similarity index 86% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/patch-log4j.yml rename to ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml index 917c2e52d7..9b996e2d34 100644 --- a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/patch-log4j.yml +++ b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml @@ -1,7 +1,7 @@ --- - name: Log4j patch block: - - name: "opendistro_for_elasticsearch : Log4j patch | Get archive" + - name: "opensearch : Log4j patch | Get archive" include_role: name: download tasks_from: download_file @@ -28,10 +28,10 @@ loop: - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/lib/ } - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/performance-analyzer-rca/lib/ } + - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/ } - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/lib/ } - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/performance-analyzer-rca/lib/ } + - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/ } - { src: "{{ download_directory }}/{{ log4j_slfj_impl }}", dest: /usr/share/elasticsearch/plugins/opendistro_security/ } vars: log4j_api: "{{ unarchive_list_files.files | select('contains', 'log4j-api-2.17.1.jar') | first }}" @@ -45,8 +45,8 @@ state: absent path: "{{ item }}" loop: - - /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/performance-analyzer-rca/lib/log4j-api-2.13.0.jar - - /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/performance-analyzer-rca/lib/log4j-core-2.13.0.jar + - /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/log4j-api-2.13.0.jar + - /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-api-2.13.0.jar - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - /usr/share/elasticsearch/lib/log4j-api-2.11.1.jar @@ -61,8 +61,8 @@ _archive_root_dir: >- {{ unarchive_list_files.files | first | dirname }} -- name: Restart opendistro-performance-analyzer service +- name: Restart opensearch-performance-analyzer service systemd: - name: opendistro-performance-analyzer + name: opensearch-performance-analyzer state: restarted when: log4j_patch.changed diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/remove-demo-certs.yml b/ansible/playbooks/roles/opensearch/tasks/remove-demo-certs.yml similarity index 100% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/remove-demo-certs.yml rename to ansible/playbooks/roles/opensearch/tasks/remove-demo-certs.yml diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/remove-known-demo-certs.yml b/ansible/playbooks/roles/opensearch/tasks/remove-known-demo-certs.yml similarity index 100% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/remove-known-demo-certs.yml rename to ansible/playbooks/roles/opensearch/tasks/remove-known-demo-certs.yml diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 similarity index 100% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 rename to ansible/playbooks/roles/opensearch/templates/jvm.options.j2 diff --git a/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 similarity index 100% rename from ansible/playbooks/roles/opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 rename to ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index 860f7db731..f1bfc1b8c1 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -38,7 +38,7 @@ unsupported_roles: - haproxy - logging - elasticsearch_curator - - opendistro_for_elasticsearch + - opensearch - elasticsearch - kibana - filebeat @@ -75,7 +75,7 @@ unsupported_roles: - haproxy - logging - elasticsearch_curator - - opendistro_for_elasticsearch + - opensearch - elasticsearch - kibana - filebeat diff --git a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml index f1fa9bf15f..4bab27d76d 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml @@ -1,7 +1,7 @@ --- -- name: Include default vars from opendistro_for_elasticsearch role +- name: Include default vars from opensearch role include_vars: - file: roles/opendistro_for_elasticsearch/defaults/main.yml + file: roles/opensearch/defaults/main.yml name: odfe - name: Set helper facts diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 7ecc3fe7f3..8d7a7d8e1f 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -62,12 +62,12 @@ net-tools nfs-utils nmap-ncat # Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opendistro-alerting-1.13.1.* -opendistro-index-management-1.13.1.* -opendistro-job-scheduler-1.13.0.* -opendistro-performance-analyzer-1.13.0.* -opendistro-security-1.13.1.* -opendistro-sql-1.13.0.* +opensearch-alerting-1.13.1.* +opensearch-index-management-1.13.1.* +opensearch-job-scheduler-1.13.0.* +opensearch-performance-analyzer-1.13.0.* +opensearch-security-1.13.1.* +opensearch-sql-1.13.0.* opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl unixODBC # for erlang diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index fe4ac78f84..0276ca3eab 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -63,12 +63,12 @@ net-tools nfs-utils nmap-ncat # Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opendistro-alerting-1.13.1.* -opendistro-index-management-1.13.1.* -opendistro-job-scheduler-1.13.0.* -opendistro-performance-analyzer-1.13.0.* -opendistro-security-1.13.1.* -opendistro-sql-1.13.0.* +opensearch-alerting-1.13.1.* +opensearch-index-management-1.13.1.* +opensearch-job-scheduler-1.13.0.* +opensearch-performance-analyzer-1.13.0.* +opensearch-security-1.13.1.* +opensearch-sql-1.13.0.* opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl perl # for vim diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index 86698c69fa..f29e2f0c80 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -61,12 +61,12 @@ net-tools nfs-utils nmap-ncat # Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opendistro-alerting-1.13.1.* -opendistro-index-management-1.13.1.* -opendistro-job-scheduler-1.13.0.* -opendistro-performance-analyzer-1.13.0.* -opendistro-security-1.13.1.* -opendistro-sql-1.13.0.* +opensearch-alerting-1.13.1.* +opensearch-index-management-1.13.1.* +opensearch-job-scheduler-1.13.0.* +opensearch-performance-analyzer-1.13.0.* +opensearch-security-1.13.1.* +opensearch-sql-1.13.0.* opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl perl # for vim diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index a37ec22874..fdc4a081ac 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -59,12 +59,12 @@ net-tools nfs-common # for nfs-common libtirpc3 -opendistro-alerting 1.13.1 -opendistro-index-management 1.13.1 -opendistro-job-scheduler 1.13.0 -opendistro-performance-analyzer 1.13.0 -opendistro-security 1.13.1 -opendistro-sql 1.13.0 +opensearch-alerting 1.13.1 +opensearch-index-management 1.13.1 +opensearch-job-scheduler 1.13.0 +opensearch-performance-analyzer 1.13.0 +opensearch-security 1.13.1 +opensearch-sql 1.13.0 opendistroforelasticsearch-kibana 1.13.1 openjdk-8-jre-headless openssl diff --git a/ansible/playbooks/roles/upgrade/defaults/main.yml b/ansible/playbooks/roles/upgrade/defaults/main.yml index e7e0a5f77a..225b8b4a28 100644 --- a/ansible/playbooks/roles/upgrade/defaults/main.yml +++ b/ansible/playbooks/roles/upgrade/defaults/main.yml @@ -5,7 +5,7 @@ logging: cert_path: /etc/elasticsearch/custom-admin.pem key_path: /etc/elasticsearch/custom-admin-key.pem -opendistro_for_elasticsearch: +opensearch: upgrade_config: custom_admin_certificate: cert_path: /etc/elasticsearch/custom-admin.pem diff --git a/ansible/playbooks/roles/upgrade/tasks/elasticsearch-curator.yml b/ansible/playbooks/roles/upgrade/tasks/elasticsearch-curator.yml index f7731c3218..81af709f8f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/elasticsearch-curator.yml +++ b/ansible/playbooks/roles/upgrade/tasks/elasticsearch-curator.yml @@ -24,6 +24,6 @@ - name: Update elasticsearch-curator package include_role: name: elasticsearch_curator - tasks_from: install-es-curator-{{ ansible_os_family }} # update only package and do not change configured cron jobs + tasks_from: install-ops-curator-{{ ansible_os_family }} # update only package and do not change configured cron jobs when: - curator_defaults.curator_version is version(ansible_facts.packages['elasticsearch-curator'][0].version, '>') diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml index b3f14e4137..7c3d910d92 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml @@ -10,14 +10,14 @@ fail_msg: elasticsearch-oss package not found, nothing to upgrade quiet: true -- name: ODFE | Include defaults from opendistro_for_elasticsearch role +- name: ODFE | Include defaults from opensearch role include_vars: - file: roles/opendistro_for_elasticsearch/defaults/main.yml + file: roles/opensearch/defaults/main.yml name: odfe_defaults - name: ODFE | Patch log4j include_role: - name: opendistro_for_elasticsearch + name: opensearch tasks_from: patch-log4j when: odfe_defaults.log4j_file_name is defined @@ -37,14 +37,14 @@ # If state file exists it means the previous run failed - name: ODFE | Check if upgrade state file exists stat: - path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false get_checksum: false get_mime: false register: stat_upgrade_state_file - name: ODFE | Upgrade Elasticsearch and ODFE plugins (part 1/2) - include_tasks: opendistro_for_elasticsearch/upgrade-elasticsearch-01.yml + include_tasks: opensearch/upgrade-opensearch-01.yml when: _target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') and stat_upgrade_state_file.stat.exists) diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-02.yml index 2b3f304465..1a34acdc31 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-02.yml @@ -2,12 +2,12 @@ # If state file exists, it means upgrade has been started by the previous play and should be continued - name: ODFE | Check if upgrade state file exists stat: - path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false get_checksum: false get_mime: false register: stat_upgrade_state_file - name: ODFE | Upgrade Elasticsearch and ODFE plugins (part 2/2) - include_tasks: opendistro_for_elasticsearch/upgrade-elasticsearch-02.yml + include_tasks: opensearch/upgrade-opensearch-02.yml when: stat_upgrade_state_file.stat.exists diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-plugins.yml b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-plugins.yml deleted file mode 100644 index 80e34e6382..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-plugins.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: ODFE plugins | Assert that opendistro-* packages are installed - assert: - that: ansible_facts.packages['{{ item }}'] is defined - fail_msg: "Missing package to upgrade: {{ item }}" - quiet: true - loop: - - opendistro-alerting - - opendistro-index-management - - opendistro-job-scheduler - - opendistro-performance-analyzer - - opendistro-security - - opendistro-sql - -- name: ODFE plugins | Upgrade opendistro-* packages - include_role: - name: opendistro_for_elasticsearch - tasks_from: install-opendistro.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml similarity index 89% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-01.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml index 806c09a3d0..f9a83243b5 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml @@ -20,7 +20,7 @@ certs_to_concatenate: - "{{ (certificates.dirs.certs, certificates.files.demo.root_ca.cert) | path_join }}" - "{{ (certificates.dirs.certs, certificates.files.root_ca.cert.filename) | path_join }}" - target_path: "{{ (certificates.dirs.certs, opendistro_for_elasticsearch.certs_migration.dual_root_ca.filename) | path_join }}" + target_path: "{{ (certificates.dirs.certs, opensearch.certs_migration.dual_root_ca.filename) | path_join }}" - name: ODFE | Load /etc/elasticsearch/elasticsearch.yml slurp: @@ -60,12 +60,12 @@ _updated_settings: opendistro_security.authcz.admin_dn: >- {{ _old_content['opendistro_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') - | union([opendistro_for_elasticsearch.certs_migration.demo_DNs.admin] + [_epiphany_DNs.admin]) }} + | union([opensearch.certs_migration.demo_DNs.admin] + [_epiphany_DNs.admin]) }} opendistro_security.nodes_dn: >- {{ _old_content['opendistro_security.nodes_dn'] | default([]) - | union([opendistro_for_elasticsearch.certs_migration.demo_DNs.node] + _epiphany_nodes_dn) }} + | union([opensearch.certs_migration.demo_DNs.node] + _epiphany_nodes_dn) }} - opendistro_security.ssl.http.pemtrustedcas_filepath: "{{ opendistro_for_elasticsearch.certs_migration.dual_root_ca.filename }}" - opendistro_security.ssl.transport.pemtrustedcas_filepath: "{{ opendistro_for_elasticsearch.certs_migration.dual_root_ca.filename }}" + opendistro_security.ssl.http.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" + opendistro_security.ssl.transport.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" _patched_content: >- {{ _old_content | combine(_updated_settings) }} diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-02.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-non-clustered.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/migrate-from-demo-certs-non-clustered.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml similarity index 98% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch-01.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml index e709502eda..675c70e605 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml @@ -4,7 +4,7 @@ - name: ODFE | Create upgrade state file become: true file: - path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + path: "{{ opensearch.upgrade_state_file_path }}" state: touch mode: u=rw,g=r,o= @@ -127,7 +127,7 @@ - name: Generate self-signed certificates include_role: - name: opendistro_for_elasticsearch + name: opensearch tasks_from: generate-certs when: es_api.cert_type != 'custom' diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml similarity index 71% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch-02.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml index 237f34d4d2..5fe1f4a8df 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml @@ -1,9 +1,9 @@ --- # This file contains flow that cannot be run in parallel on multiple hosts because of rolling upgrades. -# It's run after upgrade-elasticsearch-01.yml so some facts are already set. +# It's run after upgrade-opensearch-01.yml so some facts are already set. # Run migration procedure - the second (serial) part -- include_tasks: opendistro_for_elasticsearch/migrate-from-demo-certs-02.yml +- include_tasks: opensearch/migrate-from-demo-certs-02.yml when: - es_api.cert_type == 'demo' - es_clustered # rolling upgrade only for clustered installation @@ -14,7 +14,7 @@ tags: [ never, debug ] # only runs when debug or never tag requested - name: ODFE | Prepare cluster for rolling upgrade - include_tasks: opendistro_for_elasticsearch/utils/prepare-cluster-for-node-restart.yml + include_tasks: opensearch/utils/prepare-cluster-for-node-restart.yml when: es_clustered - name: ODFE | Stop elasticsearch service @@ -24,13 +24,13 @@ - name: ODFE | Include Elasticsearch installation tasks include_role: - name: opendistro_for_elasticsearch - tasks_from: install-es.yml + name: opensearch + tasks_from: install-ops.yml - name: ODFE | Include Elasticsearch configuration tasks include_role: - name: opendistro_for_elasticsearch - tasks_from: configure-es.yml + name: opensearch + tasks_from: configure-ops.yml vars: _old: "{{ existing_config.main }}" # Keep the same data structure as for apply mode @@ -48,9 +48,9 @@ enforce_hostname_verification: "{{ _old['opendistro_security.ssl.transport.enforce_hostname_verification'] }}" _demo_DNs: - admin: "{{ opendistro_for_elasticsearch.certs_migration.demo_DNs.admin }}" - node: "{{ opendistro_for_elasticsearch.certs_migration.demo_DNs.node }}" - _dual_root_ca_filename: "{{ opendistro_for_elasticsearch.certs_migration.dual_root_ca.filename }}" + admin: "{{ opensearch.certs_migration.demo_DNs.admin }}" + node: "{{ opensearch.certs_migration.demo_DNs.node }}" + _dual_root_ca_filename: "{{ opensearch.certs_migration.dual_root_ca.filename }}" _epiphany_root_ca_filename: "{{ certificates.files.root_ca.cert.filename }}" _updated_existing_config: opendistro_security.authcz.admin_dn: "{{ _old['opendistro_security.authcz.admin_dn'] | reject('search', _demo_DNs.admin) }}" @@ -66,12 +66,12 @@ existing_es_config: "{{ _old | combine(_updated_existing_config) }}" - name: ODFE | Include upgrade plugins tasks - include_tasks: opendistro_for_elasticsearch/upgrade-plugins.yml + include_tasks: opensearch/upgrade-plugins.yml # Restart elasticsearch service (unconditionally to ensure this task is not skipped in case of rerunning after interruption) -- include_tasks: opendistro_for_elasticsearch/utils/restart-node.yml +- include_tasks: opensearch/utils/restart-node.yml vars: - daemon_reload: true # opendistro-performance-analyzer provides opendistro-performance-analyzer.service + daemon_reload: true # opensearch-performance-analyzer provides opensearch-performance-analyzer.service target_inventory_hostname: "{{ inventory_hostname }}" # Post-upgrade tasks @@ -79,18 +79,18 @@ - name: Re-enable shard allocation when: es_clustered block: - - include_tasks: opendistro_for_elasticsearch/utils/enable-shard-allocation.yml + - include_tasks: opensearch/utils/enable-shard-allocation.yml - - include_tasks: opendistro_for_elasticsearch/utils/wait-for-shard-allocation.yml + - include_tasks: opensearch/utils/wait-for-shard-allocation.yml # Read cluster health status from before the upgrade - name: Load upgrade state file slurp: - src: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + src: "{{ opensearch.upgrade_state_file_path }}" register: slurp_upgrade_state_file # Verify cluster status -- include_tasks: opendistro_for_elasticsearch/utils/wait-for-cluster-status.yml +- include_tasks: opensearch/utils/wait-for-cluster-status.yml when: not es_clustered or (es_clustered and inventory_hostname == ansible_play_hosts_all[-1]) # for 'green' status at least 2 nodes must be already upgraded vars: @@ -99,11 +99,11 @@ - name: ODFE | Remove dual root CA temporary file file: - path: "{{ (certificates.dirs.certs, opendistro_for_elasticsearch.certs_migration.dual_root_ca.filename) | path_join }}" + path: "{{ (certificates.dirs.certs, opensearch.certs_migration.dual_root_ca.filename) | path_join }}" state: absent when: es_api.cert_type == 'Epiphany' - name: ODFE | Remove upgrade state file file: - path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + path: "{{ opensearch.upgrade_state_file_path }}" state: absent diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml new file mode 100644 index 0000000000..0f714a75fb --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml @@ -0,0 +1,18 @@ +--- +- name: ODFE plugins | Assert that opensearch-* packages are installed + assert: + that: ansible_facts.packages['{{ item }}'] is defined + fail_msg: "Missing package to upgrade: {{ item }}" + quiet: true + loop: + - opensearch-alerting + - opensearch-index-management + - opensearch-job-scheduler + - opensearch-performance-analyzer + - opensearch-security + - opensearch-sql + +- name: ODFE plugins | Upgrade opensearch-* packages + include_role: + name: opensearch + tasks_from: install-ops-plugins.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/assert-api-access.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/assert-api-access.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/assert-cert-files-exist.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/assert-cert-files-exist.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/create-dual-cert-file.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/create-dual-cert-file.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/enable-shard-allocation.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/enable-shard-allocation.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/get-cluster-health.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/get-cluster-health.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/get-config-from-files.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/get-config-from-files.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/prepare-cluster-for-node-restart.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/prepare-cluster-for-node-restart.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/restart-node.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/restart-node.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/save-initial-cluster-status.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml similarity index 76% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/save-initial-cluster-status.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml index 9050c7799a..8c37c863f4 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/save-initial-cluster-status.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml @@ -1,7 +1,7 @@ --- - name: ODFE | Get size of upgrade state file stat: - path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false get_checksum: false get_mime: false @@ -15,4 +15,4 @@ - name: ODFE | Save cluster health to upgrade state file copy: content: "{{ cluster_health.json }}" - dest: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + dest: "{{ opensearch.upgrade_state_file_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/test-api-access.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/test-api-access.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/wait-for-cluster-status.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/wait-for-cluster-status.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/wait-for-node-to-join.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/wait-for-node-to-join.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/wait-for-shard-allocation.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/utils/wait-for-shard-allocation.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index 1b8fb40e20..6987b484a8 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -148,7 +148,7 @@ tasks: - include_role: name: upgrade - tasks_from: opendistro_for_elasticsearch-01 + tasks_from: opensearch-01 when: "'logging' in upgrade_components or upgrade_components|length == 0" vars: current_group_name: logging @@ -162,30 +162,30 @@ tasks: - include_role: name: upgrade - tasks_from: opendistro_for_elasticsearch-02 + tasks_from: opensearch-02 when: "'logging' in upgrade_components or upgrade_components|length == 0" vars: current_group_name: logging -# === opendistro_for_elasticsearch === +# === opensearch === # Some pre-upgrade tasks can be run in parallel (what saves time) while others must be run in serial (to support rolling upgrades). # Such a separation in Ansible can be applied only at play level thus we have two plays below. # play 1/2: parallel tasks -- hosts: opendistro_for_elasticsearch +- hosts: opensearch become: true become_method: sudo tasks: - include_role: name: upgrade - tasks_from: opendistro_for_elasticsearch-01 - when: "'opendistro_for_elasticsearch' in upgrade_components or upgrade_components|length == 0" + tasks_from: opensearch-01 + when: "'opensearch' in upgrade_components or upgrade_components|length == 0" vars: - current_group_name: opendistro_for_elasticsearch + current_group_name: opensearch # play 2/2: serial tasks -- hosts: opendistro_for_elasticsearch +- hosts: opensearch become: true become_method: sudo gather_facts: false # gathered by previous play @@ -193,10 +193,10 @@ tasks: - include_role: name: upgrade - tasks_from: opendistro_for_elasticsearch-02 - when: "'opendistro_for_elasticsearch' in upgrade_components or upgrade_components|length == 0" + tasks_from: opensearch-02 + when: "'opensearch' in upgrade_components or upgrade_components|length == 0" vars: - current_group_name: opendistro_for_elasticsearch + current_group_name: opensearch - hosts: kibana become: true diff --git a/cli/epicli.py b/cli/epicli.py index 60537b6c0b..89c76b178e 100644 --- a/cli/epicli.py +++ b/cli/epicli.py @@ -261,7 +261,7 @@ def upgrade_parser(subparsers): 'load_balancer', 'logging', 'node_exporter', - 'opendistro_for_elasticsearch', + 'opensearch', 'postgresql', 'postgres_exporter', 'prometheus', diff --git a/docs/changelogs/CHANGELOG-0.10.md b/docs/changelogs/CHANGELOG-0.10.md index b5e772c48b..444099610e 100644 --- a/docs/changelogs/CHANGELOG-0.10.md +++ b/docs/changelogs/CHANGELOG-0.10.md @@ -50,7 +50,7 @@ Version 0.10 won't be supported anymore. Instead, we introduced version 1.0 whic ### Breaking changes -- Feature `elasticsearch` removed in favor of `opendistro-for-elasticsearch`. +- Feature `elasticsearch` removed in favor of `opensearch-for-elasticsearch`. ### Known issues diff --git a/docs/changelogs/CHANGELOG-0.5.md b/docs/changelogs/CHANGELOG-0.5.md index 9f1a8f9e36..1d49ed03a0 100644 --- a/docs/changelogs/CHANGELOG-0.5.md +++ b/docs/changelogs/CHANGELOG-0.5.md @@ -83,7 +83,7 @@ - [#782](https://github.com/epiphany-platform/epiphany/issues/781) - All disks encryption documentation - AWS - [#782](https://github.com/epiphany-platform/epiphany/issues/782) - All disks encryption documentation - Azure - [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for Elasticsearch - - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) + - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch) - [Centralized logging](/docs/home/howto/LOGGING.md#centralized-logging-setup) - [#755](https://github.com/epiphany-platform/epiphany/issues/755) - Create Ansible playbook to install Apache Ignite as a service on VM diff --git a/docs/changelogs/CHANGELOG-0.8.md b/docs/changelogs/CHANGELOG-0.8.md index 2b77d60cda..5e169f719b 100644 --- a/docs/changelogs/CHANGELOG-0.8.md +++ b/docs/changelogs/CHANGELOG-0.8.md @@ -66,4 +66,4 @@ ### Known issues -- [1647](https://github.com/epiphany-platform/epiphany/issues/1647) - `epicli upgrade` fails on `[opendistro_for_elasticsearch : Provide jvm configuration file]` task +- [1647](https://github.com/epiphany-platform/epiphany/issues/1647) - `epicli upgrade` fails on `[opensearch : Provide jvm configuration file]` task diff --git a/docs/changelogs/CHANGELOG-0.9.md b/docs/changelogs/CHANGELOG-0.9.md index eda84f658b..1b0876d81a 100644 --- a/docs/changelogs/CHANGELOG-0.9.md +++ b/docs/changelogs/CHANGELOG-0.9.md @@ -38,7 +38,7 @@ ### Deprecated -- Elasticsearch OSS v6 (feature name: `elasticsearch`), succesor: Elasticsearch OSS v7 (feature name: `opendistro-for-elasticsearch`). It may be removed in the next major release. +- Elasticsearch OSS v6 (feature name: `elasticsearch`), succesor: Elasticsearch OSS v7 (feature name: `opensearch-for-elasticsearch`). It may be removed in the next major release. ### Breaking changes diff --git a/docs/design-docs/arm/centos-arm-analysis.md b/docs/design-docs/arm/centos-arm-analysis.md index b7211606dc..8d37ff8df7 100644 --- a/docs/design-docs/arm/centos-arm-analysis.md +++ b/docs/design-docs/arm/centos-arm-analysis.md @@ -61,12 +61,12 @@ | net-tools | + | | + | | nfs-utils | + | | + | | nmap-ncat | + | | ? | -| opendistro-alerting-1.10.1* | + | | + | -| opendistro-index-management-1.10.1* | + | | + | -| opendistro-job-scheduler-1.10.1* | + | | + | -| opendistro-performance-analyzer-1.10.1* | + | | + | -| opendistro-security-1.10.1* | + | | + | -| opendistro-sql-1.10.1* | + | | + | +| opensearch-alerting-1.10.1* | + | | + | +| opensearch-index-management-1.10.1* | + | | + | +| opensearch-job-scheduler-1.10.1* | + | | + | +| opensearch-performance-analyzer-1.10.1* | + | | + | +| opensearch-security-1.10.1* | + | | + | +| opensearch-sql-1.10.1* | + | | + | | opendistroforelasticsearch-kibana-1.10.1* | --- | opendistroforelasticsearch-kibana-1.13.0 | + | | openssl | + | | + | | perl | + | | + | diff --git a/docs/design-docs/arm/redhat-arm-analysis.md b/docs/design-docs/arm/redhat-arm-analysis.md index 17c13a5f1f..32b37f5e5c 100644 --- a/docs/design-docs/arm/redhat-arm-analysis.md +++ b/docs/design-docs/arm/redhat-arm-analysis.md @@ -60,12 +60,12 @@ | net-tools | + | | + | | nfs-utils | + | | + | | nmap-ncat | + | | ? | -| opendistro-alerting-1.13.1* | + | | + | -| opendistro-index-management-1.13.1* | + | | + | -| opendistro-job-scheduler-1.13.1* | + | | + | -| opendistro-performance-analyzer-1.13.1* | + | | + | -| opendistro-security-1.13.1* | + | | + | -| opendistro-sql-1.13.1* | + | | + | +| opensearch-alerting-1.13.1* | + | | + | +| opensearch-index-management-1.13.1* | + | | + | +| opensearch-job-scheduler-1.13.1* | + | | + | +| opensearch-performance-analyzer-1.13.1* | + | | + | +| opensearch-security-1.13.1* | + | | + | +| opensearch-sql-1.13.1* | + | | + | | opendistroforelasticsearch-kibana-1.13.1* | + | | + | | unixODBC | + | | + | | openssl | + | | + | diff --git a/docs/design-docs/arm/ubuntu-arm-analysis.md b/docs/design-docs/arm/ubuntu-arm-analysis.md index 0a5c635ead..8db31ea512 100644 --- a/docs/design-docs/arm/ubuntu-arm-analysis.md +++ b/docs/design-docs/arm/ubuntu-arm-analysis.md @@ -52,12 +52,12 @@ | netcat | + | | + | | net-tools | + | | + | | nfs-common | + | | + | -| opendistro-alerting | + | | + | -| opendistro-index-management | + | | + | -| opendistro-job-scheduler | + | | + | -| opendistro-performance-analyzer | + | | + | -| opendistro-security | + | | + | -| opendistro-sql | + | | + | +| opensearch-alerting | + | | + | +| opensearch-index-management | + | | + | +| opensearch-job-scheduler | + | | + | +| opensearch-performance-analyzer | + | | + | +| opensearch-security | + | | + | +| opensearch-sql | + | | + | | opendistroforelasticsearch-kibana | + | | + | | openjdk-8-jre-headless | + | | + | | openssl | + | | + | diff --git a/docs/home/ARM.md b/docs/home/ARM.md index c53945efd7..39d1c82fe7 100644 --- a/docs/home/ARM.md +++ b/docs/home/ARM.md @@ -29,7 +29,7 @@ Besides making sure that the selected providers, operating systems, components a | monitoring | :heavy_check_mark: | :x: | :x: | | load_balancer | :heavy_check_mark: | :x: | :x: | | postgresql | :heavy_check_mark: | :x: | :x: | -| opendistro_for_elasticsearch | :heavy_check_mark: | :x: | :x: | +| opensearch | :heavy_check_mark: | :x: | :x: | | single_machine | :heavy_check_mark: | :x: | :x: | ***Notes*** @@ -92,9 +92,9 @@ specification: rabbitmq: count: 2 machine: rabbitmq-machine-arm - opendistro_for_elasticsearch: + opensearch: count: 1 - machine: opendistro-machine-arm + machine: opensearch-machine-arm repository: count: 1 machine: repository-machine-arm @@ -164,7 +164,7 @@ specification: ip: x.x.x.x --- kind: infrastructure/virtual-machine -name: opendistro-machine-arm +name: opensearch-machine-arm provider: any based_on: logging-machine specification: @@ -327,9 +327,9 @@ specification: subnets: - availability_zone: eu-west-1a address_pool: 10.1.8.0/24 - opendistro_for_elasticsearch: + opensearch: count: 1 - machine: opendistro-machine-arm + machine: opensearch-machine-arm subnets: - availability_zone: eu-west-1a address_pool: 10.1.10.0/24 @@ -404,7 +404,7 @@ specification: size: a1.medium --- kind: infrastructure/virtual-machine -name: opendistro-machine-arm +name: opensearch-machine-arm provider: aws based_on: logging-machine specification: diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 17c8ed78da..7d48c838e4 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -89,8 +89,8 @@ - [How to switchover database nodes](./howto/DATABASES.md#how-to-switchover-database-nodes) - [How to set up PGBouncer, PgPool and PostgreSQL parameters](./howto/DATABASES.md#how-to-set-up-pgbouncer-pgpool-and-postgresql-parameters) - [How to set up PostgreSQL audit logging](./howto/DATABASES.md#how-to-set-up-postgresql-audit-logging) - - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) - - [How to manage Opendistro for Elasticsearch data](./howto/LOGGING.md#how-to-manage-opendistro-for-elasticsearch-data) + - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch) + - [How to manage Opendistro for Elasticsearch data](./howto/LOGGING.md#how-to-manage-opensearch-for-elasticsearch-data) - [Backup and Recovery](./howto/BACKUP.md) - [Epiphany backup and restore](./howto/BACKUP.md#epiphany-backup-and-restore) diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index c2f6914fd7..a0be06e2ff 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -638,7 +638,7 @@ specification: count: 0 rabbitmq: count: 0 - opendistro_for_elasticsearch: + opensearch: count: 0 single_machine: count: 1 diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index cdb5e8a67f..1e81806a8c 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -475,14 +475,14 @@ specification: ... logging: count: 1 - opendistro_for_elasticsearch: + opensearch: count: 2 ``` **Installation with more than one node will always be clustered** - Option to configure the non-clustered installation of more than one node for Open Distro is not supported. ```yaml -kind: configuration/opendistro-for-elasticsearch +kind: configuration/opensearch-for-elasticsearch title: OpenDistro for Elasticsearch Config name: default specification: @@ -490,7 +490,7 @@ specification: ``` By default, Kibana is deployed only for `logging` component. If you want to deploy Kibana -for `opendistro_for_elasticsearch` you have to modify feature mapping. Use below configuration in your manifest. +for `opensearch` you have to modify feature mapping. Use below configuration in your manifest. ```yaml kind: configuration/feature-mapping @@ -498,12 +498,12 @@ title: "Feature mapping to roles" name: default specification: roles_mapping: - opendistro_for_elasticsearch: - - opendistro-for-elasticsearch + opensearch: + - opensearch-for-elasticsearch - node-exporter - filebeat - firewall - kibana ``` -Filebeat running on `opendistro_for_elasticsearch` hosts will always point to centralized logging hosts (./LOGGING.md). +Filebeat running on `opensearch` hosts will always point to centralized logging hosts (./LOGGING.md). diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index e419b2a543..dd57f26da7 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -36,8 +36,8 @@ logging: The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0. -Default configuration of `logging` and `opendistro_for_elasticsearch` roles is identical ( -./DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch). To modify configuration of centralized logging +Default configuration of `logging` and `opensearch` roles is identical ( +./DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch). To modify configuration of centralized logging adjust and use the following defaults in your manifest: ```yaml diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index 3f2917c2d0..62e4b3cdbf 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -253,7 +253,7 @@ By default Kibana adjusts the UTC time in `@timestamp` to the browser's local ti ## How to configure default user passwords for Kibana - Open Distro, Open Distro for Elasticsearch and Filebeat To configure admin password for Kibana - Open Distro and Open Distro for Elasticsearch you need to follow the procedure below. -There are separate procedures for `logging` and `opendistro-for-elasticsearch` roles since most of the times for `opendistro-for-elasticsearch`, `kibanaserver` and `logstash` users are not required to be present. +There are separate procedures for `logging` and `opensearch-for-elasticsearch` roles since most of the times for `opensearch-for-elasticsearch`, `kibanaserver` and `logstash` users are not required to be present. ### Logging component @@ -299,7 +299,7 @@ in [Logging role](#-logging-role). ### Open Distro for Elasticsearch component By default Epiphany removes all demo users except `admin` user. Those users are listed in `demo_users_to_remove` section -of `configuration/opendistro-for-elasticsearch` doc. If you want to keep `kibanaserver` user (needed by default Epiphany installation of Kibana), +of `configuration/opensearch-for-elasticsearch` doc. If you want to keep `kibanaserver` user (needed by default Epiphany installation of Kibana), you need to remove it from `demo_users_to_remove` list and set `kibanaserver_user_active` to `true` in order to change the default password. We strongly advice to set different password for each user. @@ -307,7 +307,7 @@ To change `admin` user's password, change value for `admin_password` key. For `k and `logstash_password` keys respectively. ```yaml -kind: configuration/opendistro-for-elasticsearch +kind: configuration/opensearch-for-elasticsearch title: Open Distro for Elasticsearch Config name: default specification: diff --git a/docs/home/howto/SECURITY_GROUPS.md b/docs/home/howto/SECURITY_GROUPS.md index f3e0b56855..27e9c524df 100644 --- a/docs/home/howto/SECURITY_GROUPS.md +++ b/docs/home/howto/SECURITY_GROUPS.md @@ -285,7 +285,7 @@ specification: count: 0 rabbitmq: count: 0 - opendistro_for_elasticsearch: + opensearch: count: 0 single_machine: count: 0 diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 9e387611d0..1c0169b820 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -194,7 +194,7 @@ specification: count: 1 rabbitmq: count: 0 - opendistro_for_elasticsearch: + opensearch: count: 0 name: clustername prefix: 'prefix' @@ -262,9 +262,9 @@ Before upgrade procedure make sure you have a data backup! --- -Since Epiphany v1.0.0 we provide upgrade elasticsearch-oss package to v7.10.2 and opendistro-\* plugins package to +Since Epiphany v1.0.0 we provide upgrade elasticsearch-oss package to v7.10.2 and opensearch-\* plugins package to v1.13.\*. Upgrade will be performed automatically when the upgrade procedure detects your `logging` -, `opendistro_for_elasticsearch` or `kibana` hosts. +, `opensearch` or `kibana` hosts. Upgrade of Elasticsearch uses API calls (GET, PUT, POST) which requires an admin TLS certificate. By default, Epiphany generates self-signed certificates for this purpose but if you use your own, you have to provide the admin certificate's @@ -277,7 +277,7 @@ logging: cert_path: /etc/elasticsearch/custom-admin.pem key_path: /etc/elasticsearch/custom-admin-key.pem -opendistro_for_elasticsearch: +opensearch: upgrade_config: custom_admin_certificate: cert_path: /etc/elasticsearch/custom-admin.pem diff --git a/schema/any/defaults/epiphany-cluster.yml b/schema/any/defaults/epiphany-cluster.yml index 4bc8719823..03850dcdb9 100644 --- a/schema/any/defaults/epiphany-cluster.yml +++ b/schema/any/defaults/epiphany-cluster.yml @@ -41,7 +41,7 @@ specification: count: 0 machine: rabbitmq-machine configuration: default - opendistro_for_elasticsearch: + opensearch: count: 0 machine: logging-machine configuration: default diff --git a/schema/aws/defaults/epiphany-cluster.yml b/schema/aws/defaults/epiphany-cluster.yml index f1dbd958a2..3a8bc2e8ff 100644 --- a/schema/aws/defaults/epiphany-cluster.yml +++ b/schema/aws/defaults/epiphany-cluster.yml @@ -80,7 +80,7 @@ specification: subnets: - availability_zone: eu-west-2a address_pool: 10.1.8.0/24 - opendistro_for_elasticsearch: + opensearch: count: 0 machine: logging-machine configuration: default diff --git a/schema/azure/defaults/epiphany-cluster.yml b/schema/azure/defaults/epiphany-cluster.yml index 331821bcea..28b3e5e00f 100644 --- a/schema/azure/defaults/epiphany-cluster.yml +++ b/schema/azure/defaults/epiphany-cluster.yml @@ -68,7 +68,7 @@ specification: configuration: default subnets: - address_pool: 10.1.8.0/24 - opendistro_for_elasticsearch: + opensearch: count: 0 machine: logging-machine configuration: default diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index 8050b2b9e3..a78035642a 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -17,7 +17,7 @@ specification: enabled: true - name: logging enabled: true - - name: opendistro-for-elasticsearch + - name: opensearch-for-elasticsearch enabled: true - name: elasticsearch-curator enabled: true @@ -121,8 +121,8 @@ specification: - node-exporter - filebeat - firewall - opendistro_for_elasticsearch: - - opendistro-for-elasticsearch + opensearch: + - opensearch-for-elasticsearch - node-exporter - filebeat - firewall diff --git a/schema/common/defaults/configuration/firewall.yml b/schema/common/defaults/configuration/firewall.yml index 8a9d66493c..3a8c2f2b33 100644 --- a/schema/common/defaults/configuration/firewall.yml +++ b/schema/common/defaults/configuration/firewall.yml @@ -71,7 +71,7 @@ specification: enabled: true ports: - 9100/tcp - opendistro_for_elasticsearch: + opensearch: enabled: true ports: - 9200/tcp diff --git a/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml b/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml index 9f3979d722..2b704ef494 100644 --- a/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml +++ b/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml @@ -1,4 +1,4 @@ -kind: configuration/opendistro-for-elasticsearch +kind: configuration/opensearch-for-elasticsearch title: Open Distro for Elasticsearch Config name: default specification: diff --git a/schema/common/validation/configuration/feature-mapping.yml b/schema/common/validation/configuration/feature-mapping.yml index 85b954b095..f6791f4b43 100644 --- a/schema/common/validation/configuration/feature-mapping.yml +++ b/schema/common/validation/configuration/feature-mapping.yml @@ -55,7 +55,7 @@ properties: type: array items: type: string - opendistro_for_elasticsearch: + opensearch: type: array items: type: string diff --git a/schema/common/validation/configuration/firewall.yml b/schema/common/validation/configuration/firewall.yml index 82148a9453..1de1ba9bcf 100644 --- a/schema/common/validation/configuration/firewall.yml +++ b/schema/common/validation/configuration/firewall.yml @@ -134,7 +134,7 @@ properties: type: array items: type: string - opendistro_for_elasticsearch: + opensearch: type: object properties: enabled: diff --git a/schema/common/validation/configuration/opendistro-for-elasticsearch.yml b/schema/common/validation/configuration/opendistro-for-elasticsearch.yml index 3992bc36ab..f240987b25 100644 --- a/schema/common/validation/configuration/opendistro-for-elasticsearch.yml +++ b/schema/common/validation/configuration/opendistro-for-elasticsearch.yml @@ -1,6 +1,6 @@ "$id": "#/specification" -title: "Opendistro-for-elasticsearch specification schema" -description: "Opendistro-for-elasticsearch specification schema" +title: "opensearch-for-elasticsearch specification schema" +description: "opensearch-for-elasticsearch specification schema" type: object properties: cluster_name: From 9aabac2d3720c5398056b10af7d254dc3cea9755 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 21 Jan 2022 11:10:32 +0100 Subject: [PATCH 002/157] ODFE-Ops migration --- .../opendistro_for_elasticsearch.yml | 2 +- .../playbooks/roles/backup/defaults/main.yml | 2 +- .../roles/filebeat/templates/filebeat.yml.j2 | 4 +-- .../roles/kibana/templates/kibana.yml.j2 | 8 ++--- .../playbooks/roles/logging/tasks/main.yml | 2 +- .../roles/opensearch/defaults/main.yml | 6 ++-- .../roles/opensearch/tasks/configure-ops.yml | 36 +++++++++---------- .../roles/opensearch/tasks/generate-certs.yml | 18 +++++----- .../playbooks/roles/opensearch/tasks/main.yml | 8 ++--- .../roles/opensearch/tasks/patch-log4j.yml | 6 ++-- .../roles/opensearch/templates/jvm.options.j2 | 2 +- .../opensearch/templates/opensearch.yml.j2 | 32 ++++++++--------- .../roles/preflight/defaults/main.yml | 4 +-- .../roles/recovery/defaults/main.yml | 2 +- .../tasks/logging_elasticsearch_etc.yml | 4 +-- .../tasks/logging_elasticsearch_snapshot.yml | 4 +-- .../centos-7/add-repositories.multiarch.sh | 10 +++--- .../redhat-7/add-repositories.multiarch.sh | 10 +++--- .../tasks/opendistro_for_elasticsearch-01.yml | 4 +-- .../opensearch/migrate-from-demo-certs-01.yml | 22 ++++++------ .../opensearch/migrate-from-demo-certs-02.yml | 10 +++--- .../migrate-from-demo-certs-non-clustered.yml | 22 ++++++------ .../opensearch/upgrade-opensearch-01.yml | 4 +-- .../opensearch/upgrade-opensearch-02.yml | 20 +++++------ .../utils/create-dual-cert-file.yml | 2 +- .../tasks/opensearch/utils/restart-node.yml | 2 +- docs/architecture/logical-view.md | 2 +- docs/changelogs/CHANGELOG-0.10.md | 4 +-- docs/changelogs/CHANGELOG-0.5.md | 4 +-- docs/changelogs/CHANGELOG-0.9.md | 2 +- docs/changelogs/CHANGELOG-1.3.md | 2 +- .../health-monitor/health-monitor.md | 2 +- docs/home/howto/DATABASES.md | 8 ++--- docs/home/howto/LOGGING.md | 6 ++-- docs/home/howto/MAINTENANCE.md | 4 +-- docs/home/howto/MONITORING.md | 4 +-- docs/home/howto/RETENTION.md | 2 +- .../configuration/feature-mapping.yml | 4 +-- .../common/defaults/configuration/logging.yml | 8 ++--- .../opendistro-for-elasticsearch.yml | 12 +++---- .../validation/configuration/logging.yml | 2 +- .../opendistro-for-elasticsearch.yml | 2 +- .../ip_change/elasticsearch/2_config_files.sh | 2 +- 43 files changed, 158 insertions(+), 158 deletions(-) diff --git a/ansible/playbooks/opendistro_for_elasticsearch.yml b/ansible/playbooks/opendistro_for_elasticsearch.yml index 4779a1efbe..89f753b35a 100644 --- a/ansible/playbooks/opendistro_for_elasticsearch.yml +++ b/ansible/playbooks/opendistro_for_elasticsearch.yml @@ -1,5 +1,5 @@ --- -# Ansible playbook for installing Elasticsearch +# Ansible playbook for installing opensearch - hosts: opensearch become: true diff --git a/ansible/playbooks/roles/backup/defaults/main.yml b/ansible/playbooks/roles/backup/defaults/main.yml index ca3f722d86..770caa87ec 100644 --- a/ansible/playbooks/roles/backup/defaults/main.yml +++ b/ansible/playbooks/roles/backup/defaults/main.yml @@ -3,4 +3,4 @@ backup_dir: /epibackup backup_destination_dir: "{{ backup_dir }}/mounted" backup_destination_host: "{{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}" elasticsearch_snapshot_repository_name: epiphany -elasticsearch_snapshot_repository_location: /var/lib/elasticsearch-snapshots +elasticsearch_snapshot_repository_location: /var/lib/opensearch-snapshots diff --git a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index f59e8bdfdd..32df1b338c 100644 --- a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -378,14 +378,14 @@ processors: #monitoring.enabled: false # Sets the UUID of the Elasticsearch cluster under which monitoring data for this -# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# Filebeat instance will appear in the Stack Monitoring UI. If output.opensearch # is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. #monitoring.cluster_uuid: # Uncomment to send the metrics to Elasticsearch. Most settings from the # Elasticsearch output are accepted here as well. # Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the Elasticsearch +# Any setting that is not set is automatically inherited from the opensearch # output configuration, so if you have the Elasticsearch output configured such # that it is pointing to your Elasticsearch monitoring cluster, you can simply # uncomment the following line. diff --git a/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 index ca8c03bf1e..7e7b73ff09 100644 --- a/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 +++ b/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 @@ -50,14 +50,14 @@ elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"] # Enables you to specify a file where Kibana stores log output. logging.dest: {{ specification.kibana_log_dir }}/kibana.log -opendistro_security.multitenancy.enabled: true -opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"] -opendistro_security.readonly_mode.roles: ["kibana_read_only"] +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] +opensearch_security.readonly_mode.roles: ["kibana_read_only"] # Provided with 1.10.1 version: # https://opendistro.github.io/for-elasticsearch-docs/docs/upgrade/1-10-1/ # Use this setting if you are running kibana without https -opendistro_security.cookie.secure: false +opensearch_security.cookie.secure: false newsfeed.enabled: false telemetry.optIn: false diff --git a/ansible/playbooks/roles/logging/tasks/main.yml b/ansible/playbooks/roles/logging/tasks/main.yml index 9262896c4b..62dd2319fc 100644 --- a/ansible/playbooks/roles/logging/tasks/main.yml +++ b/ansible/playbooks/roles/logging/tasks/main.yml @@ -10,7 +10,7 @@ run_once: true no_log: true # contains sensitive data -- name: Install and configure OpenDistro for Elasticsearch +- name: Install and configure OpenDistro for opensearch import_role: name: opensearch vars: diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index b0f5b1186d..33444a8239 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -8,7 +8,7 @@ versions: ops_version: "1.2.3" certificates: dirs: - certs: /etc/elasticsearch + certs: /etc/opensearch ca_key: /etc/elasticsearch/private csr: /etc/elasticsearch/csr dn_attributes_order: ['CN', 'OU', 'O', 'L', 'S', 'C', 'DC'] @@ -22,8 +22,8 @@ certificates: node: cert: esnode.pem key: esnode-key.pem - opendistro_security: - allow_unsafe_democertificates: false # if 'false' all demo files must be removed to start Elasticsearch + opensearch_security: + allow_unsafe_democertificates: false # if 'false' all demo files must be removed to start opensearch common: subject: &common-subject O: Epiphany diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index cbd2c5436a..16317901aa 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -5,8 +5,8 @@ file: path: "{{ specification.paths.repo }}/" state: directory - owner: elasticsearch - group: elasticsearch + owner: opensearch + group: opensearch mode: u=rwx,go= - name: Provide JVM configuration file @@ -15,7 +15,7 @@ src: jvm.options.j2 dest: /etc/elasticsearch/jvm.options owner: root - group: elasticsearch + group: opensearch mode: ug=rw,o= register: change_jvm_config vars: @@ -37,30 +37,30 @@ src: opensearch.yml.j2 dest: /etc/elasticsearch/elasticsearch.yml owner: root - group: elasticsearch + group: opensearch mode: ug=rw,o= register: change_config vars: node_cert_filename: http: >- - {{ existing_es_config['opendistro_security.ssl.http.pemcert_filepath'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.ssl.http.pemcert_filepath'] if (is_upgrade_run) else certificates.files.node.cert.filename }} transport: >- - {{ existing_es_config['opendistro_security.ssl.transport.pemcert_filepath'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.ssl.transport.pemcert_filepath'] if (is_upgrade_run) else certificates.files.node.cert.filename }} node_key_filename: http: >- - {{ existing_es_config['opendistro_security.ssl.http.pemkey_filepath'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.ssl.http.pemkey_filepath'] if (is_upgrade_run) else certificates.files.node.key.filename }} transport: >- - {{ existing_es_config['opendistro_security.ssl.transport.pemkey_filepath'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.ssl.transport.pemkey_filepath'] if (is_upgrade_run) else certificates.files.node.key.filename }} root_ca_cert_filename: http: >- - {{ existing_es_config['opendistro_security.ssl.http.pemtrustedcas_filepath'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.ssl.http.pemtrustedcas_filepath'] if (is_upgrade_run) else certificates.files.root_ca.cert.filename }} transport: >- - {{ existing_es_config['opendistro_security.ssl.transport.pemtrustedcas_filepath'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.ssl.transport.pemtrustedcas_filepath'] if (is_upgrade_run) else certificates.files.root_ca.cert.filename }} _epiphany_subjects: admin: "{{ certificates.files.admin.cert.subject }}" @@ -76,7 +76,7 @@ {{ _epiphany_dn_attributes.node | zip(_epiphany_dn_attributes.node | map('extract', _epiphany_subjects.node)) | map('join','=') | join(',') }} admin_dn: >- - {{ existing_es_config['opendistro_security.authcz.admin_dn'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.authcz.admin_dn'] if (is_upgrade_run) else [ _epiphany_DNs.admin ] }} _epiphany_nodes_dn: >- {%- if groups[current_group_name] | length > 1 -%} @@ -90,14 +90,14 @@ {%- if not loop.last -%},{%- else -%}]{%- endif -%} {%- endfor -%} nodes_dn: >- - {{ existing_es_config['opendistro_security.nodes_dn'] if (is_upgrade_run) else + {{ existing_es_config['opensearch_security.nodes_dn'] if (is_upgrade_run) else _epiphany_nodes_dn }} - opendistro_security_allow_unsafe_democertificates: "{{ certificates.files.demo.opendistro_security.allow_unsafe_democertificates }}" + opensearch_security_allow_unsafe_democertificates: "{{ certificates.files.demo.opensearch_security.allow_unsafe_democertificates }}" http_port: "{{ is_upgrade_run | ternary(existing_es_config['http.port'], ports.http) }}" transport_port: "{{ is_upgrade_run | ternary(existing_es_config['transport.port'], ports.transport) }}" -# When 'opendistro_security.allow_unsafe_democertificates' is set to 'false' all demo certificate files must be removed, +# When 'opensearch_security.allow_unsafe_democertificates' is set to 'false' all demo certificate files must be removed, # otherwise elasticsearch service doesn't start. # For apply mode, demo certificate files are removed based only on their names. For upgrade mode, # public key fingerprints are checked to protect against unintentional deletion (what takes additional time). @@ -105,16 +105,16 @@ - name: Remove demo certificate files include_tasks: file: "{{ is_upgrade_run | ternary('remove-known-demo-certs.yml', 'remove-demo-certs.yml') }}" - when: not certificates.files.demo.opendistro_security.allow_unsafe_democertificates + when: not certificates.files.demo.opensearch_security.allow_unsafe_democertificates - name: Include log4j patch include_tasks: patch-log4j.yml - name: Restart elasticsearch service systemd: - name: elasticsearch + name: opensearch state: restarted - register: restart_elasticsearch + register: restart_opensearch when: change_config.changed or log4j_patch.changed or change_jvm_config.changed @@ -123,7 +123,7 @@ - name: Enable and start elasticsearch service systemd: - name: elasticsearch + name: opensearch state: started enabled: yes diff --git a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml index 898d6cbe35..6edf0585ef 100644 --- a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml +++ b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml @@ -6,7 +6,7 @@ state: directory path: "{{ certificates.dirs.ca_key }}" owner: root - group: elasticsearch + group: opensearch mode: u=rwx,g=rx,o= # elasticsearch.service requires 'rx' for group # csr files are kept only for idempotency @@ -15,7 +15,7 @@ state: directory path: "{{ certificates.dirs.csr }}" owner: root - group: elasticsearch + group: opensearch mode: u=rwx,g=rx,o= # CSR file doesn't contain private key - name: Generate keys and certificates on first node @@ -26,19 +26,19 @@ type: RSA mode: u=rw,go= owner: root - group: elasticsearch + group: opensearch format: pkcs8 community.crypto.openssl_csr: mode: u=rw,g=r,o= owner: root - group: elasticsearch + group: opensearch use_common_name_for_san: false community.crypto.x509_certificate: selfsigned_digest: sha256 ownca_digest: sha256 mode: u=rw,g=r,o= owner: root - group: elasticsearch + group: opensearch block: # --- Generate CA root certificate --- @@ -122,7 +122,7 @@ module_defaults: copy: owner: root - group: elasticsearch + group: opensearch block: - name: Get certificate files from the first host slurp: @@ -173,7 +173,7 @@ type: RSA mode: u=rw,g=r,o= # elasticsearch.service requires 'r' for group owner: root - group: elasticsearch + group: opensearch return_content: false register: node_key @@ -200,7 +200,7 @@ use_common_name_for_san: false mode: u=rw,g=r,o= owner: root - group: elasticsearch + group: opensearch register: node_csr vars: _unique_hostnames: "{{ [ansible_hostname, ansible_nodename, ansible_fqdn] | unique }}" @@ -218,4 +218,4 @@ ownca_digest: sha256 mode: u=rw,go=r owner: root - group: elasticsearch + group: opensearch diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index c04c7d3c4b..34de2b377f 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Configure OS limits (open files, processes and locked-in-memory address space) pam_limits: - domain: elasticsearch + domain: opensearch limit_type: "{{ item.limit_type }}" limit_item: "{{ item.limit_item }}" value: "{{ item.value }}" @@ -15,7 +15,7 @@ - include_tasks: install-ops.yml -- include_tasks: install-ops-plugins.yml +# - include_tasks: install-ops-plugins.yml -- name: Include configuration tasks - include_tasks: configure-ops.yml +# - name: Include configuration tasks +# include_tasks: configure-ops.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml index 9b996e2d34..1947ae27b8 100644 --- a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml +++ b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml @@ -21,7 +21,7 @@ copy: src: "{{ item.src }}" dest: "{{ item.dest }}" - owner: elasticsearch + owner: opensearch group: root mode: u=rw,g=r,o= remote_src: true @@ -32,7 +32,7 @@ - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/lib/ } - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_slfj_impl }}", dest: /usr/share/elasticsearch/plugins/opendistro_security/ } + - { src: "{{ download_directory }}/{{ log4j_slfj_impl }}", dest: /usr/share/elasticsearch/plugins/opensearch_security/ } vars: log4j_api: "{{ unarchive_list_files.files | select('contains', 'log4j-api-2.17.1.jar') | first }}" log4j_core: "{{ unarchive_list_files.files | select('contains', 'log4j-core-2.17.1.jar') | first }}" @@ -51,7 +51,7 @@ - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - /usr/share/elasticsearch/lib/log4j-api-2.11.1.jar - /usr/share/elasticsearch/lib/log4j-core-2.11.1.jar - - /usr/share/elasticsearch/plugins/opendistro_security/log4j-slf4j-impl-2.11.1.jar + - /usr/share/elasticsearch/plugins/opensearch_security/log4j-slf4j-impl-2.11.1.jar - name: Log4j patch | Delete temporary dir file: diff --git a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 index e91e6b6635..1fcfe8da4c 100644 --- a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 +++ b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 @@ -61,7 +61,7 @@ # specify an alternative path for heap dumps; ensure the directory exists and # has sufficient space --XX:HeapDumpPath=/var/lib/elasticsearch +-XX:HeapDumpPath=/var/lib/opensearch # specify an alternative path for JVM fatal error logs -XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 index 0214fcc7d0..cb09da9dcc 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 @@ -111,31 +111,31 @@ cluster.initial_master_nodes: ["{{ ansible_hostname }}"] ######## Start OpenDistro for Elasticsearch Security Configuration ######## # WARNING: revise all the lines below before you go into production -opendistro_security.ssl.transport.pemcert_filepath: {{ node_cert_filename.transport }} -opendistro_security.ssl.transport.pemkey_filepath: {{ node_key_filename.transport }} -opendistro_security.ssl.transport.pemtrustedcas_filepath: {{ root_ca_cert_filename.transport }} -opendistro_security.ssl.transport.enforce_hostname_verification: {{ specification.opendistro_security.ssl.transport.enforce_hostname_verification | lower }} -opendistro_security.ssl.http.enabled: true -opendistro_security.ssl.http.pemcert_filepath: {{ node_cert_filename.http }} -opendistro_security.ssl.http.pemkey_filepath: {{ node_key_filename.http }} -opendistro_security.ssl.http.pemtrustedcas_filepath: {{ root_ca_cert_filename.http }} -opendistro_security.allow_unsafe_democertificates: {{ opendistro_security_allow_unsafe_democertificates | lower }} -opendistro_security.allow_default_init_securityindex: true -opendistro_security.authcz.admin_dn: +opensearch_security.ssl.transport.pemcert_filepath: {{ node_cert_filename.transport }} +opensearch_security.ssl.transport.pemkey_filepath: {{ node_key_filename.transport }} +opensearch_security.ssl.transport.pemtrustedcas_filepath: {{ root_ca_cert_filename.transport }} +opensearch_security.ssl.transport.enforce_hostname_verification: {{ specification.opensearch_security.ssl.transport.enforce_hostname_verification | lower }} +opensearch_security.ssl.http.enabled: true +opensearch_security.ssl.http.pemcert_filepath: {{ node_cert_filename.http }} +opensearch_security.ssl.http.pemkey_filepath: {{ node_key_filename.http }} +opensearch_security.ssl.http.pemtrustedcas_filepath: {{ root_ca_cert_filename.http }} +opensearch_security.allow_unsafe_democertificates: {{ opensearch_security_allow_unsafe_democertificates | lower }} +opensearch_security.allow_default_init_securityindex: true +opensearch_security.authcz.admin_dn: {% for dn in admin_dn %} - '{{ dn }}' {% endfor %} {% if nodes_dn | count > 0 %} -opendistro_security.nodes_dn: +opensearch_security.nodes_dn: {% for dn in nodes_dn %} - '{{ dn }}' {% endfor %} {% endif %} -opendistro_security.audit.type: internal_elasticsearch -opendistro_security.enable_snapshot_restore_privilege: true -opendistro_security.check_snapshot_restore_write_privileges: true -opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +opensearch_security.audit.type: internal_opensearch +opensearch_security.enable_snapshot_restore_privilege: true +opensearch_security.check_snapshot_restore_write_privileges: true +opensearch_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] cluster.routing.allocation.disk.threshold_enabled: false node.max_local_storage_nodes: 3 ######## End OpenDistro for Elasticsearch Security Configuration ######## diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index f1bfc1b8c1..5972f161b4 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -39,7 +39,7 @@ unsupported_roles: - logging - elasticsearch_curator - opensearch - - elasticsearch + - opensearch - kibana - filebeat - prometheus @@ -76,7 +76,7 @@ unsupported_roles: - logging - elasticsearch_curator - opensearch - - elasticsearch + - opensearch - kibana - filebeat - prometheus diff --git a/ansible/playbooks/roles/recovery/defaults/main.yml b/ansible/playbooks/roles/recovery/defaults/main.yml index 88be45c8a6..d11dc98aef 100644 --- a/ansible/playbooks/roles/recovery/defaults/main.yml +++ b/ansible/playbooks/roles/recovery/defaults/main.yml @@ -3,4 +3,4 @@ recovery_dir: /epibackup recovery_source_dir: "{{ recovery_dir }}/mounted" recovery_source_host: "{{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}" elasticsearch_snapshot_repository_name: epiphany -elasticsearch_snapshot_repository_location: /var/lib/elasticsearch-snapshots +elasticsearch_snapshot_repository_location: /var/lib/opensearch-snapshots diff --git a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml index 7c81954bf5..427dab580b 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml @@ -17,7 +17,7 @@ - name: Stop elasticsearch service systemd: - name: elasticsearch + name: opensearch state: stopped - name: Clear directories @@ -34,5 +34,5 @@ - name: Start elasticsearch service systemd: - name: elasticsearch + name: opensearch state: started diff --git a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml index 4bab27d76d..fa43b02982 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml @@ -56,8 +56,8 @@ - name: Change snapshot directory permissions file: path: "{{ elasticsearch_snapshot_repository_location }}/" - owner: elasticsearch - group: elasticsearch + owner: opensearch + group: opensearch recurse: true - name: Reconstruct the snapshot_name diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh index c59a6a3e74..45287ee5f3 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh @@ -15,7 +15,7 @@ ELASTIC_6_REPO_CONF=$(cat <<'EOF' name=Elastic repository for 6.x packages baseurl=https://artifacts.elastic.co/packages/oss-6.x/yum gpgcheck=1 -gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +gpgkey=https://artifacts.elastic.co/GPG-KEY-opensearch enabled=1 autorefresh=1 type=rpm-md @@ -27,7 +27,7 @@ ELASTICSEARCH_7_REPO_CONF=$(cat <<'EOF' name=Elasticsearch repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/oss-7.x/yum gpgcheck=1 -gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +gpgkey=https://artifacts.elastic.co/GPG-KEY-opensearch enabled=1 autorefresh=1 type=rpm-md @@ -39,7 +39,7 @@ ELASTICSEARCH_CURATOR_REPO_CONF=$(cat <<'EOF' name=CentOS/RHEL 7 repository for Elasticsearch Curator 5.x packages baseurl=https://packages.elastic.co/curator/5/centos/7 gpgcheck=1 -gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch +gpgkey=https://packages.elastic.co/GPG-KEY-opensearch enabled=1 EOF ) @@ -57,10 +57,10 @@ EOF OPENDISTRO_REPO_CONF=$(cat <<'EOF' [opendistroforelasticsearch-artifacts-repo] -name=Release RPM artifacts of OpenDistroForElasticsearch +name=Release RPM artifacts of OpenDistroForopensearch baseurl=https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/ enabled=1 -gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch +gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforopensearch gpgcheck=1 repo_gpgcheck=1 autorefresh=1 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh index c59a6a3e74..45287ee5f3 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh @@ -15,7 +15,7 @@ ELASTIC_6_REPO_CONF=$(cat <<'EOF' name=Elastic repository for 6.x packages baseurl=https://artifacts.elastic.co/packages/oss-6.x/yum gpgcheck=1 -gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +gpgkey=https://artifacts.elastic.co/GPG-KEY-opensearch enabled=1 autorefresh=1 type=rpm-md @@ -27,7 +27,7 @@ ELASTICSEARCH_7_REPO_CONF=$(cat <<'EOF' name=Elasticsearch repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/oss-7.x/yum gpgcheck=1 -gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +gpgkey=https://artifacts.elastic.co/GPG-KEY-opensearch enabled=1 autorefresh=1 type=rpm-md @@ -39,7 +39,7 @@ ELASTICSEARCH_CURATOR_REPO_CONF=$(cat <<'EOF' name=CentOS/RHEL 7 repository for Elasticsearch Curator 5.x packages baseurl=https://packages.elastic.co/curator/5/centos/7 gpgcheck=1 -gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch +gpgkey=https://packages.elastic.co/GPG-KEY-opensearch enabled=1 EOF ) @@ -57,10 +57,10 @@ EOF OPENDISTRO_REPO_CONF=$(cat <<'EOF' [opendistroforelasticsearch-artifacts-repo] -name=Release RPM artifacts of OpenDistroForElasticsearch +name=Release RPM artifacts of OpenDistroForopensearch baseurl=https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/ enabled=1 -gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch +gpgkey=https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforopensearch gpgcheck=1 repo_gpgcheck=1 autorefresh=1 diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml index 7c3d910d92..d7fc09b7ef 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml @@ -23,9 +23,9 @@ - name: Restart elasticsearch service systemd: - name: elasticsearch + name: opensearch state: restarted - register: restart_elasticsearch + register: restart_opensearch when: odfe_defaults.log4j_file_name is defined and log4j_patch.changed - name: ODFE | Print elasticsearch-oss versions diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml index f9a83243b5..4d7b11a221 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml @@ -8,10 +8,10 @@ # 3. Save cluster status to file (done in pre-migration part) # 4. Create dual root CA file for the migration (demo + Epiphany root CAs concatenated), needed temporarily # 5. Patch the following properties in existing elasticsearch.yml: -# a) opendistro_security.authcz.admin_dn - add Epiphany admin cert -# b) opendistro_security.nodes_dn - by default not present, add all Epiphany node certs -# c) opendistro_security.ssl.http.pemtrustedcas_filepath - replace demo root CA with the dual root CA file -# d) opendistro_security.ssl.transport.pemtrustedcas_filepath - replace demo root CA with the dual root CA file +# a) opensearch_security.authcz.admin_dn - add Epiphany admin cert +# b) opensearch_security.nodes_dn - by default not present, add all Epiphany node certs +# c) opensearch_security.ssl.http.pemtrustedcas_filepath - replace demo root CA with the dual root CA file +# d) opensearch_security.ssl.transport.pemtrustedcas_filepath - replace demo root CA with the dual root CA file # B) Serial part (node by node) - tasks from migrate-from-demo-certs-02.yml # Create dual root CA transitional file @@ -33,7 +33,7 @@ content: "{{ _patched_content | to_nice_yaml }}" mode: u=rw,g=rw,o= owner: root - group: elasticsearch + group: opensearch backup: true vars: _epiphany_subjects: @@ -58,14 +58,14 @@ _old_content: >- {{ _elasticsearch_yml.content | b64decode | from_yaml }} _updated_settings: - opendistro_security.authcz.admin_dn: >- - {{ _old_content['opendistro_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') + opensearch_security.authcz.admin_dn: >- + {{ _old_content['opensearch_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') | union([opensearch.certs_migration.demo_DNs.admin] + [_epiphany_DNs.admin]) }} - opendistro_security.nodes_dn: >- - {{ _old_content['opendistro_security.nodes_dn'] | default([]) + opensearch_security.nodes_dn: >- + {{ _old_content['opensearch_security.nodes_dn'] | default([]) | union([opensearch.certs_migration.demo_DNs.node] + _epiphany_nodes_dn) }} - opendistro_security.ssl.http.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" - opendistro_security.ssl.transport.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" + opensearch_security.ssl.http.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" + opensearch_security.ssl.transport.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" _patched_content: >- {{ _old_content | combine(_updated_settings) }} diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml index 223f6968df..c93fb4c028 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml @@ -46,7 +46,7 @@ content: "{{ _patched_content | to_nice_yaml }}" mode: u=rw,g=rw,o= owner: root - group: elasticsearch + group: opensearch backup: true delegate_to: "{{ target_inventory_hostname }}" delegate_facts: true @@ -62,10 +62,10 @@ _old_content: >- {{ _elasticsearch_yml.results[loop_index0].content | b64decode | from_yaml }} _updated_settings: - opendistro_security.ssl.http.pemcert_filepath: "{{ _epiphany_node_cert.cert_filename }}" - opendistro_security.ssl.http.pemkey_filepath: "{{ _epiphany_node_cert.key_filename }}" - opendistro_security.ssl.transport.pemcert_filepath: "{{ _epiphany_node_cert.cert_filename }}" - opendistro_security.ssl.transport.pemkey_filepath: "{{ _epiphany_node_cert.key_filename }}" + opensearch_security.ssl.http.pemcert_filepath: "{{ _epiphany_node_cert.cert_filename }}" + opensearch_security.ssl.http.pemkey_filepath: "{{ _epiphany_node_cert.key_filename }}" + opensearch_security.ssl.transport.pemcert_filepath: "{{ _epiphany_node_cert.cert_filename }}" + opensearch_security.ssl.transport.pemkey_filepath: "{{ _epiphany_node_cert.key_filename }}" _patched_content: >- {{ _old_content | combine(_updated_settings) }} diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml index addd327aa3..273c8508c1 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml @@ -10,7 +10,7 @@ content: "{{ _patched_content | to_nice_yaml }}" mode: u=rw,g=rw,o= owner: root - group: elasticsearch + group: opensearch backup: true vars: _epiphany_subjects: @@ -29,20 +29,20 @@ _old_content: >- {{ _elasticsearch_yml.content | b64decode | from_yaml }} _updated_settings: - opendistro_security.authcz.admin_dn: >- - {{ _old_content['opendistro_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') + opensearch_security.authcz.admin_dn: >- + {{ _old_content['opensearch_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') | union([_epiphany_DNs.admin]) }} - opendistro_security.nodes_dn: >- - {{ _old_content['opendistro_security.nodes_dn'] | default([]) + opensearch_security.nodes_dn: >- + {{ _old_content['opensearch_security.nodes_dn'] | default([]) | union([_epiphany_DNs.node]) }} - opendistro_security.ssl.http.pemcert_filepath: "{{ certificates.files.node.cert.filename }}" - opendistro_security.ssl.http.pemkey_filepath: "{{ certificates.files.node.key.filename }}" - opendistro_security.ssl.transport.pemcert_filepath: "{{ certificates.files.node.cert.filename }}" - opendistro_security.ssl.transport.pemkey_filepath: "{{ certificates.files.node.key.filename }}" + opensearch_security.ssl.http.pemcert_filepath: "{{ certificates.files.node.cert.filename }}" + opensearch_security.ssl.http.pemkey_filepath: "{{ certificates.files.node.key.filename }}" + opensearch_security.ssl.transport.pemcert_filepath: "{{ certificates.files.node.cert.filename }}" + opensearch_security.ssl.transport.pemkey_filepath: "{{ certificates.files.node.key.filename }}" - opendistro_security.ssl.http.pemtrustedcas_filepath: "{{ certificates.files.root_ca.cert.filename }}" - opendistro_security.ssl.transport.pemtrustedcas_filepath: "{{ certificates.files.root_ca.cert.filename }}" + opensearch_security.ssl.http.pemtrustedcas_filepath: "{{ certificates.files.root_ca.cert.filename }}" + opensearch_security.ssl.transport.pemtrustedcas_filepath: "{{ certificates.files.root_ca.cert.filename }}" _patched_content: >- {{ _old_content | combine(_updated_settings) }} diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml index 675c70e605..720531147a 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml @@ -10,7 +10,7 @@ - name: ODFE | Ensure elasticsearch service is running systemd: - name: elasticsearch + name: opensearch enabled: yes state: started register: elasticsearch_state @@ -54,7 +54,7 @@ - name: ODFE | Get information on root CA certificate community.crypto.x509_certificate_info: # 'pemtrustedcas_filepath' is a relative path - path: "{{ ('/etc/elasticsearch', existing_config.main['opendistro_security.ssl.transport.pemtrustedcas_filepath']) | path_join }}" + path: "{{ ('/etc/elasticsearch', existing_config.main['opensearch_security.ssl.transport.pemtrustedcas_filepath']) | path_join }}" register: _root_ca_info - name: ODFE | Check if demo or Epiphany certificates are in use # self-signed diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml index 5fe1f4a8df..1eea82b5d8 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml @@ -19,7 +19,7 @@ - name: ODFE | Stop elasticsearch service systemd: - name: elasticsearch + name: opensearch state: stopped - name: ODFE | Include Elasticsearch installation tasks @@ -40,12 +40,12 @@ clustered: "{{ 'True' if _old['discovery.seed_hosts'] | length > 1 else 'False' }}" paths: data: "{{ _old['path.data'] }}" - repo: "{{ _old['path.repo'] | default('/var/lib/elasticsearch-snapshots') }}" # absent in Epiphany v0.6 thus we use default + repo: "{{ _old['path.repo'] | default('/var/lib/opensearch-snapshots') }}" # absent in Epiphany v0.6 thus we use default logs: "{{ _old['path.logs'] }}" - opendistro_security: + opensearch_security: ssl: transport: - enforce_hostname_verification: "{{ _old['opendistro_security.ssl.transport.enforce_hostname_verification'] }}" + enforce_hostname_verification: "{{ _old['opensearch_security.ssl.transport.enforce_hostname_verification'] }}" _demo_DNs: admin: "{{ opensearch.certs_migration.demo_DNs.admin }}" @@ -53,12 +53,12 @@ _dual_root_ca_filename: "{{ opensearch.certs_migration.dual_root_ca.filename }}" _epiphany_root_ca_filename: "{{ certificates.files.root_ca.cert.filename }}" _updated_existing_config: - opendistro_security.authcz.admin_dn: "{{ _old['opendistro_security.authcz.admin_dn'] | reject('search', _demo_DNs.admin) }}" - opendistro_security.nodes_dn: "{{ _old['opendistro_security.nodes_dn'] | default([]) | reject('search', _demo_DNs.node) }}" - opendistro_security.ssl.http.pemtrustedcas_filepath: >- - {{ _old['opendistro_security.ssl.http.pemtrustedcas_filepath'] | replace(_dual_root_ca_filename, _epiphany_root_ca_filename) }} - opendistro_security.ssl.transport.pemtrustedcas_filepath: >- - {{ _old['opendistro_security.ssl.transport.pemtrustedcas_filepath'] | replace(_dual_root_ca_filename, _epiphany_root_ca_filename) }} + opensearch_security.authcz.admin_dn: "{{ _old['opensearch_security.authcz.admin_dn'] | reject('search', _demo_DNs.admin) }}" + opensearch_security.nodes_dn: "{{ _old['opensearch_security.nodes_dn'] | default([]) | reject('search', _demo_DNs.node) }}" + opensearch_security.ssl.http.pemtrustedcas_filepath: >- + {{ _old['opensearch_security.ssl.http.pemtrustedcas_filepath'] | replace(_dual_root_ca_filename, _epiphany_root_ca_filename) }} + opensearch_security.ssl.transport.pemtrustedcas_filepath: >- + {{ _old['opensearch_security.ssl.transport.pemtrustedcas_filepath'] | replace(_dual_root_ca_filename, _epiphany_root_ca_filename) }} http.port: "{{ _old['http.port'] | default(odfe_defaults.ports.http) }}" transport.port: "{{ _old['transport.port'] | default(odfe_defaults.ports.transport) }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml index 01946b94f6..d655dc1887 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml @@ -15,4 +15,4 @@ content: "{{ _files.results | map(attribute='content') | map('b64decode') | join('') }}" mode: u=rw,g=r,o= owner: root - group: elasticsearch + group: opensearch diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml index c6348f7ee9..772d4e0390 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml @@ -12,7 +12,7 @@ - name: ODFE | Restart elasticsearch service systemd: - name: elasticsearch + name: opensearch state: restarted daemon_reload: "{{ daemon_reload | default(omit) }}" diff --git a/docs/architecture/logical-view.md b/docs/architecture/logical-view.md index d82c707d16..7a33aca8be 100644 --- a/docs/architecture/logical-view.md +++ b/docs/architecture/logical-view.md @@ -54,7 +54,7 @@ Docker containers | Kubernetes components that run in a container `Filebeat`, unlike `Grafana`, pushes data to database (`Elasticsearch`) instead of pulling them. [Read more](https://www.elastic.co/products/beats/filebeat) about `Filebeat`. -### Elasticsearch +### opensearch `Elasticsearch` is highly scalable and full-text search enabled analytics engine. Epiphany Platform uses it for storage and analysis of logs. diff --git a/docs/changelogs/CHANGELOG-0.10.md b/docs/changelogs/CHANGELOG-0.10.md index 444099610e..30201c9086 100644 --- a/docs/changelogs/CHANGELOG-0.10.md +++ b/docs/changelogs/CHANGELOG-0.10.md @@ -12,12 +12,12 @@ Version 0.10 won't be supported anymore. Instead, we introduced version 1.0 whic - [#1798](https://github.com/epiphany-platform/epiphany/issues/1798) - Additional alerts for Prometheus - [#1355](https://github.com/epiphany-platform/epiphany/issues/1355) - Updating cloud based OS images - configuration required for Azure RHEL LVM images - [#2081](https://github.com/epiphany-platform/epiphany/issues/2081) - Replace Skopeo with Crane -- [#1323](https://github.com/epiphany-platform/epiphany/issues/1323) - Documentation how to handle data in Opendistro for Elasticsearch +- [#1323](https://github.com/epiphany-platform/epiphany/issues/1323) - Documentation how to handle data in Opendistro for opensearch - [#1789](https://github.com/epiphany-platform/epiphany/issues/1789) - [Ubuntu] Add retry feature for downloading packages in download-requirements.sh ### Fixed -- [#1870](https://github.com/epiphany-platform/epiphany/issues/1870) - Do not install Filebeat when there is no Elasticsearch +- [#1870](https://github.com/epiphany-platform/epiphany/issues/1870) - Do not install Filebeat when there is no opensearch - [#1881](https://github.com/epiphany-platform/epiphany/issues/1881) - epicli: wrong informations in help messages - [#1959](https://github.com/epiphany-platform/epiphany/issues/1959) - Network traffic not allowed from load balancer's subnet to Kubernetes's subnet in AWS - [#1991](https://github.com/epiphany-platform/epiphany/issues/1991) - When custom repo is used backup/recovery stops working diff --git a/docs/changelogs/CHANGELOG-0.5.md b/docs/changelogs/CHANGELOG-0.5.md index 1d49ed03a0..198400405d 100644 --- a/docs/changelogs/CHANGELOG-0.5.md +++ b/docs/changelogs/CHANGELOG-0.5.md @@ -50,7 +50,7 @@ - [#854](https://github.com/epiphany-platform/epiphany/issues/854) - PostgreSQL: PGBouncer implementation - [#905](https://github.com/epiphany-platform/epiphany/pull/905) - PostgreSQL: pgAudit extension for audit logging -- PostgreSQL: Send logs to Elasticsearch +- PostgreSQL: Send logs to opensearch - [#915](https://github.com/epiphany-platform/epiphany/pull/915) - PostgreSQL: Add logrotate configuration ### Fixed @@ -82,7 +82,7 @@ - [#381](https://github.com/epiphany-platform/epiphany/issues/381) - Add AWS EC2 Root Volume encryption - [#782](https://github.com/epiphany-platform/epiphany/issues/781) - All disks encryption documentation - AWS - [#782](https://github.com/epiphany-platform/epiphany/issues/782) - All disks encryption documentation - Azure -- [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for Elasticsearch +- [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for opensearch - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch) - [Centralized logging](/docs/home/howto/LOGGING.md#centralized-logging-setup) diff --git a/docs/changelogs/CHANGELOG-0.9.md b/docs/changelogs/CHANGELOG-0.9.md index 1b0876d81a..1321890269 100644 --- a/docs/changelogs/CHANGELOG-0.9.md +++ b/docs/changelogs/CHANGELOG-0.9.md @@ -23,7 +23,7 @@ - [#1888](https://github.com/epiphany-platform/epiphany/issues/1888) - epicli upgrade of cluster created by Epiphany v0.5 may fail - [#1884](https://github.com/epiphany-platform/epiphany/issues/1884) - Prometheus is not able to scrape metrics from AKS/EKS nodes - [#1887](https://github.com/epiphany-platform/epiphany/issues/1887) - epicli upgrade of cluster created by Epiphany v0.6 fails on "Store preflight facts" task -- [#1866](https://github.com/epiphany-platform/epiphany/issues/1866) - No logs from K8s apps in Elasticsearch +- [#1866](https://github.com/epiphany-platform/epiphany/issues/1866) - No logs from K8s apps in opensearch ### Updated diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index 44dbb461cc..e45eab2ce4 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -68,7 +68,7 @@ - [#2748](https://github.com/epiphany-platform/epiphany/issues/2748) - Upgrade Kafka exporter to the version 1.4.0 - [#2750](https://github.com/epiphany-platform/epiphany/issues/2750) - Upgrade JMX exporter to the newest version - [#2699](https://github.com/epiphany-platform/epiphany/issues/2699) - Upgrade Grafana to 8.3.2 -- [#2788](https://github.com/epiphany-platform/epiphany/issues/2788) - Upgrade Log4j in Open Distro for Elasticsearch +- [#2788](https://github.com/epiphany-platform/epiphany/issues/2788) - Upgrade Log4j in Open Distro for opensearch - [#2661](https://github.com/epiphany-platform/epiphany/issues/2661) - Update K8s documentation according to the latest version Epiphany supports - [#2752](https://github.com/epiphany-platform/epiphany/issues/2752) - Upgrade postgresql exporter to the version 0.10.0 - [#2856](https://github.com/epiphany-platform/epiphany/issues/2856) - Update cloud OS images to the latest diff --git a/docs/design-docs/health-monitor/health-monitor.md b/docs/design-docs/health-monitor/health-monitor.md index f66170f3b3..bdb0ec8c70 100644 --- a/docs/design-docs/health-monitor/health-monitor.md +++ b/docs/design-docs/health-monitor/health-monitor.md @@ -21,7 +21,7 @@ Components that Health Monitor should check: - Prometheus - Kafka - ZooKeeper -- ElasticSearch +- opensearch - RabbitMQ `*` means MVP version. diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 1e81806a8c..4e49708b8a 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -455,9 +455,9 @@ Properly configured application (kubernetes service) to use fully HA configurati PostgreSQL native replication is now deprecated and removed. Use [PostgreSQL HA replication with repmgr](#how-to-set-up-postgresql-ha-replication-with-repmgr-cluster) instead. -## How to start working with OpenDistro for Elasticsearch +## How to start working with OpenDistro for opensearch -OpenDistro for Elasticsearch +OpenDistro for opensearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration: @@ -482,7 +482,7 @@ specification: **Installation with more than one node will always be clustered** - Option to configure the non-clustered installation of more than one node for Open Distro is not supported. ```yaml -kind: configuration/opensearch-for-elasticsearch +kind: configuration/opensearch-for-opensearch title: OpenDistro for Elasticsearch Config name: default specification: @@ -499,7 +499,7 @@ name: default specification: roles_mapping: opensearch: - - opensearch-for-elasticsearch + - opensearch-for-opensearch - node-exporter - filebeat - firewall diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index dd57f26da7..18db4b909d 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -48,9 +48,9 @@ specification: cluster_name: EpiphanyElastic clustered: True paths: - data: /var/lib/elasticsearch - repo: /var/lib/elasticsearch-snapshots - logs: /var/log/elasticsearch + data: /var/lib/opensearch + repo: /var/lib/opensearch-snapshots + logs: /var/log/opensearch ``` ## How to manage Opendistro for Elasticsearch data diff --git a/docs/home/howto/MAINTENANCE.md b/docs/home/howto/MAINTENANCE.md index e9497faa6c..765ac61097 100644 --- a/docs/home/howto/MAINTENANCE.md +++ b/docs/home/howto/MAINTENANCE.md @@ -112,12 +112,12 @@ To check status of Node Exporter we can use the command: status prometheus-node-exporter ``` -#### - Elasticsearch +#### - opensearch To check status of Elasticsearch we can use the command: ```shell -systemct status elasticsearch +systemct status opensearch ``` We can check if service is listening on 9200 (API communication port): diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index 62e4b3cdbf..c717c1f52b 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -307,8 +307,8 @@ To change `admin` user's password, change value for `admin_password` key. For `k and `logstash_password` keys respectively. ```yaml -kind: configuration/opensearch-for-elasticsearch -title: Open Distro for Elasticsearch Config +kind: configuration/opensearch-for-opensearch +title: OpenSearch Config name: default specification: ... diff --git a/docs/home/howto/RETENTION.md b/docs/home/howto/RETENTION.md index 3fa6ccdb9c..3ba3a3dc5e 100644 --- a/docs/home/howto/RETENTION.md +++ b/docs/home/howto/RETENTION.md @@ -1,7 +1,7 @@ An Epiphany cluster has a number of components which log, collect and retain data. To make sure that these do not exceed the usable storage of the machines they running on, the following configurations are available. -## Elasticsearch +## opensearch TODO diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index a78035642a..e4c05ca6c4 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -17,7 +17,7 @@ specification: enabled: true - name: logging enabled: true - - name: opensearch-for-elasticsearch + - name: opensearch-for-opensearch enabled: true - name: elasticsearch-curator enabled: true @@ -122,7 +122,7 @@ specification: - filebeat - firewall opensearch: - - opensearch-for-elasticsearch + - opensearch-for-opensearch - node-exporter - filebeat - firewall diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index be687c2e65..bcd1992463 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -13,12 +13,12 @@ specification: - readall - snapshotrestore paths: - data: /var/lib/elasticsearch - repo: /var/lib/elasticsearch-snapshots - logs: /var/log/elasticsearch + data: /var/lib/opensearch + repo: /var/lib/opensearch-snapshots + logs: /var/log/opensearch jvm_options: Xmx: 1g # see https://www.elastic.co/guide/en/elasticsearch/reference/7.9/heap-size.html - opendistro_security: + opensearch_security: ssl: transport: enforce_hostname_verification: true diff --git a/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml b/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml index 2b704ef494..99d3845260 100644 --- a/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml +++ b/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml @@ -1,5 +1,5 @@ -kind: configuration/opensearch-for-elasticsearch -title: Open Distro for Elasticsearch Config +kind: configuration/opensearch-for-opensearch +title: OpenSearch Config name: default specification: cluster_name: EpiphanyElastic @@ -16,12 +16,12 @@ specification: - logstash - kibanaserver paths: - data: /var/lib/elasticsearch - repo: /var/lib/elasticsearch-snapshots - logs: /var/log/elasticsearch + data: /var/lib/opensearch + repo: /var/lib/opensearch-snapshots + logs: /var/log/opensearch jvm_options: Xmx: 1g # see https://www.elastic.co/guide/en/elasticsearch/reference/7.9/heap-size.html - opendistro_security: + opensearch_security: ssl: transport: enforce_hostname_verification: true diff --git a/schema/common/validation/configuration/logging.yml b/schema/common/validation/configuration/logging.yml index 2a434160a0..12dcf7bea8 100644 --- a/schema/common/validation/configuration/logging.yml +++ b/schema/common/validation/configuration/logging.yml @@ -32,7 +32,7 @@ properties: properties: Xmx: type: string - opendistro_security: + opensearch_security: type: object properties: ssl: diff --git a/schema/common/validation/configuration/opendistro-for-elasticsearch.yml b/schema/common/validation/configuration/opendistro-for-elasticsearch.yml index f240987b25..cdefba3076 100644 --- a/schema/common/validation/configuration/opendistro-for-elasticsearch.yml +++ b/schema/common/validation/configuration/opendistro-for-elasticsearch.yml @@ -35,7 +35,7 @@ properties: properties: Xmx: type: string - opendistro_security: + opensearch_security: type: object properties: ssl: diff --git a/tools/ip_change/elasticsearch/2_config_files.sh b/tools/ip_change/elasticsearch/2_config_files.sh index 6f09052de5..30b6d229ef 100644 --- a/tools/ip_change/elasticsearch/2_config_files.sh +++ b/tools/ip_change/elasticsearch/2_config_files.sh @@ -42,5 +42,5 @@ systemctl restart kibana echo "==== Kibana restarted ====" echo "==== Restarting Elasticsearch ====" -systemctl restart elasticsearch +systemctl restart opensearch echo "==== Elasticsearch restarted ====" From 9191e31dfbfee26359d172f14beb2272082d2334 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 21 Jan 2022 16:35:42 +0100 Subject: [PATCH 003/157] Migratioin to opensearch --- .../{opendistro-for-elasticsearch.yml => opensearch.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename schema/common/defaults/configuration/{opendistro-for-elasticsearch.yml => opensearch.yml} (100%) diff --git a/schema/common/defaults/configuration/opendistro-for-elasticsearch.yml b/schema/common/defaults/configuration/opensearch.yml similarity index 100% rename from schema/common/defaults/configuration/opendistro-for-elasticsearch.yml rename to schema/common/defaults/configuration/opensearch.yml From c95823b95ab5250c15236c93de059193da1d0cf2 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Sun, 23 Jan 2022 17:49:00 +0100 Subject: [PATCH 004/157] Migratioin to opensearch --- ...o_for_elasticsearch.yml => opensearch.yml} | 0 .../roles/opensearch/defaults/main.yml | 17 +- .../roles/opensearch/tasks/configure-ops.yml | 275 +++++++++--------- .../roles/opensearch/tasks/install-ops.yml | 47 ++- .../playbooks/roles/opensearch/tasks/main.yml | 4 +- .../templates/opensearch.service.j2 | 51 ++++ .../centos-7/requirements.aarch64.txt | 9 - .../centos-7/requirements.x86_64.txt | 11 +- .../redhat-7/requirements.x86_64.txt | 11 +- .../ubuntu-20.04/requirements.x86_64.txt | 11 +- 10 files changed, 224 insertions(+), 212 deletions(-) rename ansible/playbooks/{opendistro_for_elasticsearch.yml => opensearch.yml} (100%) create mode 100644 ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 diff --git a/ansible/playbooks/opendistro_for_elasticsearch.yml b/ansible/playbooks/opensearch.yml similarity index 100% rename from ansible/playbooks/opendistro_for_elasticsearch.yml rename to ansible/playbooks/opensearch.yml diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 33444a8239..c90a434f01 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -3,14 +3,14 @@ versions: RedHat: - ops_version: "1.2.3" + ops_version: "1.2.4" Debian: - ops_version: "1.2.3" + ops_version: "1.2.4" certificates: dirs: certs: /etc/opensearch - ca_key: /etc/elasticsearch/private - csr: /etc/elasticsearch/csr + ca_key: /etc/opensearch/private + csr: /etc/opensearch/csr dn_attributes_order: ['CN', 'OU', 'O', 'L', 'S', 'C', 'DC'] files: demo: @@ -59,4 +59,11 @@ ports: http: 9200 # defaults to range but we want static port transport: 9300 # defaults to range but we want static port log4j_file_name: apache-log4j-2.17.1-bin.tar.gz -ops_user: opensearch \ No newline at end of file +# ToDo: move these variables out of here: +ops_user: opensearch +ops_home: /usr/share/opensearch +ops_conf_dir: /usr/share/opensearch/config +ops_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin +ops_sec_plugin_conf_path: /usr/share/opensearch/plugins/opensearch-security/securityconfig +ops_sec_plugin_tools_path: /usr/share/opensearch/plugins/opensearch-security/tools +ops_api_port: 9200 \ No newline at end of file diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index 16317901aa..c1d7c78e67 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -5,15 +5,15 @@ file: path: "{{ specification.paths.repo }}/" state: directory - owner: opensearch - group: opensearch + owner: "{{ ops_user }}" + group: "{{ ops_user }}" mode: u=rwx,go= - name: Provide JVM configuration file template: backup: yes src: jvm.options.j2 - dest: /etc/elasticsearch/jvm.options + dest: /etc/opensearch/jvm.options owner: root group: opensearch mode: ug=rw,o= @@ -31,11 +31,11 @@ - include_tasks: generate-certs.yml -- name: Provide Elasticsearch configuration file +- name: Provide opensearch configuration file template: backup: yes src: opensearch.yml.j2 - dest: /etc/elasticsearch/elasticsearch.yml + dest: /etc/opensearch/opensearch.yml owner: root group: opensearch mode: ug=rw,o= @@ -98,7 +98,7 @@ transport_port: "{{ is_upgrade_run | ternary(existing_es_config['transport.port'], ports.transport) }}" # When 'opensearch_security.allow_unsafe_democertificates' is set to 'false' all demo certificate files must be removed, -# otherwise elasticsearch service doesn't start. +# otherwise opensearch service doesn't start. # For apply mode, demo certificate files are removed based only on their names. For upgrade mode, # public key fingerprints are checked to protect against unintentional deletion (what takes additional time). @@ -107,157 +107,154 @@ file: "{{ is_upgrade_run | ternary('remove-known-demo-certs.yml', 'remove-demo-certs.yml') }}" when: not certificates.files.demo.opensearch_security.allow_unsafe_democertificates -- name: Include log4j patch - include_tasks: patch-log4j.yml +# - name: Include log4j patch +# include_tasks: patch-log4j.yml -- name: Restart elasticsearch service +- name: Restart opensearch service systemd: name: opensearch state: restarted register: restart_opensearch when: change_config.changed - or log4j_patch.changed or change_jvm_config.changed - or install_elasticsearch_package.changed - or (install_opendistro_packages is defined and install_opendistro_packages.changed) -- name: Enable and start elasticsearch service +- name: Enable and start opensearch service systemd: name: opensearch state: started enabled: yes -- name: Change default users - when: not is_upgrade_run - block: - - name: Wait for elasticsearch service to start up - when: restart_elasticsearch.changed - wait_for: - port: 9200 - host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" +# - name: Change default users +# when: not is_upgrade_run +# block: +# - name: Wait for opensearch service to start up +# when: restart_opensearch.changed +# wait_for: +# port: 9200 +# host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" - - name: Set helper facts - set_fact: - elasticsearch_endpoint: https://{{ ansible_default_ipv4.address }}:9200 - vars: - uri_template: &uri - client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert.filename }}" - client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key.filename }}" - validate_certs: false - body_format: json +# - name: Set helper facts +# set_fact: +# opensearch_endpoint: https://{{ ansible_default_ipv4.address }}:9200 +# vars: +# uri_template: &uri +# client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert.filename }}" +# client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key.filename }}" +# validate_certs: false +# body_format: json - - name: Check if default admin user exists - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/admin" - method: GET - # 404 code is used there as someone can remove admin user on its own. - status_code: [200, 404] - register: admin_check_response - until: admin_check_response is success - retries: 60 - delay: 1 - run_once: true +# - name: Check if default admin user exists +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/admin" +# method: GET +# # 404 code is used there as someone can remove admin user on its own. +# status_code: [200, 404] +# register: admin_check_response +# until: admin_check_response is success +# retries: 60 +# delay: 1 +# run_once: true - - name: Set OpenDistro admin password - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/" - method: PATCH - status_code: [200] - body: - - op: "replace" - path: "/admin" - value: - password: "{{ specification.admin_password }}" - reserved: "true" - backend_roles: - - "admin" - description: "Admin user" - register: uri_response - until: uri_response is success - retries: 15 - delay: 1 - run_once: true - when: admin_check_response.status == 200 +# - name: Set OpenDistro admin password +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" +# method: PATCH +# status_code: [200] +# body: +# - op: "replace" +# path: "/admin" +# value: +# password: "{{ specification.admin_password }}" +# reserved: "true" +# backend_roles: +# - "admin" +# description: "Admin user" +# register: uri_response +# until: uri_response is success +# retries: 15 +# delay: 1 +# run_once: true +# when: admin_check_response.status == 200 - - name: Check if default kibanaserver user exists - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/kibanaserver" - method: GET - status_code: [200] - register: kibanaserver_check_response - until: kibanaserver_check_response is success - retries: 60 - delay: 1 - run_once: true - when: specification.kibanaserver_user_active +# - name: Check if default kibanaserver user exists +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/kibanaserver" +# method: GET +# status_code: [200] +# register: kibanaserver_check_response +# until: kibanaserver_check_response is success +# retries: 60 +# delay: 1 +# run_once: true +# when: specification.kibanaserver_user_active - - name: Set OpenDistro kibanaserver password - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/" - method: PATCH - status_code: [200] - body: - - op: "replace" - path: "/kibanaserver" - value: - password: "{{ specification.kibanaserver_password }}" - reserved: "true" - description: "Kibana server user" - register: uri_response - until: uri_response is success - retries: 15 - delay: 1 - run_once: true - when: specification.kibanaserver_user_active +# - name: Set OpenDistro kibanaserver password +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" +# method: PATCH +# status_code: [200] +# body: +# - op: "replace" +# path: "/kibanaserver" +# value: +# password: "{{ specification.kibanaserver_password }}" +# reserved: "true" +# description: "Kibana server user" +# register: uri_response +# until: uri_response is success +# retries: 15 +# delay: 1 +# run_once: true +# when: specification.kibanaserver_user_active - - name: Check if default logstash user exists - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/logstash" - method: GET - status_code: [200] - register: logstash_check_response - until: logstash_check_response is success - retries: 60 - delay: 1 - run_once: true - when: specification.logstash_user_active +# - name: Check if default logstash user exists +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash" +# method: GET +# status_code: [200] +# register: logstash_check_response +# until: logstash_check_response is success +# retries: 60 +# delay: 1 +# run_once: true +# when: specification.logstash_user_active - - name: Set OpenDistro logstash password - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/" - method: PATCH - status_code: [200] - body: - - op: "replace" - path: "/logstash" - value: - password: "{{ specification.logstash_password }}" - reserved: "true" - backend_roles: - - "logstash" - description: "Logstash user" - register: uri_response - until: uri_response is success - retries: 3 - delay: 5 - run_once: true - when: specification.logstash_user_active +# - name: Set OpenDistro logstash password +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" +# method: PATCH +# status_code: [200] +# body: +# - op: "replace" +# path: "/logstash" +# value: +# password: "{{ specification.logstash_password }}" +# reserved: "true" +# backend_roles: +# - "logstash" +# description: "Logstash user" +# register: uri_response +# until: uri_response is success +# retries: 3 +# delay: 5 +# run_once: true +# when: specification.logstash_user_active - - name: Remove OpenDistro demo users - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_opendistro/_security/api/internalusers/{{ item }}" - method: DELETE - status_code: [200, 404] - register: uri_response - until: uri_response is success - retries: 15 - delay: 1 - run_once: true - loop: "{{ specification.demo_users_to_remove }}" +# - name: Remove OpenDistro demo users +# uri: +# <<: *uri +# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/{{ item }}" +# method: DELETE +# status_code: [200, 404] +# register: uri_response +# until: uri_response is success +# retries: 15 +# delay: 1 +# run_once: true +# loop: "{{ specification.demo_users_to_remove }}" diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index f27786714a..878c8569e3 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -1,48 +1,35 @@ --- -- name: OpenSearch Install | Download opensearch {{ os_version }} - get_url: - url: "{{ os_download_url }}/{{ os_version }}/opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" - dest: "/tmp/opensearch.tar.gz" - register: download +- name: Download Opensearch + include_role: + name: download + tasks_from: download_file + vars: + file_name: "opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" -- name: OpenSearch Install | Create opensearch user +- name: OpenSearch Install | Ensure Opensearch service user exists user: name: "{{ ops_user }}" state: present shell: /bin/bash - when: download.changed -- name: OpenSearch Install | Create home directory +- name: OpenSearch Install | Ensure home directory exists file: - path: "{{ os_home }}" + path: "{{ ops_home }}" state: directory owner: "{{ ops_user }}" group: "{{ ops_user }}" - when: download.changed -- name: OpenSearch Install | Extract the tar file - command: chdir=/tmp/ tar -xvzf opensearch.tar.gz -C "{{ os_home }}" --strip-components=1 - when: download.changed - -- name: OpenSearch Install | Copy Configuration File - template: - src: opensearch.yml - dest: "{{os_conf_dir}}/opensearch.yml" +- name: OpenSearch Install | Ensure config directory exists + file: + path: "/etc/opensearch" + state: directory owner: "{{ ops_user }}" group: "{{ ops_user }}" - mode: 0644 - backup: yes -- name: OpenSearch Install | Copy jvm.options File for Instance - template: - src: jvm.options - dest: "{{os_conf_dir}}/jvm.options" - owner: "{{ ops_user }}" - group: "{{ ops_user }}" - mode: 0644 - force: yes +- name: OpenSearch Install | Extract the tar file + command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ ops_home }} --strip-components=1" - name: OpenSearch Install | create systemd service template: - src: opensearch.service - dest: "{{ systemctl_path }}/opensearch.service" + src: opensearch.service.j2 + dest: "/etc/systemd/system/opensearch.service" diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index 34de2b377f..75a1c033e9 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -17,5 +17,5 @@ # - include_tasks: install-ops-plugins.yml -# - name: Include configuration tasks -# include_tasks: configure-ops.yml +- name: Include configuration tasks + include_tasks: configure-ops.yml diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 new file mode 100644 index 0000000000..13f4ff9bea --- /dev/null +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 @@ -0,0 +1,51 @@ +[Unit] +Description=opensearch +Wants=network-online.target +After=network-online.target + +[Service] +RuntimeDirectory=opensearch +PrivateTmp=true + +WorkingDirectory={{ ops_home }} + +User={{ ops_user }} +Group={{ ops_user }} + +ExecStart={{ ops_home }}/bin/opensearch -p {{ ops_home }}/opensearch.pid -q + +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65536 + +# Specifies the memory lock settings +LimitMEMLOCK=infinity + +# Specifies the maximum number of processes +LimitNPROC=4096 + +# Specifies the maximum size of virtual memory +LimitAS=infinity + +# Specifies the maximum file size +LimitFSIZE=infinity + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop the Java process +KillSignal=SIGTERM + +# Send the signal only to the JVM rather than its control group +KillMode=process + +# Java process is never killed +SendSIGKILL=no + +# When a JVM receives a SIGTERM signal it exits with code 143 +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 8d7a7d8e1f..f31b6c4bf3 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -35,7 +35,6 @@ docker-ce-20.10.8 docker-ce-cli-20.10.8 docker-ce-rootless-extras-20.10.8 ebtables -elasticsearch-oss-7.10.2 # for opendistroforelasticsearch & logging roles ethtool filebeat-7.9.2 firewalld @@ -61,14 +60,6 @@ logrotate net-tools nfs-utils nmap-ncat -# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opensearch-alerting-1.13.1.* -opensearch-index-management-1.13.1.* -opensearch-job-scheduler-1.13.0.* -opensearch-performance-analyzer-1.13.0.* -opensearch-security-1.13.1.* -opensearch-sql-1.13.0.* -opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl unixODBC # for erlang perl # for vim diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 0276ca3eab..653a8f52ce 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -36,7 +36,6 @@ docker-ce-cli-20.10.8 docker-ce-rootless-extras-20.10.8 ebtables elasticsearch-curator-5.8.3 -elasticsearch-oss-7.10.2 # for opendistroforelasticsearch & logging roles ethtool filebeat-7.9.2 firewalld @@ -62,14 +61,6 @@ logrotate net-tools nfs-utils nmap-ncat -# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opensearch-alerting-1.13.1.* -opensearch-index-management-1.13.1.* -opensearch-job-scheduler-1.13.0.* -opensearch-performance-analyzer-1.13.0.* -opensearch-security-1.13.1.* -opensearch-sql-1.13.0.* -opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl perl # for vim perl-Getopt-Long # for vim @@ -195,6 +186,8 @@ https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_17 https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6663.json # RabbitMQ cluster monitoring (via Prometheus) https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json +# OpenSearch +https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index f29e2f0c80..268e268975 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -34,7 +34,6 @@ docker-ce-cli-20.10.8 docker-ce-rootless-extras-20.10.8 ebtables elasticsearch-curator-5.8.3 -elasticsearch-oss-7.10.2 # for opendistroforelasticsearch & logging roles ethtool filebeat-7.9.2 firewalld @@ -60,14 +59,6 @@ logrotate net-tools nfs-utils nmap-ncat -# Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package) -opensearch-alerting-1.13.1.* -opensearch-index-management-1.13.1.* -opensearch-job-scheduler-1.13.0.* -opensearch-performance-analyzer-1.13.0.* -opensearch-security-1.13.1.* -opensearch-sql-1.13.0.* -opendistroforelasticsearch-kibana-1.13.1 # kibana has shorter version openssl perl # for vim perl-Getopt-Long # for vim @@ -193,6 +184,8 @@ https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_17 https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6663.json # RabbitMQ cluster monitoring (via Prometheus) https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json +# OpenSearch +https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index fdc4a081ac..d610e35690 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -15,8 +15,6 @@ docker-ce 5:20.10.8 docker-ce-cli 5:20.10.8 docker-ce-rootless-extras 5:20.10.8 ebtables -# for opendistroforelasticsearch & logging roles -elasticsearch-oss 7.10.2 # Erlang packages must be compatible with RabbitMQ version. # Metapackages such as erlang and erlang-nox must only be used @@ -59,13 +57,6 @@ net-tools nfs-common # for nfs-common libtirpc3 -opensearch-alerting 1.13.1 -opensearch-index-management 1.13.1 -opensearch-job-scheduler 1.13.0 -opensearch-performance-analyzer 1.13.0 -opensearch-security 1.13.1 -opensearch-sql 1.13.0 -opendistroforelasticsearch-kibana 1.13.1 openjdk-8-jre-headless openssl postgresql-13 @@ -230,6 +221,8 @@ https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_17 https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6663.json # RabbitMQ cluster monitoring (via Prometheus) https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json +# OpenSearch +https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz [images] haproxy:2.2.2-alpine From ebc0e5ca2274b8a3134896990f13bb07485ce8a9 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 26 Jan 2022 16:40:59 +0100 Subject: [PATCH 005/157] Migration ODFE -> OpS --- ansible/playbooks/opensearch_dashboards.yml | 10 + .../roles/opensearch/defaults/main.yml | 25 +- .../roles/opensearch/tasks/configure-ops.yml | 258 +++++++++--------- .../roles/opensearch/tasks/install-ops.yml | 24 +- .../playbooks/roles/opensearch/tasks/main.yml | 3 + .../playbooks/roles/opensearch/tasks/tune.yml | 12 + .../roles/opensearch/templates/jvm.options.j2 | 8 +- .../opensearch/templates/opensearch.yml.j2 | 56 ++-- .../opensearch_dashboards/defaults/main.yml | 28 ++ .../opensearch_dashboards/handlers/main.yml | 3 + .../tasks/dashboards.yml | 37 +++ .../opensearch_dashboards/tasks/etchosts.yml | 13 + .../opensearch_dashboards/tasks/main.yml | 30 ++ .../templates/dashboards.service | 48 ++++ .../templates/opensearch_dashboards.yml.j2 | 13 + .../roles/opensearch_dashboards/vars/main.yml | 3 + .../centos-7/requirements.x86_64.txt | 3 + .../redhat-7/requirements.x86_64.txt | 2 + .../ubuntu-20.04/requirements.x86_64.txt | 2 + docs/home/howto/DATABASES.md | 2 +- docs/home/howto/LOGGING.md | 2 +- .../common/defaults/configuration/logging.yml | 15 +- .../configuration/opensearch-dashboards.yml | 11 + .../defaults/configuration/opensearch.yml | 15 +- .../configuration/opensearch-dashboards.yml | 5 + 25 files changed, 425 insertions(+), 203 deletions(-) create mode 100644 ansible/playbooks/opensearch_dashboards.yml create mode 100644 ansible/playbooks/roles/opensearch/tasks/tune.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service create mode 100644 ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 create mode 100644 ansible/playbooks/roles/opensearch_dashboards/vars/main.yml create mode 100644 schema/common/defaults/configuration/opensearch-dashboards.yml create mode 100644 schema/common/validation/configuration/opensearch-dashboards.yml diff --git a/ansible/playbooks/opensearch_dashboards.yml b/ansible/playbooks/opensearch_dashboards.yml new file mode 100644 index 0000000000..66c8bd5ed8 --- /dev/null +++ b/ansible/playbooks/opensearch_dashboards.yml @@ -0,0 +1,10 @@ +--- +# Ansible playbook for installing OpenSearch Dashboards +- hosts: repository # to gather facts + tasks: [] + +- hosts: opensearch_dashboards + become: true + become_method: sudo + roles: + - opensearch_dashboards diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index c90a434f01..1e95990215 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -7,10 +7,10 @@ versions: Debian: ops_version: "1.2.4" certificates: - dirs: - certs: /etc/opensearch - ca_key: /etc/opensearch/private - csr: /etc/opensearch/csr + dirs: # must be under the config directory, specified using a relative path + certs: /usr/share/opensearch/config/cert + ca_key: /usr/share/opensearch/config/private + csr: /usr/share/opensearch/config/csr dn_attributes_order: ['CN', 'OU', 'O', 'L', 'S', 'C', 'DC'] files: demo: @@ -59,11 +59,12 @@ ports: http: 9200 # defaults to range but we want static port transport: 9300 # defaults to range but we want static port log4j_file_name: apache-log4j-2.17.1-bin.tar.gz -# ToDo: move these variables out of here: -ops_user: opensearch -ops_home: /usr/share/opensearch -ops_conf_dir: /usr/share/opensearch/config -ops_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin -ops_sec_plugin_conf_path: /usr/share/opensearch/plugins/opensearch-security/securityconfig -ops_sec_plugin_tools_path: /usr/share/opensearch/plugins/opensearch-security/tools -ops_api_port: 9200 \ No newline at end of file + +admin_password: "{{ specification.admin_password }}" +ops_user: "{{ specification.ops_user }}" +ops_home: "{{ specification.paths.ops_home }}" +ops_conf_dir: "{{ specification.paths.ops_conf_dir }}" +ops_log_dir: "{{ specification.paths.ops_log_dir }}" +ops_plugin_bin_path: "{{ specification.paths.ops_plugin_bin_path }}" +ops_api_port: "{{ ports.http }}" +systemctl_path: /etc/systemd/system \ No newline at end of file diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index c1d7c78e67..327618a300 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -3,7 +3,7 @@ - name: Ensure snapshot folder exists file: - path: "{{ specification.paths.repo }}/" + path: "{{ specification.paths.ops_repo }}/" state: directory owner: "{{ ops_user }}" group: "{{ ops_user }}" @@ -13,7 +13,7 @@ template: backup: yes src: jvm.options.j2 - dest: /etc/opensearch/jvm.options + dest: "{{ ops_conf_dir }}/jvm.options" owner: root group: opensearch mode: ug=rw,o= @@ -35,7 +35,7 @@ template: backup: yes src: opensearch.yml.j2 - dest: /etc/opensearch/opensearch.yml + dest: "{{ ops_conf_dir }}/opensearch.yml" owner: root group: opensearch mode: ug=rw,o= @@ -124,137 +124,137 @@ state: started enabled: yes -# - name: Change default users -# when: not is_upgrade_run -# block: -# - name: Wait for opensearch service to start up -# when: restart_opensearch.changed -# wait_for: -# port: 9200 -# host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" +- name: Change default users + when: not is_upgrade_run + block: + - name: Wait for opensearch service to start up + when: restart_opensearch.changed + wait_for: + port: 9200 + host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" -# - name: Set helper facts -# set_fact: -# opensearch_endpoint: https://{{ ansible_default_ipv4.address }}:9200 -# vars: -# uri_template: &uri -# client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert.filename }}" -# client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key.filename }}" -# validate_certs: false -# body_format: json + - name: Set helper facts + set_fact: + opensearch_endpoint: https://{{ ansible_default_ipv4.address }}:9200 + vars: + uri_template: &uri + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert.filename }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key.filename }}" + validate_certs: false + body_format: json -# - name: Check if default admin user exists -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/admin" -# method: GET -# # 404 code is used there as someone can remove admin user on its own. -# status_code: [200, 404] -# register: admin_check_response -# until: admin_check_response is success -# retries: 60 -# delay: 1 -# run_once: true + - name: Check if default admin user exists + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/admin" + method: GET + # 404 code is used there as someone can remove admin user on its own. + status_code: [200, 404] + register: admin_check_response + until: admin_check_response is success + retries: 60 + delay: 1 + run_once: true -# - name: Set OpenDistro admin password -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" -# method: PATCH -# status_code: [200] -# body: -# - op: "replace" -# path: "/admin" -# value: -# password: "{{ specification.admin_password }}" -# reserved: "true" -# backend_roles: -# - "admin" -# description: "Admin user" -# register: uri_response -# until: uri_response is success -# retries: 15 -# delay: 1 -# run_once: true -# when: admin_check_response.status == 200 + - name: Set OpenSearch admin password + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" + method: PATCH + status_code: [200] + body: + - op: "replace" + path: "/admin" + value: + password: "{{ specification.admin_password }}" + reserved: "true" + backend_roles: + - "admin" + description: "Admin user" + register: uri_response + until: uri_response is success + retries: 15 + delay: 1 + run_once: true + when: admin_check_response.status == 200 -# - name: Check if default kibanaserver user exists -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/kibanaserver" -# method: GET -# status_code: [200] -# register: kibanaserver_check_response -# until: kibanaserver_check_response is success -# retries: 60 -# delay: 1 -# run_once: true -# when: specification.kibanaserver_user_active + - name: Check if default kibanaserver user exists + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/kibanaserver" + method: GET + status_code: [200] + register: kibanaserver_check_response + until: kibanaserver_check_response is success + retries: 60 + delay: 1 + run_once: true + when: specification.kibanaserver_user_active -# - name: Set OpenDistro kibanaserver password -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" -# method: PATCH -# status_code: [200] -# body: -# - op: "replace" -# path: "/kibanaserver" -# value: -# password: "{{ specification.kibanaserver_password }}" -# reserved: "true" -# description: "Kibana server user" -# register: uri_response -# until: uri_response is success -# retries: 15 -# delay: 1 -# run_once: true -# when: specification.kibanaserver_user_active + - name: Set OpenSearch kibanaserver password + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" + method: PATCH + status_code: [200] + body: + - op: "replace" + path: "/kibanaserver" + value: + password: "{{ specification.kibanaserver_password }}" + reserved: "true" + description: "Kibana server user" + register: uri_response + until: uri_response is success + retries: 15 + delay: 1 + run_once: true + when: specification.kibanaserver_user_active -# - name: Check if default logstash user exists -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash" -# method: GET -# status_code: [200] -# register: logstash_check_response -# until: logstash_check_response is success -# retries: 60 -# delay: 1 -# run_once: true -# when: specification.logstash_user_active + - name: Check if default logstash user exists + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/logstash" + method: GET + status_code: [200] + register: logstash_check_response + until: logstash_check_response is success + retries: 60 + delay: 1 + run_once: true + when: specification.logstash_user_active -# - name: Set OpenDistro logstash password -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" -# method: PATCH -# status_code: [200] -# body: -# - op: "replace" -# path: "/logstash" -# value: -# password: "{{ specification.logstash_password }}" -# reserved: "true" -# backend_roles: -# - "logstash" -# description: "Logstash user" -# register: uri_response -# until: uri_response is success -# retries: 3 -# delay: 5 -# run_once: true -# when: specification.logstash_user_active + - name: Set OpenSearch logstash password + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/" + method: PATCH + status_code: [200] + body: + - op: "replace" + path: "/logstash" + value: + password: "{{ specification.logstash_password }}" + reserved: "true" + backend_roles: + - "logstash" + description: "Logstash user" + register: uri_response + until: uri_response is success + retries: 3 + delay: 5 + run_once: true + when: specification.logstash_user_active -# - name: Remove OpenDistro demo users -# uri: -# <<: *uri -# url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/{{ item }}" -# method: DELETE -# status_code: [200, 404] -# register: uri_response -# until: uri_response is success -# retries: 15 -# delay: 1 -# run_once: true -# loop: "{{ specification.demo_users_to_remove }}" + - name: Remove OpenSearch demo users + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_opendistro/_security/api/internalusers/{{ item }}" + method: DELETE + status_code: [200, 404] + register: uri_response + until: uri_response is success + retries: 15 + delay: 1 + run_once: true + loop: "{{ specification.demo_users_to_remove }}" diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 878c8569e3..e1f18c9f42 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -6,30 +6,28 @@ vars: file_name: "opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" -- name: OpenSearch Install | Ensure Opensearch service user exists +- name: Ensure Opensearch service user exists user: name: "{{ ops_user }}" state: present shell: /bin/bash -- name: OpenSearch Install | Ensure home directory exists +- name: Ensure directory structure exists file: - path: "{{ ops_home }}" + path: "{{ item }}" state: directory owner: "{{ ops_user }}" group: "{{ ops_user }}" + with_items: + - "{{ ops_home }}" + - "{{ ops_log_dir }}" + - "{{ ops_conf_dir }}" + - "{{ certificates.dirs.certs }}" -- name: OpenSearch Install | Ensure config directory exists - file: - path: "/etc/opensearch" - state: directory - owner: "{{ ops_user }}" - group: "{{ ops_user }}" - -- name: OpenSearch Install | Extract the tar file +- name: Extract the tar file command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ ops_home }} --strip-components=1" -- name: OpenSearch Install | create systemd service +- name: Create systemd service template: src: opensearch.service.j2 - dest: "/etc/systemd/system/opensearch.service" + dest: "{{ systemctl_path }}/opensearch.service" diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index 75a1c033e9..8d73a431da 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -13,6 +13,9 @@ - { limit_type: 'soft', limit_item: 'memlock', value: unlimited } - { limit_type: 'hard', limit_item: 'memlock', value: unlimited } +- name: Tune the system settings + import_tasks: tune.yml + - include_tasks: install-ops.yml # - include_tasks: install-ops-plugins.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/tune.yml b/ansible/playbooks/roles/opensearch/tasks/tune.yml new file mode 100644 index 0000000000..113fdd1797 --- /dev/null +++ b/ansible/playbooks/roles/opensearch/tasks/tune.yml @@ -0,0 +1,12 @@ +--- +- name: Set open files limit in sysctl.conf + sysctl: + name: fs.file-max + value: "65536" + state: present + +- name: Set maximum number of memory map areas limit in sysctl.conf + sysctl: + name: vm.max_map_count + value: "262144" + state: present diff --git a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 index 1fcfe8da4c..def6b9e830 100644 --- a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 +++ b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 @@ -51,7 +51,7 @@ 14-:-XX:InitiatingHeapOccupancyPercent=30 ## JVM temporary directory --Djava.io.tmpdir=${ES_TMPDIR} +-Djava.io.tmpdir=${OPENSEARCH_TMPDIR} ## heap dumps @@ -64,20 +64,20 @@ -XX:HeapDumpPath=/var/lib/opensearch # specify an alternative path for JVM fatal error logs --XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log +-XX:ErrorFile=/var/log/opensearch/hs_err_pid%p.log ## JDK 8 GC logging 8:-XX:+PrintGCDetails 8:-XX:+PrintGCDateStamps 8:-XX:+PrintTenuringDistribution 8:-XX:+PrintGCApplicationStoppedTime -8:-Xloggc:/var/log/elasticsearch/gc.log +8:-Xloggc:/var/log/opensearch/gc.log 8:-XX:+UseGCLogFileRotation 8:-XX:NumberOfGCLogFiles=32 8:-XX:GCLogFileSize=64m # JDK 9+ GC logging -9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m +9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/opensearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m ## OpenDistro Performance Analyzer -Dclk.tck=100 diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 index cb09da9dcc..673d61092e 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 @@ -1,16 +1,6 @@ #jinja2: lstrip_blocks: True # {{ ansible_managed }} -# ======================== Elasticsearch Configuration ========================= -# -# NOTE: Elasticsearch comes with reasonable defaults for most settings. -# Before you set out to tweak and tune the configuration, make sure you -# understand what are you trying to accomplish and the consequences. -# -# The primary way of configuring a node is via this file. This template lists -# the most important settings you may want to configure for a production cluster. -# -# Please consult the documentation for further information on configuration options: -# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# ======================== OpenSearch Configuration ========================= # # ---------------------------------- Cluster ----------------------------------- # @@ -32,15 +22,15 @@ node.name: {{ ansible_hostname }} # # Path to directory where to store the data (separate multiple locations by comma): # -path.data: {{ specification.paths.data }} +# path.data: {{ specification.paths.ops_data }} # # Path to directory where the shared storage should be mounted: # -path.repo: {{ specification.paths.repo }} +# path.repo: {{ specification.paths.ops_repo }} # # Path to log files: # -path.logs: {{ specification.paths.logs }} +# path.logs: {{ specification.paths.ops_logs }} # # ----------------------------------- Memory ----------------------------------- # @@ -52,7 +42,7 @@ path.logs: {{ specification.paths.logs }} # on the system and that the owner of the process is allowed to use this # limit. # -# Elasticsearch performs poorly when the system is swapping the memory. +# OpenSearch performs poorly when the system is swapping the memory. # # ---------------------------------- Network ----------------------------------- # @@ -109,33 +99,33 @@ cluster.initial_master_nodes: ["{{ ansible_hostname }}"] # #action.destructive_requires_name: true -######## Start OpenDistro for Elasticsearch Security Configuration ######## +######## OpenSearch Security Configuration ######## # WARNING: revise all the lines below before you go into production -opensearch_security.ssl.transport.pemcert_filepath: {{ node_cert_filename.transport }} -opensearch_security.ssl.transport.pemkey_filepath: {{ node_key_filename.transport }} -opensearch_security.ssl.transport.pemtrustedcas_filepath: {{ root_ca_cert_filename.transport }} -opensearch_security.ssl.transport.enforce_hostname_verification: {{ specification.opensearch_security.ssl.transport.enforce_hostname_verification | lower }} -opensearch_security.ssl.http.enabled: true -opensearch_security.ssl.http.pemcert_filepath: {{ node_cert_filename.http }} -opensearch_security.ssl.http.pemkey_filepath: {{ node_key_filename.http }} -opensearch_security.ssl.http.pemtrustedcas_filepath: {{ root_ca_cert_filename.http }} -opensearch_security.allow_unsafe_democertificates: {{ opensearch_security_allow_unsafe_democertificates | lower }} -opensearch_security.allow_default_init_securityindex: true -opensearch_security.authcz.admin_dn: +plugins.security.ssl.transport.pemcert_filepath: "{{ certificates.dirs.certs }}/{{ node_cert_filename.transport }}" +plugins.security.ssl.transport.pemkey_filepath: "{{ certificates.dirs.certs }}/{{ node_key_filename.transport }}" +plugins.security.ssl.transport.pemtrustedcas_filepath: "{{ certificates.dirs.certs }}/{{ root_ca_cert_filename.transport }}" +plugins.security.ssl.transport.enforce_hostname_verification: {{ specification.opensearch_security.ssl.transport.enforce_hostname_verification | lower }} +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemcert_filepath: "{{ certificates.dirs.certs }}/{{ node_cert_filename.http }}" +plugins.security.ssl.http.pemkey_filepath: "{{ certificates.dirs.certs }}/{{ node_key_filename.http }}" +plugins.security.ssl.http.pemtrustedcas_filepath: "{{ certificates.dirs.certs }}/{{ root_ca_cert_filename.http }}" +plugins.security.allow_unsafe_democertificates: {{ opensearch_security_allow_unsafe_democertificates | lower }} +plugins.security.allow_default_init_securityindex: true +plugins.security.authcz.admin_dn: {% for dn in admin_dn %} - '{{ dn }}' {% endfor %} {% if nodes_dn | count > 0 %} -opensearch_security.nodes_dn: +plugins.security.nodes_dn: {% for dn in nodes_dn %} - '{{ dn }}' {% endfor %} {% endif %} -opensearch_security.audit.type: internal_opensearch -opensearch_security.enable_snapshot_restore_privilege: true -opensearch_security.check_snapshot_restore_write_privileges: true -opensearch_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] cluster.routing.allocation.disk.threshold_enabled: false node.max_local_storage_nodes: 3 -######## End OpenDistro for Elasticsearch Security Configuration ######## +######## End OpenSearch Security Configuration ######## diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml new file mode 100644 index 0000000000..8793d21e14 --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -0,0 +1,28 @@ +--- +versions: + RedHat: + ops_dashboards_version: "1.2.0" + Debian: + ops_dashboards_version: "1.2.0" +ops_nodes: |- + {% for item in groups['ops-cluster'] -%} + {{ hostvars[item]['ip'] }}{% if not loop.last %}","{% endif %} + {%- endfor %} + +populate_inventory_to_hosts_file: true + +ops_dashboards_user: "{{ specification.ops_dashboards_user }}" +ops_dashboards_password: "{{ specification.ops_dashboards_password }}" +ops_user: "{{ specification.ops_user }}" +ops_download_url: https://artifacts.opensearch.org/releases/bundle/opensearch +ops_dashboards_home: "{{ specification.ops_dashboards_home }}" +ops_dashboards_conf_dir: "{{ specification.ops_dashboards_conf_dir }}" +ops_plugin_bin_path: "{{ specification.ops_plugin_bin_path }}" + +ops_api_port: 9200 +ops_nodes_dashboards: |- + {% for item in groups['opensearch_dashboards'] -%} + https://{{ hostvars[item]['ansible_host'] }}:{{ ops_api_port }}{% if not loop.last %}","{% endif %} + {%- endfor %} + +systemctl_path: /etc/systemd/system diff --git a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml new file mode 100644 index 0000000000..69d704bd1f --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart dashboards + systemd: name=dashboards state=restarted enabled=yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml new file mode 100644 index 0000000000..58d723542e --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -0,0 +1,37 @@ +--- +- name: Download Opensearch dashbaords {{ versions[ansible_os_family].ops_dashboards_version }} + include_role: + name: download + tasks_from: download_file + vars: + file_name: "opensearch-dashboards-{{ versions[ansible_os_family].ops_dashboards_version }}-linux-x64.tar.gz" + +- name: Create opensearch Dashboards user + user: + name: "{{ ops_dashboards_user }}" + state: present + shell: /bin/bash + +- name: Create home directory + file: + path: "{{ ops_dashboards_home }}" + state: directory + owner: "{{ ops_dashboards_user }}" + group: "{{ ops_dashboards_user }}" + +- name: Extract the tar file + command: "chdir=/tmp/ tar -xvzf opensearch-dashboards-{{ versions[ansible_os_family].ops_dashboards_version }}-linux-x64.tar.gz -C {{ ops_dashboards_home }} --strip-components=1" + +- name: Copy Configuration File + template: + src: opensearch_dashboards.yml.j2 + dest: "{{ ops_dashboards_conf_dir }}/opensearch_dashboards.yml" + owner: "{{ ops_dashboards_user }}" + group: "{{ ops_dashboards_user }}" + mode: 0644 + backup: yes + +- name: Create systemd service + template: + src: dashboards.service + dest: "{{ systemctl_path }}/dashboards.service" diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml new file mode 100644 index 0000000000..e7b614f05a --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml @@ -0,0 +1,13 @@ +--- +- name: Hosts | populate inventory into hosts file + blockinfile: + dest: /etc/hosts + block: |- + {% for item in groups['dashboards'] %} + {{ hostvars[item]['ip'] }} {{ item }}.{{ domain_name }} {{ item }} + {% endfor %} + state: present + create: yes + backup: yes + marker: "# Ansible inventory hosts {mark}" + when: populate_inventory_to_hosts_file diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml new file mode 100644 index 0000000000..0cb5e1ce22 --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -0,0 +1,30 @@ +--- + +- hostname: + name: "{{ inventory_hostname }}" + +# Disabling for Amazon Linux 2 as selinux is disabled by default. +- name: Disable the selinux + selinux: + state: disabled + when: (ansible_distribution != "Ubuntu") and (ansible_distribution != "Amazon") + +# - name: Populate the nodes to /etc/hosts +# import_tasks: etchosts.yml + +- name: include dashboards installation + include: dashboards.yml + +- name: Make sure opensearch dashboards is started + service: + name: dashboards + state: started + enabled: yes + +- name: Get all the installed dashboards plugins + command: "sudo -u {{ ops_dashboards_user }} {{ ops_plugin_bin_path }} list" + register: list_plugins + +- name: Show all the installed dashboards plugins + debug: + msg: "{{ list_plugins.stdout }}" diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service new file mode 100644 index 0000000000..a0b36e7f9a --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service @@ -0,0 +1,48 @@ +[Unit] +Description=opensearch-dashboards +Wants=network-online.target +After=network-online.target + +[Service] +RuntimeDirectory=opensearch-dashboards +PrivateTmp=true + +WorkingDirectory={{ ops_dashboards_home }} + +User={{ ops_user }} +Group={{ ops_user }} + +ExecStart={{ ops_dashboards_home }}/bin/opensearch-dashboards -q + +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65536 + +# Specifies the maximum number of processes +LimitNPROC=4096 + +# Specifies the maximum size of virtual memory +LimitAS=infinity + +# Specifies the maximum file size +LimitFSIZE=infinity + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop the Java process +KillSignal=SIGTERM + +# Send the signal only to the JVM rather than its control group +KillMode=process + +# Java process is never killed +SendSIGKILL=no + +# When a JVM receives a SIGTERM signal it exits with code 143 +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 new file mode 100644 index 0000000000..5b997e313e --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -0,0 +1,13 @@ +server.port: 5601 +server.host: "{{ hostvars[inventory_hostname]['ansible_host'] }}" +opensearch.hosts: ["{{ ops_nodes_dashboards }}"] +opensearch.ssl.verificationMode: none +opensearch.username: "{{ ops_dashboards_user }}" +opensearch.password: "{{ ops_dashboards_password }}" +opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] + +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] +opensearch_security.readonly_mode.roles: ["kibana_read_only"] +# Use this setting if you are running dashboards without https +opensearch_security.cookie.secure: false diff --git a/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml b/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml new file mode 100644 index 0000000000..3b120cca51 --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml @@ -0,0 +1,3 @@ +--- +# vars file for opensearch +java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 653a8f52ce..5a8d3db0fa 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -188,6 +188,9 @@ https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6 https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json # OpenSearch https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz +# OpenSearch Dashboards +https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz + [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index 268e268975..6f050d61df 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -186,6 +186,8 @@ https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6 https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json # OpenSearch https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz +# OpenSearch Dashboards +https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index d610e35690..64b5c89646 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -223,6 +223,8 @@ https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6 https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json # OpenSearch https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz +# OpenSearch Dashboards +https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz [images] haproxy:2.2.2-alpine diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 4e49708b8a..adc929ea8c 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -486,7 +486,7 @@ kind: configuration/opensearch-for-opensearch title: OpenDistro for Elasticsearch Config name: default specification: - cluster_name: EpiphanyElastic + cluster_name: EpiphanyOpensearch ``` By default, Kibana is deployed only for `logging` component. If you want to deploy Kibana diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 18db4b909d..fba7555a4d 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -45,7 +45,7 @@ kind: configuration/logging title: Logging Config name: default specification: - cluster_name: EpiphanyElastic + cluster_name: EpiphanyOpensearch clustered: True paths: data: /var/lib/opensearch diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index bcd1992463..8d4f227f35 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -2,7 +2,8 @@ kind: configuration/logging title: Logging Config name: default specification: - cluster_name: EpiphanyElastic + cluster_name: EpiphanyOpensearch + ops_user: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: true @@ -13,11 +14,15 @@ specification: - readall - snapshotrestore paths: - data: /var/lib/opensearch - repo: /var/lib/opensearch-snapshots - logs: /var/log/opensearch + ops_home: /usr/share/opensearch + ops_conf_dir: /usr/share/opensearch/config + ops_log_dir: /var/log/opensearch + ops_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin + ops_repo: /var/lib/opensearch-snapshots + ops_data: /var/lib/opensearch + ops_logs: /var/log/opensearch jvm_options: - Xmx: 1g # see https://www.elastic.co/guide/en/elasticsearch/reference/7.9/heap-size.html + Xmx: 1g opensearch_security: ssl: transport: diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml new file mode 100644 index 0000000000..b750863247 --- /dev/null +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -0,0 +1,11 @@ +kind: configuration/opensearch-dashboards +title: "OpenSearch-dashboards" +name: default +specification: + ops_dashboards_user: admin + ops_dashboards_password: PASSWORD_TO_CHANGE + ops_user: opensearch + ops_dashboards_log_dir: /var/log/opensearchdashboards + ops_dashboards_home: /usr/share/opensearch-dashboards + ops_dashboards_conf_dir: /usr/share/opensearch-dashboards/config + ops_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 99d3845260..8fbc5a876c 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -2,8 +2,9 @@ kind: configuration/opensearch-for-opensearch title: OpenSearch Config name: default specification: - cluster_name: EpiphanyElastic + cluster_name: EpiphanyOpensearch clustered: true + ops_user: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: false @@ -16,11 +17,15 @@ specification: - logstash - kibanaserver paths: - data: /var/lib/opensearch - repo: /var/lib/opensearch-snapshots - logs: /var/log/opensearch + ops_home: /usr/share/opensearch + ops_conf_dir: /usr/share/opensearch/config + ops_log_dir: /var/log/opensearch + ops_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin + ops_repo: /var/lib/opensearch-snapshots + ops_data: /var/lib/opensearch + ops_logs: /var/log/opensearch jvm_options: - Xmx: 1g # see https://www.elastic.co/guide/en/elasticsearch/reference/7.9/heap-size.html + Xmx: 1g opensearch_security: ssl: transport: diff --git a/schema/common/validation/configuration/opensearch-dashboards.yml b/schema/common/validation/configuration/opensearch-dashboards.yml new file mode 100644 index 0000000000..23cfbfa7b5 --- /dev/null +++ b/schema/common/validation/configuration/opensearch-dashboards.yml @@ -0,0 +1,5 @@ +kind: configuration/opensearch-dashboards +title: "OpenSearch-dashboards" +name: default +specification: + ops_dashboards_log_dir: /var/log/opensearchdashboards From f279f162dbd1cbb165c3b9e7af4ef8102cc8a473 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 27 Jan 2022 23:50:26 +0100 Subject: [PATCH 006/157] Migration ODFE -> OpS --- .../{opendistro_for_elasticsearch-01.yml => opensearch-01.yml} | 0 .../{opendistro_for_elasticsearch-02.yml => opensearch-02.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch-01.yml => opensearch-01.yml} (100%) rename ansible/playbooks/roles/upgrade/tasks/{opendistro_for_elasticsearch-02.yml => opensearch-02.yml} (100%) diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-01.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml similarity index 100% rename from ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch-02.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml From 556fb3a770747460aba6bfb742d6f6c36777b3ea Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 28 Jan 2022 15:07:32 +0100 Subject: [PATCH 007/157] get rid of schema references --- .../roles/opensearch/defaults/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 1e95990215..5ef9d18f08 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -59,12 +59,13 @@ ports: http: 9200 # defaults to range but we want static port transport: 9300 # defaults to range but we want static port log4j_file_name: apache-log4j-2.17.1-bin.tar.gz +systemctl_path: /etc/systemd/system -admin_password: "{{ specification.admin_password }}" -ops_user: "{{ specification.ops_user }}" -ops_home: "{{ specification.paths.ops_home }}" -ops_conf_dir: "{{ specification.paths.ops_conf_dir }}" -ops_log_dir: "{{ specification.paths.ops_log_dir }}" -ops_plugin_bin_path: "{{ specification.paths.ops_plugin_bin_path }}" -ops_api_port: "{{ ports.http }}" -systemctl_path: /etc/systemd/system \ No newline at end of file +# TODO: Remove the below entries +# admin_password: "{{ specification.admin_password }}" +# ops_user: "{{ specification.ops_user }}" +# ops_home: "{{ specification.paths.ops_home }}" +# ops_conf_dir: "{{ specification.paths.ops_conf_dir }}" +# ops_log_dir: "{{ specification.paths.ops_log_dir }}" +# ops_plugin_bin_path: "{{ specification.paths.ops_plugin_bin_path }}" +# ops_api_port: "{{ ports.http }}" \ No newline at end of file From 70b422ee63c711c5a81abce9668cc08b7561380f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 28 Jan 2022 15:09:04 +0100 Subject: [PATCH 008/157] Reanming as a part of migration --- .../{opendistro-for-elasticsearch.yml => opensearch.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename schema/common/validation/configuration/{opendistro-for-elasticsearch.yml => opensearch.yml} (87%) diff --git a/schema/common/validation/configuration/opendistro-for-elasticsearch.yml b/schema/common/validation/configuration/opensearch.yml similarity index 87% rename from schema/common/validation/configuration/opendistro-for-elasticsearch.yml rename to schema/common/validation/configuration/opensearch.yml index cdefba3076..e31f942979 100644 --- a/schema/common/validation/configuration/opendistro-for-elasticsearch.yml +++ b/schema/common/validation/configuration/opensearch.yml @@ -1,6 +1,6 @@ "$id": "#/specification" -title: "opensearch-for-elasticsearch specification schema" -description: "opensearch-for-elasticsearch specification schema" +title: "opensearch schema" +description: "opensearch specification schema" type: object properties: cluster_name: From 0d9bd804711d16bf5b2422ea1e8f469b7ed28573 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 28 Jan 2022 15:11:29 +0100 Subject: [PATCH 009/157] Reanming as a part of migration --- ansible/playbooks/filebeat.yml | 2 +- cli/epicli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/filebeat.yml b/ansible/playbooks/filebeat.yml index 94233b3970..952fefa1aa 100644 --- a/ansible/playbooks/filebeat.yml +++ b/ansible/playbooks/filebeat.yml @@ -1,7 +1,7 @@ --- # Ansible playbook that installs and configures Filebeat -- hosts: opensearch:logging:kibana # to gather facts +- hosts: opensearch:logging:opensearch_dashboards # to gather facts tasks: [] - hosts: filebeat diff --git a/cli/epicli.py b/cli/epicli.py index de1e7f1f9b..fa4cd7dea7 100644 --- a/cli/epicli.py +++ b/cli/epicli.py @@ -256,7 +256,7 @@ def upgrade_parser(subparsers): 'jmx_exporter', 'kafka', 'kafka_exporter', - 'kibana', + 'opensearch_dashboards', 'kubernetes', 'load_balancer', 'logging', From a296a0dcc6439d7a9d37ab008f080a040bacbf1f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 28 Jan 2022 15:12:35 +0100 Subject: [PATCH 010/157] get rid of schema references --- .../roles/opensearch/tasks/configure-ops.yml | 8 ++++---- .../roles/opensearch/tasks/install-ops.yml | 14 +++++++------- .../opensearch/templates/opensearch.service.j2 | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index 327618a300..d0f83642d3 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -5,15 +5,15 @@ file: path: "{{ specification.paths.ops_repo }}/" state: directory - owner: "{{ ops_user }}" - group: "{{ ops_user }}" + owner: "{{ specification.ops_user }}" + group: "{{ specification.ops_user }}" mode: u=rwx,go= - name: Provide JVM configuration file template: backup: yes src: jvm.options.j2 - dest: "{{ ops_conf_dir }}/jvm.options" + dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" owner: root group: opensearch mode: ug=rw,o= @@ -35,7 +35,7 @@ template: backup: yes src: opensearch.yml.j2 - dest: "{{ ops_conf_dir }}/opensearch.yml" + dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" owner: root group: opensearch mode: ug=rw,o= diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index e1f18c9f42..0c350181c8 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -8,7 +8,7 @@ - name: Ensure Opensearch service user exists user: - name: "{{ ops_user }}" + name: "{{ specification.ops_user }}" state: present shell: /bin/bash @@ -16,16 +16,16 @@ file: path: "{{ item }}" state: directory - owner: "{{ ops_user }}" - group: "{{ ops_user }}" + owner: "{{ specification.ops_user }}" + group: "{{ specification.ops_user }}" with_items: - - "{{ ops_home }}" - - "{{ ops_log_dir }}" - - "{{ ops_conf_dir }}" + - "{{ specification.paths.ops_home }}" + - "{{ specification.paths.ops_log_dir }}" + - "{{ specification.paths.ops_conf_dir }}" - "{{ certificates.dirs.certs }}" - name: Extract the tar file - command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ ops_home }} --strip-components=1" + command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ specification.paths.ops_home }} --strip-components=1" - name: Create systemd service template: diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 index 13f4ff9bea..2978f29c49 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 @@ -7,12 +7,12 @@ After=network-online.target RuntimeDirectory=opensearch PrivateTmp=true -WorkingDirectory={{ ops_home }} +WorkingDirectory={{ specification.paths.ops_home }} -User={{ ops_user }} -Group={{ ops_user }} +User={{ specification.ops_user }} +Group={{ specification.ops_user }} -ExecStart={{ ops_home }}/bin/opensearch -p {{ ops_home }}/opensearch.pid -q +ExecStart={{ specification.paths.ops_home }}/bin/opensearch -p {{ specification.paths.ops_home }}/opensearch.pid -q StandardOutput=journal StandardError=inherit From 70b6342e2bcdc67078ddf3126d5986a0d9ac4414 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 31 Jan 2022 14:59:03 +0100 Subject: [PATCH 011/157] TODO later on --- .../playbooks/roles/opensearch/tasks/patch-log4j.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml index 1947ae27b8..01578552f6 100644 --- a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml +++ b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml @@ -61,8 +61,9 @@ _archive_root_dir: >- {{ unarchive_list_files.files | first | dirname }} -- name: Restart opensearch-performance-analyzer service - systemd: - name: opensearch-performance-analyzer - state: restarted - when: log4j_patch.changed +# TODO: add instllation of PerfAn to opensearch role +# - name: Restart opensearch-performance-analyzer service +# systemd: +# name: opensearch-performance-analyzer +# state: restarted +# when: log4j_patch.changed From 41f5c95c824bee2cc860beafb717e38522232bba Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 31 Jan 2022 14:59:37 +0100 Subject: [PATCH 012/157] Unified location of state file --- ansible/playbooks/roles/upgrade/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/upgrade/defaults/main.yml b/ansible/playbooks/roles/upgrade/defaults/main.yml index 225b8b4a28..3f96e23b88 100644 --- a/ansible/playbooks/roles/upgrade/defaults/main.yml +++ b/ansible/playbooks/roles/upgrade/defaults/main.yml @@ -18,7 +18,7 @@ opensearch: dual_root_ca: filename: demo2epiphany-certs-migration-root-CAs.pem - upgrade_state_file_path: /etc/elasticsearch/epicli-upgrade-started.state + upgrade_state_file_path: /var/lib/epiphany/upgrade/state/opensearch-upgrade.uncompleted kubernetes: upgrade_state_file_path: /var/lib/epiphany/upgrade/state/kubernetes-{{ ver }}.uncompleted From 39d2eae89928cf2cdb72f2973b6a759d92836c90 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 31 Jan 2022 15:00:04 +0100 Subject: [PATCH 013/157] THe corect one --- .../roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml index 720531147a..e91b1681bb 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml @@ -10,7 +10,7 @@ - name: ODFE | Ensure elasticsearch service is running systemd: - name: opensearch + name: elasticsearch enabled: yes state: started register: elasticsearch_state From 73a91fc1d52c0b010ae03c04538e2c565969737d Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 31 Jan 2022 15:00:30 +0100 Subject: [PATCH 014/157] THe corect one --- schema/common/defaults/configuration/opensearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 8fbc5a876c..96dbd04ed3 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -1,4 +1,4 @@ -kind: configuration/opensearch-for-opensearch +kind: configuration/opensearch title: OpenSearch Config name: default specification: From 0591058945b27823b83b11b91b3262c42769109a Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 31 Jan 2022 15:00:56 +0100 Subject: [PATCH 015/157] Migration to opensearch --- .../roles/upgrade/tasks/opensearch-01.yml | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index d7fc09b7ef..17a784fb65 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -1,41 +1,64 @@ --- -- name: ODFE | Get information about installed packages as facts +- name: OPS | Get information about installed packages as facts package_facts: manager: auto when: ansible_facts.packages is undefined -- name: ODFE | Assert that elasticsearch-oss package is installed +- name: OPS | Assert that elasticsearch-oss package is installed assert: that: ansible_facts.packages['elasticsearch-oss'] is defined fail_msg: elasticsearch-oss package not found, nothing to upgrade quiet: true -- name: ODFE | Include defaults from opensearch role +- name: OPS | Include defaults from opensearch role include_vars: file: roles/opensearch/defaults/main.yml - name: odfe_defaults + name: ops_defaults -- name: ODFE | Patch log4j +- name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml + include_vars: + file: roles/opensearch/vars/main.yml + name: ops_vars + +- name: Ensure Opensearch service user exists + user: + name: "{{ ops_vars.specification.ops_user }}" + state: present + shell: /bin/bash + +- name: Ensure directory structure exists + file: + path: "{{ item }}" + state: directory + owner: "{{ ops_vars.specification.ops_user }}" + group: "{{ ops_vars.specification.ops_user }}" + with_items: + - "{{ ops_vars.specification.paths.ops_home }}" + - "{{ ops_vars.specification.paths.ops_log_dir }}" + - "{{ ops_vars.specification.paths.ops_conf_dir }}" + - "{{ ops_defaults.certificates.dirs.certs }}" + +- name: OPS | Patch log4j include_role: name: opensearch tasks_from: patch-log4j - when: odfe_defaults.log4j_file_name is defined + when: ops_defaults.log4j_file_name is defined - name: Restart elasticsearch service systemd: name: opensearch state: restarted register: restart_opensearch - when: odfe_defaults.log4j_file_name is defined and log4j_patch.changed + when: ops_defaults.log4j_file_name is defined and log4j_patch.changed -- name: ODFE | Print elasticsearch-oss versions +- name: OPS | Print elasticsearch-oss versions debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Target version: {{ odfe_defaults.versions[ansible_os_family].elasticsearch_oss }}" + - "Target version: {{ ops_defaults.versions[ansible_os_family].ops_version }}" # If state file exists it means the previous run failed -- name: ODFE | Check if upgrade state file exists +- name: OPS | Check if upgrade state file exists stat: path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false @@ -43,10 +66,11 @@ get_mime: false register: stat_upgrade_state_file -- name: ODFE | Upgrade Elasticsearch and ODFE plugins (part 1/2) +- name: OPS | Upgrade Elasticsearch and ODFE plugins (part 1/2) include_tasks: opensearch/upgrade-opensearch-01.yml - when: _target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') - or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') - and stat_upgrade_state_file.stat.exists) + # This check will be reenabled when the upgrade will be done again within sinlge product + # when: _target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') + # or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') + # and stat_upgrade_state_file.stat.exists) vars: - _target_version: "{{ odfe_defaults.versions[ansible_os_family].elasticsearch_oss }}" + _target_version: "{{ ops_defaults.versions[ansible_os_family].ops_version }}" From d8a0c2de07a1d9f7125d12a54d3839750259e2b2 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 2 Feb 2022 11:25:07 +0100 Subject: [PATCH 016/157] A new flag for product replacement instead oof version upgrade --- schema/common/defaults/configuration/logging.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 8d4f227f35..9775112518 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -3,6 +3,7 @@ title: Logging Config name: default specification: cluster_name: EpiphanyOpensearch + odfe_migration: false ops_user: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE From 779194db54d4ae949036d81d3642ed8348d2eff1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Feb 2022 09:22:50 +0100 Subject: [PATCH 017/157] ODFE product migration to OPS --- .../roles/upgrade/tasks/odfe-migration.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml new file mode 100644 index 0000000000..9ef2d23505 --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml @@ -0,0 +1,35 @@ +--- +- name: ODFE migr | Ensure elasticsearch service is running + systemd: + name: elasticsearch + enabled: yes + state: started + register: elasticsearch_state + +- include_tasks: opensearch/utils/get-config-from-files.yml # Sets 'existing_config' fact + +- name: ODFE migr | Set common facts + set_fact: + certificates: "{{ ops_defaults.certificates }}" + es_host: "{{ existing_config.main['network.host'] | default('_local_') }}" + es_http_port: "{{ existing_config.main['http.port'] | default(ops_defaults.ports.http) }}" + es_transport_port: "{{ existing_config.main['transport.port'] | default(ops_defaults.ports.transport) }}" + es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" + es_node_name: "{{ existing_config.main['node.name'] }}" + +- name: ODFE migr | Prepare for ODFE to OPS migration + include_tasks: + file: opensearch/utils/prepare-cluster-for-node-restart.yml + apply: + delegate_to: "{{ target_inventory_hostname }}" + delegate_facts: true + loop: "{{ ansible_play_hosts_all }}" + loop_control: + loop_var: target_inventory_hostname + vars: + es_api: + cert_type: Epiphany + cert_path: &epi_cert_path "{{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join }}" + key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join }}" + url: https://{{ es_host }}:{{ es_http_port }} + fail_msg: API access test failed. From e991ad9d28886515ec7904ac706751b769c14daf Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Feb 2022 09:23:11 +0100 Subject: [PATCH 018/157] Needed for ODFE product migration to OPS --- schema/common/defaults/configuration/opensearch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 96dbd04ed3..79a8a29d19 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -3,6 +3,7 @@ title: OpenSearch Config name: default specification: cluster_name: EpiphanyOpensearch + odfe_migration: false clustered: true ops_user: opensearch admin_password: PASSWORD_TO_CHANGE From 623a78887c1a6f53053da0d5d064403dfdcc8091 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Feb 2022 09:23:28 +0100 Subject: [PATCH 019/157] Migration to opensearch --- .../roles/upgrade/tasks/opensearch-01.yml | 39 +++++++++++-------- .../opensearch/upgrade-opensearch-01.yml | 22 +++++------ 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index 17a784fb65..d6a9c16af6 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -38,20 +38,21 @@ - "{{ ops_vars.specification.paths.ops_conf_dir }}" - "{{ ops_defaults.certificates.dirs.certs }}" -- name: OPS | Patch log4j - include_role: - name: opensearch - tasks_from: patch-log4j - when: ops_defaults.log4j_file_name is defined +# TODO: Remove this part of code +# - name: OPS | Patch log4j +# include_role: +# name: opensearch +# tasks_from: patch-log4j +# when: ops_defaults.log4j_file_name is defined -- name: Restart elasticsearch service - systemd: - name: opensearch - state: restarted - register: restart_opensearch - when: ops_defaults.log4j_file_name is defined and log4j_patch.changed +# - name: Restart elasticsearch service +# systemd: +# name: elasticsearch +# state: restarted +# register: restart_opensearch +# when: ops_defaults.log4j_file_name is defined and log4j_patch.changed -- name: OPS | Print elasticsearch-oss versions +- name: OPS | Print elasticsearch ond opensearch versions debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" @@ -68,9 +69,15 @@ - name: OPS | Upgrade Elasticsearch and ODFE plugins (part 1/2) include_tasks: opensearch/upgrade-opensearch-01.yml - # This check will be reenabled when the upgrade will be done again within sinlge product - # when: _target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') - # or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') - # and stat_upgrade_state_file.stat.exists) + when: _target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') + or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') + and stat_upgrade_state_file.stat.exists) vars: _target_version: "{{ ops_defaults.versions[ansible_os_family].ops_version }}" + +- include_role: + name: upgrade + tasks_from: odfe-migration + when: ops_vars.specification.odfe_migration + vars: + current_group_name: logging \ No newline at end of file diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml index e91b1681bb..c5b951c2e2 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml @@ -1,14 +1,14 @@ --- # This file contains only pre-upgrade tasks that can be run in parallel on all hosts -- name: ODFE | Create upgrade state file +- name: OPS | Create upgrade state file become: true file: path: "{{ opensearch.upgrade_state_file_path }}" state: touch mode: u=rw,g=r,o= -- name: ODFE | Ensure elasticsearch service is running +- name: OPS | Ensure elasticsearch service is running systemd: name: elasticsearch enabled: yes @@ -18,16 +18,16 @@ # Sets 'existing_config' fact - include_tasks: utils/get-config-from-files.yml -- name: ODFE | Set common facts +- name: OPS | Set common facts set_fact: - certificates: "{{ odfe_defaults.certificates }}" + certificates: "{{ ops_defaults.certificates }}" es_host: "{{ existing_config.main['network.host'] | default('_local_') }}" - es_http_port: "{{ existing_config.main['http.port'] | default(odfe_defaults.ports.http) }}" - es_transport_port: "{{ existing_config.main['transport.port'] | default(odfe_defaults.ports.transport) }}" + es_http_port: "{{ existing_config.main['http.port'] | default(ops_defaults.ports.http) }}" + es_transport_port: "{{ existing_config.main['transport.port'] | default(ops_defaults.ports.transport) }}" es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" es_node_name: "{{ existing_config.main['node.name'] }}" -- name: ODFE | Wait for elasticsearch service to start up +- name: OPS | Wait for elasticsearch service to start up wait_for: port: "{{ es_transport_port }}" host: "{{ es_host if (es_host is not regex('^_.+_$')) else '0.0.0.0' }}" # 0.0.0.0 means any IP @@ -51,20 +51,20 @@ name: certificate tasks_from: install-packages # requirements for Ansible certificate modules -- name: ODFE | Get information on root CA certificate +- name: OPS | Get information on root CA certificate community.crypto.x509_certificate_info: # 'pemtrustedcas_filepath' is a relative path - path: "{{ ('/etc/elasticsearch', existing_config.main['opensearch_security.ssl.transport.pemtrustedcas_filepath']) | path_join }}" + path: "{{ ('/etc/elasticsearch', existing_config.main['opendistro_security.ssl.transport.pemtrustedcas_filepath']) | path_join }}" register: _root_ca_info -- name: ODFE | Check if demo or Epiphany certificates are in use # self-signed +- name: OPS | Check if demo or Epiphany certificates are in use # self-signed set_fact: _is_demo_cert_in_use: "{{ 'True' if _root_ca_info.subject.commonName == 'Example Com Inc. Root CA' else 'False' }}" _is_epiphany_cert_in_use: "{{ 'True' if _root_ca_info.subject.commonName == 'Epiphany Managed ODFE Root CA' else 'False' }}" # For custom admin cert (non-demo and non-Epiphany), we use workaround (upgrade_config.custom_admin_certificate). # The workaround should be replaced after implementing task #2127. -- name: ODFE | Set API access facts +- name: OPS | Set API access facts set_fact: es_api: cert_path: "{{ _cert_path[_cert_type] }}" From 44f177811fcdf0c741bc7178f40d91b9d064cbd3 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Feb 2022 15:24:36 +0100 Subject: [PATCH 020/157] Temporary, old certs --- .../roles/upgrade/tasks/odfe-migration.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml index 9ef2d23505..00569fe85b 100644 --- a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml @@ -17,6 +17,18 @@ es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" es_node_name: "{{ existing_config.main['node.name'] }}" +- name: ODFE migr | Assure Elastisearch files location will be used + set_fact: + certificates: + dirs: + certs: "/etc/elasticsearch" + ca_key: "/etc/elasticsearch/private" + csr: "/etc/elasticsearch/csr" + files: + admin: + cert: "epiphany-admin.pem" + key: "epiphany-admin-key.pem" + - name: ODFE migr | Prepare for ODFE to OPS migration include_tasks: file: opensearch/utils/prepare-cluster-for-node-restart.yml @@ -29,7 +41,7 @@ vars: es_api: cert_type: Epiphany - cert_path: &epi_cert_path "{{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join }}" - key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join }}" + cert_path: &epi_cert_path "{{ (certificates.dirs.certs, certificates.files.admin.cert) | path_join }}" + key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key) | path_join }}" url: https://{{ es_host }}:{{ es_http_port }} fail_msg: API access test failed. From 14976568eaf4edd164374558b2e3063d76aaab44 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Feb 2022 09:55:11 +0100 Subject: [PATCH 021/157] Migration ODFE to OPS and OPSD --- .../roles/opensearch/defaults/main.yml | 11 +- .../roles/opensearch/tasks/configure-ops.yml | 6 - .../roles/opensearch/tasks/install-ops.yml | 4 +- .../roles/opensearch/templates/jvm.options.j2 | 2 +- .../opensearch/templates/opensearch.yml.j2 | 10 +- .../roles/upgrade/tasks/kibana-migration.yml | 36 ++++ .../roles/upgrade/tasks/odfe-migration.yml | 161 ++++++++++++++++++ .../roles/upgrade/tasks/opensearch-01.yml | 9 +- 8 files changed, 218 insertions(+), 21 deletions(-) create mode 100644 ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 5ef9d18f08..5bc1bd232f 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -4,8 +4,10 @@ versions: RedHat: ops_version: "1.2.4" + opsd_version: "1.2.0" Debian: ops_version: "1.2.4" + opsd_version: "1.2.0" certificates: dirs: # must be under the config directory, specified using a relative path certs: /usr/share/opensearch/config/cert @@ -60,12 +62,3 @@ ports: transport: 9300 # defaults to range but we want static port log4j_file_name: apache-log4j-2.17.1-bin.tar.gz systemctl_path: /etc/systemd/system - -# TODO: Remove the below entries -# admin_password: "{{ specification.admin_password }}" -# ops_user: "{{ specification.ops_user }}" -# ops_home: "{{ specification.paths.ops_home }}" -# ops_conf_dir: "{{ specification.paths.ops_conf_dir }}" -# ops_log_dir: "{{ specification.paths.ops_log_dir }}" -# ops_plugin_bin_path: "{{ specification.paths.ops_plugin_bin_path }}" -# ops_api_port: "{{ ports.http }}" \ No newline at end of file diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index d0f83642d3..bd451f3995 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -118,12 +118,6 @@ when: change_config.changed or change_jvm_config.changed -- name: Enable and start opensearch service - systemd: - name: opensearch - state: started - enabled: yes - - name: Change default users when: not is_upgrade_run block: diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 0c350181c8..48d1c65fe5 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -22,6 +22,8 @@ - "{{ specification.paths.ops_home }}" - "{{ specification.paths.ops_log_dir }}" - "{{ specification.paths.ops_conf_dir }}" + - "{{ specification.paths.ops_data }}" + - "{{ specification.paths.ops_logs }}" - "{{ certificates.dirs.certs }}" - name: Extract the tar file @@ -29,5 +31,5 @@ - name: Create systemd service template: - src: opensearch.service.j2 + src: roles/opensearch/templates/opensearch.service.j2 dest: "{{ systemctl_path }}/opensearch.service" diff --git a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 index def6b9e830..d60c44fb1f 100644 --- a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 +++ b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 @@ -51,7 +51,7 @@ 14-:-XX:InitiatingHeapOccupancyPercent=30 ## JVM temporary directory --Djava.io.tmpdir=${OPENSEARCH_TMPDIR} +-Djava.io.tmpdir=${ES_TMPDIR} ## heap dumps diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 index 673d61092e..6e23774b88 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 @@ -2,6 +2,10 @@ # {{ ansible_managed }} # ======================== OpenSearch Configuration ========================= # +# ------------------- Legacy Clients Compability Flag ------------------------- +# +compatibility.override_main_response_version: true +# # ---------------------------------- Cluster ----------------------------------- # # Use a descriptive name for your cluster: @@ -22,15 +26,15 @@ node.name: {{ ansible_hostname }} # # Path to directory where to store the data (separate multiple locations by comma): # -# path.data: {{ specification.paths.ops_data }} +path.data: {{ specification.paths.ops_data }} # # Path to directory where the shared storage should be mounted: # -# path.repo: {{ specification.paths.ops_repo }} +path.repo: {{ specification.paths.ops_repo }} # # Path to log files: # -# path.logs: {{ specification.paths.ops_logs }} +path.logs: {{ specification.paths.ops_logs }} # # ----------------------------------- Memory ----------------------------------- # diff --git a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml b/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml new file mode 100644 index 0000000000..5ed8ce2c6d --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml @@ -0,0 +1,36 @@ +--- +- name: Kibana migr | Stop elasticsearch service + systemd: + name: elasticsearch + enabled: yes + state: stopped + register: elasticsearch_state + +- name: Download Opensearch + include_role: + name: download + tasks_from: download_file + vars: + file_name: "opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + +- name: Extract the tar file + command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ specification.paths.ops_home }} --strip-components=1" + +- name: Kibana migr | Clone kibana settings + copy: + src: /etc/kibana/kibana.yml # Hardoced for testing purposes only + dest: "{{ specification.paths.ops_conf_dir }}/opensearch_dashboards.yml" + remote_src: yes + owner: opensearch + group: root + mode: ug=rw,o= + backup: yes + +- name: Kibana migr | Porting kibana settings to OpenSearch Dashboards + replace: + path: "{{ specification.paths.ops_conf_dir }}/opensearch_dashboards.yml" + regexp: "{{ item.1 }}" + replace: "{{ item.2 }}" + with_items: + - { 1: 'elasticsearch', 2: 'opensearch' } + - { 1: 'kibana', 2: 'opensearchDashboards' } diff --git a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml index 00569fe85b..503c7707f7 100644 --- a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml @@ -1,4 +1,15 @@ --- +- name: OPS | Get information about installed packages as facts + package_facts: + manager: auto + when: ansible_facts.packages is undefined + +- name: OPS | Print elasticsearch ond opensearch versions + debug: + msg: + - "Elasticsearch version curently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" + - "Opensearch version to be installed: {{ ops_defaults.versions[ansible_os_family].ops_version }}" + - name: ODFE migr | Ensure elasticsearch service is running systemd: name: elasticsearch @@ -45,3 +56,153 @@ key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key) | path_join }}" url: https://{{ es_host }}:{{ es_http_port }} fail_msg: API access test failed. + +- name: ODFE migr | Stop elasticsearch service + systemd: + name: elasticsearch + enabled: yes + state: stopped + register: elasticsearch_state + +- name: ODFE migr | Include vars 1/2 for Opensearch binaries instalaltion + include_vars: + file: roles/opensearch/defaults/main.yml + +- name: ODFE migr | Include vars 2/2 for Opensearch binaries instalaltion + include_vars: + file: roles/opensearch/vars/main.yml + +- name: ODFE migr | Inastall Opensearch binaries + include_tasks: roles/opensearch/tasks/install-ops.yml + +- name: ODFE migr | Copy data and logs directories + copy: + src: "/var/lib/elasticsearch" # Hardoced for testing purposes only + dest: "{{ specification.paths.ops_data }}" + remote_src: yes + directory_mode: yes + +- name: ODFE migr | Copy snapshots directories + copy: + src: "/var/lib/elasticsearch-snapshots" # Hardoced for testing purposes only + dest: "{{ specification.paths.ops_repo }}" + remote_src: yes + directory_mode: yes + +- name: ODFE migr | Prepare a list of certs and keys to OPS directories + find: + paths: "/etc/elasticsearch/" + patterns: "*pem" + register: pem_files + +- name: ODFE migr | Copy a list of certs and keys to OPS directories + copy: + src: "{{ item.path }}" + dest: "{{ specification.paths.ops_conf_dir }}/" + remote_src: yes + with_items: "{{ pem_files.files }}" + +- name: ODFE migr | Clone JVM configuration file + copy: + src: /etc/elasticsearch/jvm.options # Hardoced for testing purposes only + dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" + remote_src: yes + owner: root + group: opensearch + mode: ug=rw,o= + backup: yes + +- name: ODFE migr | Update JVM configuration file + replace: + path: "{{ specification.paths.ops_conf_dir }}/jvm.options" + regexp: "{{ item.1 }}" + replace: "{{ item.2 }}" + with_items: + - { 1: 'elasticsearch', 2: 'opensearch' } + - { 1: '\${ES_TMPDIR}', 2: '/tmp' } + +- name: ODFE migr | Clone main configuration file + copy: + src: /etc/elasticsearch/elasticsearch.yml # Hardoced for testing purposes only + dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" + remote_src: yes + owner: root + group: opensearch + mode: ug=rw,o= + backup: yes + +- name: ODFE migr | Update main configuration file + replace: + path: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" + regexp: "{{ item.1 }}" + replace: "{{ item.2 }}" + with_items: + - { 1: 'elasticsearch', 2: 'opensearch' } + # - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } + - { 1: 'opendistro_security.', 2: 'plugins.security.' } + +- name: ODFE migr | Start opensearch service + systemd: + name: opensearch + state: started + enabled: yes + register: restart_opensearch + +- name: ODFE migr | Wait for opensearch to startup + wait_for: + port: 9200 + host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" + sleep: 6 + + +# - name: ODFE migr | Check if default admin user exists +# uri: +# url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/admin" +# method: GET +# # 404 code is used there as someone can remove admin user on its own. +# status_code: [200, 404] +# validate_certs: no +# url_username: admin +# url_password: admin +# register: admin_check_response +# until: admin_check_response is success +# retries: 60 +# delay: 1 +# run_once: true + +- name: ODFE migr | Set Opensearch admin password + uri: + url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" + method: PATCH + status_code: [200] + body: + - op: "replace" + path: "/admin" + value: + password: "{{ specification.admin_password }}" + reserved: "true" + backend_roles: + - "admin" + description: "Admin user" + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key }}" + body_format: json + validate_certs: no + register: uri_response + until: uri_response is success + retries: 5 + delay: 1 + run_once: true + # when: admin_check_response.status == 200 + +- name: ODFE migr | Check the opensearch status + command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k + register: ops_status + +- debug: + msg: "Cluster {{ inventory_hostname }}: {{ specification.admin_password }}" + +- name: ODFE migr | Show the opensearch status + debug: + msg: "{{ ops_status.stdout }}" + failed_when: "'number_of_nodes' not in ops_status.stdout" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index d6a9c16af6..8f0d5afcec 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -80,4 +80,11 @@ tasks_from: odfe-migration when: ops_vars.specification.odfe_migration vars: - current_group_name: logging \ No newline at end of file + current_group_name: logging + +- include_role: + name: upgrade + tasks_from: kibana-migration + when: ops_vars.specification.odfe_migration + vars: + current_group_name: logging From d7bba164501ac725fe850cefe597ab57af43a741 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Feb 2022 21:18:54 +0100 Subject: [PATCH 022/157] Kibana role removal --- .../playbooks/roles/kibana/defaults/main.yml | 8 --- ansible/playbooks/roles/kibana/tasks/main.yml | 68 ------------------- .../roles/kibana/tasks/setup-logging.yml | 27 -------- .../roles/kibana/templates/kibana.yml.j2 | 64 ----------------- .../roles/kibana/templates/logrotate.conf.j2 | 8 --- 5 files changed, 175 deletions(-) delete mode 100644 ansible/playbooks/roles/kibana/defaults/main.yml delete mode 100644 ansible/playbooks/roles/kibana/tasks/main.yml delete mode 100644 ansible/playbooks/roles/kibana/tasks/setup-logging.yml delete mode 100644 ansible/playbooks/roles/kibana/templates/kibana.yml.j2 delete mode 100644 ansible/playbooks/roles/kibana/templates/logrotate.conf.j2 diff --git a/ansible/playbooks/roles/kibana/defaults/main.yml b/ansible/playbooks/roles/kibana/defaults/main.yml deleted file mode 100644 index f07c1f3457..0000000000 --- a/ansible/playbooks/roles/kibana/defaults/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kibana_version: - RedHat: "1.13.1" - Debian: "1.13.1" - -# Required and used for upgrade Open Distro for Elasticsearch - Kibana: -specification: - kibana_log_dir: /var/log/kibana diff --git a/ansible/playbooks/roles/kibana/tasks/main.yml b/ansible/playbooks/roles/kibana/tasks/main.yml deleted file mode 100644 index 1e68011a4a..0000000000 --- a/ansible/playbooks/roles/kibana/tasks/main.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -- name: Install Kibana package - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: - - opendistroforelasticsearch-kibana={{ kibana_version[ansible_os_family] }} - RedHat: - - opendistroforelasticsearch-kibana-{{ kibana_version[ansible_os_family] }} - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } - -- name: Include logging configuration tasks - include_tasks: setup-logging.yml - -- name: Load variables from logging/opensearch role - when: context is undefined or context != "upgrade" - block: - - name: Load variables from logging role - include_vars: - file: roles/logging/vars/main.yml - name: opendistro_for_logging_vars - when: "'logging' in group_names" - - - name: Load variables from opensearch role - include_vars: - file: roles/opensearch/vars/main.yml - name: opendistro_for_data_vars - when: "'opensearch' in group_names" - -- name: Update Kibana configuration file - template: - backup: yes - src: kibana.yml.j2 - dest: /etc/kibana/kibana.yml - owner: kibana - group: root - mode: u=rw,go= - register: change_config - -- name: Restart Kibana service - systemd: - name: kibana - state: restarted - when: change_config.changed - -- name: Start kibana service - service: - name: kibana - state: started - enabled: yes - -- name: Wait for kibana to start listening - wait_for: - host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" - port: 5601 - delay: 5 - -- name: Wait for Kibana to be ready - uri: - url: http://{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}:5601/api/status - method: GET - register: response - until: "'kbn_name' in response and response.status == 200" - retries: 120 - delay: 2 diff --git a/ansible/playbooks/roles/kibana/tasks/setup-logging.yml b/ansible/playbooks/roles/kibana/tasks/setup-logging.yml deleted file mode 100644 index d87de424ce..0000000000 --- a/ansible/playbooks/roles/kibana/tasks/setup-logging.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Create log directory for Kibana - file: path={{ specification.kibana_log_dir }} state=directory - -- name: Create logfile for Kibana - copy: - dest: "{{ specification.kibana_log_dir }}/kibana.log" - owner: kibana - group: kibana - mode: 0644 - force: no - content: "" - -- name: Set permissions on logfile for Kibana - file: - path: "{{ specification.kibana_log_dir }}/kibana.log" - owner: kibana - group: kibana - mode: 0644 - -- name: Copy logrotate config - template: - dest: /etc/logrotate.d/kibana - owner: root - group: root - mode: 0644 - src: logrotate.conf.j2 diff --git a/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 b/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 deleted file mode 100644 index 7e7b73ff09..0000000000 --- a/ansible/playbooks/roles/kibana/templates/kibana.yml.j2 +++ /dev/null @@ -1,64 +0,0 @@ -# {{ ansible_managed }} - -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -# Description: -# Default Kibana configuration for Open Distro. - -server.host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" -elasticsearch.hosts: -{% if 'logging' in group_names %} - # Logging hosts: - {% for host in groups['logging'] %} - - "https://{{hostvars[host]['ansible_hostname']}}:9200" - {% endfor %} -{% elif 'opensearch' in group_names %} - # Data hosts: - {% for host in groups['opensearch'] %} - - "https://{{hostvars[host]['ansible_hostname']}}:9200" - {% endfor %} -{% endif %} - -elasticsearch.ssl.verificationMode: none -elasticsearch.username: kibanaserver -{% set password = 'kibanaserver' %} -{% if context is undefined or context != 'upgrade' -%} - {# mode: apply -#} - {% if 'logging' in group_names -%} - {% set password = opendistro_for_logging_vars.specification.kibanaserver_password -%} - {% elif 'opensearch' in group_names -%} - {% set password = opendistro_for_data_vars.specification.kibanaserver_password -%} - {% endif %} -{% else -%} - {# mode: upgrade -#} - {% set password = existing_es_password %} -{% endif %} -elasticsearch.password: {{ "'%s'" % password | replace("'","''") }} -elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"] - -# Enables you to specify a file where Kibana stores log output. -logging.dest: {{ specification.kibana_log_dir }}/kibana.log - -opensearch_security.multitenancy.enabled: true -opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] -opensearch_security.readonly_mode.roles: ["kibana_read_only"] - -# Provided with 1.10.1 version: -# https://opendistro.github.io/for-elasticsearch-docs/docs/upgrade/1-10-1/ -# Use this setting if you are running kibana without https -opensearch_security.cookie.secure: false - -newsfeed.enabled: false -telemetry.optIn: false -telemetry.enabled: false diff --git a/ansible/playbooks/roles/kibana/templates/logrotate.conf.j2 b/ansible/playbooks/roles/kibana/templates/logrotate.conf.j2 deleted file mode 100644 index d550d97e19..0000000000 --- a/ansible/playbooks/roles/kibana/templates/logrotate.conf.j2 +++ /dev/null @@ -1,8 +0,0 @@ -{{ specification.kibana_log_dir }}/*.log { - rotate 5 - daily - compress - missingok - notifempty - delaycompress -} From 1199b8c504169f42c808f22b15c52a1566fba784 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Feb 2022 21:19:35 +0100 Subject: [PATCH 023/157] Migration to opensearch --- .../roles/opensearch/defaults/main.yml | 2 - .../roles/opensearch/tasks/install-ops.yml | 8 +- .../opensearch_dashboards/defaults/main.yml | 4 +- .../templates/dashboards.service | 10 +-- .../roles/upgrade/tasks/kibana-migration.yml | 79 ++++++++++++++++--- .../roles/upgrade/tasks/odfe-migration.yml | 45 +++++------ ansible/playbooks/upgrade.yml | 10 --- .../common/defaults/configuration/logging.yml | 5 ++ .../configuration/opensearch-dashboards.yml | 16 ++-- .../defaults/configuration/opensearch.yml | 1 + 10 files changed, 116 insertions(+), 64 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 5bc1bd232f..9aeebc06ed 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -4,10 +4,8 @@ versions: RedHat: ops_version: "1.2.4" - opsd_version: "1.2.0" Debian: ops_version: "1.2.4" - opsd_version: "1.2.0" certificates: dirs: # must be under the config directory, specified using a relative path certs: /usr/share/opensearch/config/cert diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 48d1c65fe5..f496cbd6ba 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -27,7 +27,13 @@ - "{{ certificates.dirs.certs }}" - name: Extract the tar file - command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ specification.paths.ops_home }} --strip-components=1" + unarchive: + src: "/tmp/opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" + dest: "{{ specification.paths.ops_home }}" + owner: "{{ specification.ops_user }}" + remote_src: yes + extra_opts: + - --strip-components=1 - name: Create systemd service template: diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index 8793d21e14..acfb1c43d4 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -1,9 +1,9 @@ --- versions: RedHat: - ops_dashboards_version: "1.2.0" + opsd_version: "1.2.0" Debian: - ops_dashboards_version: "1.2.0" + opsd_version: "1.2.0" ops_nodes: |- {% for item in groups['ops-cluster'] -%} {{ hostvars[item]['ip'] }}{% if not loop.last %}","{% endif %} diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service index a0b36e7f9a..7961a744ac 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service @@ -7,12 +7,12 @@ After=network-online.target RuntimeDirectory=opensearch-dashboards PrivateTmp=true -WorkingDirectory={{ ops_dashboards_home }} +WorkingDirectory={{ specification.paths.opsd_home }} -User={{ ops_user }} -Group={{ ops_user }} +User={{ specification.opsd_user }} +Group={{ specification.opsd_user }} -ExecStart={{ ops_dashboards_home }}/bin/opensearch-dashboards -q +ExecStart={{ specification.paths.opsd_home }}/bin/opensearch-dashboards -q StandardOutput=journal StandardError=inherit @@ -45,4 +45,4 @@ SendSIGKILL=no SuccessExitStatus=143 [Install] -WantedBy=multi-user.target +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml b/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml index 5ed8ce2c6d..bd783be1ec 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml @@ -1,36 +1,91 @@ --- -- name: Kibana migr | Stop elasticsearch service +- name: Kibana migr | Load deafults from Opensearch Dashboards role + include_vars: + file: roles/opensearch_dashboards/defaults/main.yml + +- name: Kibana migr | Load vars from Opensearch Dashboards role # requires epicli upgrade -f .yml + include_vars: + file: roles/opensearch_dashboards/vars/main.yml + +- name: Kibana migr | Stop Kibana service systemd: - name: elasticsearch - enabled: yes + name: kibana + enabled: no state: stopped - register: elasticsearch_state -- name: Download Opensearch +- name: Kibana migr | Download Opensearch Dashboards binary include_role: name: download tasks_from: download_file vars: file_name: "opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" -- name: Extract the tar file - command: "chdir=/tmp/ tar -xvzf opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz -C {{ specification.paths.ops_home }} --strip-components=1" +- name: Kibana migr | Create opensearch-dashboards user + user: + name: "{{ specification.opsd_user }}" + password: "{{ specification.opsd_password }}" + state: present + shell: /bin/bash + +- name: Kibana migr | Create OPSD directories + file: + path: "{{ item }}" + state: directory + owner: "{{ specification.opsd_user }}" + group: "{{ specification.opsd_user }}" + mode: ug=rwx,o=rx + with_items: + - "{{ specification.paths.opsd_log_dir }}" + - "{{ specification.paths.opsd_home }}" + +- name: Kibana migr | Extract the tar file + unarchive: + src: "/tmp/opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + dest: "{{ specification.paths.opsd_home }}" + owner: "{{ specification.opsd_user }}" + remote_src: yes + extra_opts: + - --strip-components=1 - name: Kibana migr | Clone kibana settings copy: - src: /etc/kibana/kibana.yml # Hardoced for testing purposes only - dest: "{{ specification.paths.ops_conf_dir }}/opensearch_dashboards.yml" + src: /etc/kibana/kibana.yml # Hardcoded for testing purposes only + dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" remote_src: yes - owner: opensearch + owner: "{{ specification.opsd_user }}" group: root mode: ug=rw,o= backup: yes - name: Kibana migr | Porting kibana settings to OpenSearch Dashboards replace: - path: "{{ specification.paths.ops_conf_dir }}/opensearch_dashboards.yml" + path: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" regexp: "{{ item.1 }}" replace: "{{ item.2 }}" with_items: - { 1: 'elasticsearch', 2: 'opensearch' } - - { 1: 'kibana', 2: 'opensearchDashboards' } + - { 1: 'kibana', 2: 'opensearchdashboards' } + - { 1: 'opendistro_security', 2: 'opensearch_security' } +# OPS claims to not recognize these 3 following Kibana variables + - { 1: 'newsfeed.enabled', 2: '#newsfeed.enabled' } + - { 1: 'telemetry.optIn', 2: '#telemetry.optIn' } + - { 1: 'telemetry.enabled', 2: '#telemetry.enabled' } + +- name: Kibana migr | Create OPSD systemd service + template: + src: roles/opensearch_dashboards/templates/dashboards.service + dest: "{{ systemctl_path }}/dashboards.service" + +- name: Kibana migr | Assure Opensearch Dashboards service is started + service: + name: dashboards + state: started + enabled: yes + +- name: Kibana migr | Get all the installed dashboards plugins + command: "sudo -u {{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + register: list_plugins + +- name: Kibana migr | Show all the installed dashboards plugins + debug: + msg: "{{ list_plugins.stdout }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml index 503c7707f7..3d67b71252 100644 --- a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml @@ -77,14 +77,14 @@ - name: ODFE migr | Copy data and logs directories copy: - src: "/var/lib/elasticsearch" # Hardoced for testing purposes only + src: "/var/lib/elasticsearch" # Hardcoded for testing purposes only dest: "{{ specification.paths.ops_data }}" remote_src: yes directory_mode: yes - name: ODFE migr | Copy snapshots directories copy: - src: "/var/lib/elasticsearch-snapshots" # Hardoced for testing purposes only + src: "/var/lib/elasticsearch-snapshots" # Hardcoded for testing purposes only dest: "{{ specification.paths.ops_repo }}" remote_src: yes directory_mode: yes @@ -104,7 +104,7 @@ - name: ODFE migr | Clone JVM configuration file copy: - src: /etc/elasticsearch/jvm.options # Hardoced for testing purposes only + src: /etc/elasticsearch/jvm.options # Hardcoded for testing purposes only dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" remote_src: yes owner: root @@ -123,7 +123,7 @@ - name: ODFE migr | Clone main configuration file copy: - src: /etc/elasticsearch/elasticsearch.yml # Hardoced for testing purposes only + src: /etc/elasticsearch/elasticsearch.yml # Hardcoded for testing purposes only dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" remote_src: yes owner: root @@ -155,22 +155,22 @@ sleep: 6 -# - name: ODFE migr | Check if default admin user exists -# uri: -# url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/admin" -# method: GET -# # 404 code is used there as someone can remove admin user on its own. -# status_code: [200, 404] -# validate_certs: no -# url_username: admin -# url_password: admin -# register: admin_check_response -# until: admin_check_response is success -# retries: 60 -# delay: 1 -# run_once: true - -- name: ODFE migr | Set Opensearch admin password +- name: ODFE migr | Check if default admin user exists + uri: + url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/admin" + method: GET + # 404 code is used there as someone can remove admin user on its own. + status_code: [200, 404] + validate_certs: no + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key }}" + register: admin_check_response + until: admin_check_response is success + retries: 60 + delay: 1 + run_once: true + +- name: "ODFE migr | Set Opensearch admin password {{ specification.ops_password }}" uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" method: PATCH @@ -179,7 +179,7 @@ - op: "replace" path: "/admin" value: - password: "{{ specification.admin_password }}" + password: "{{ specification.ops_password }}" reserved: "true" backend_roles: - "admin" @@ -199,9 +199,6 @@ command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k register: ops_status -- debug: - msg: "Cluster {{ inventory_hostname }}: {{ specification.admin_password }}" - - name: ODFE migr | Show the opensearch status debug: msg: "{{ ops_status.stdout }}" diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index 6987b484a8..7576823741 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -198,16 +198,6 @@ vars: current_group_name: opensearch -- hosts: kibana - become: true - become_method: sudo - serial: 1 - tasks: - - import_role: - name: upgrade - tasks_from: kibana - when: "'kibana' in upgrade_components or upgrade_components|length == 0" - - hosts: grafana become: true become_method: sudo diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 9775112518..e7b9b914c6 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -5,6 +5,8 @@ specification: cluster_name: EpiphanyOpensearch odfe_migration: false ops_user: opensearch + opsd_user: opensearchboard + opsd_password: PASSWORD_TO_CHANGE admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: true @@ -22,6 +24,9 @@ specification: ops_repo: /var/lib/opensearch-snapshots ops_data: /var/lib/opensearch ops_logs: /var/log/opensearch + opsd_home: /usr/share/opensearch-dashboards + opsd_conf_dir: /usr/share/opensearch-dashboards/config + opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin jvm_options: Xmx: 1g opensearch_security: diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml index b750863247..07281b528d 100644 --- a/schema/common/defaults/configuration/opensearch-dashboards.yml +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -1,11 +1,11 @@ kind: configuration/opensearch-dashboards -title: "OpenSearch-dashboards" +title: "OpenSearch-Dashboards" name: default specification: - ops_dashboards_user: admin - ops_dashboards_password: PASSWORD_TO_CHANGE - ops_user: opensearch - ops_dashboards_log_dir: /var/log/opensearchdashboards - ops_dashboards_home: /usr/share/opensearch-dashboards - ops_dashboards_conf_dir: /usr/share/opensearch-dashboards/config - ops_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin + opsd_user: opensearchdboard + opsd_password: PASSWORD_TO_CHANGE + paths: + opsd_home: /usr/share/opensearch-dashboards + opsd_conf_dir: /usr/share/opensearch-dashboards/config + opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin + opsd_log_dir: /var/log/opensearchdashboards diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 79a8a29d19..7eadc7620c 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -25,6 +25,7 @@ specification: ops_repo: /var/lib/opensearch-snapshots ops_data: /var/lib/opensearch ops_logs: /var/log/opensearch + opsd_home: /usr/share/opensearch-dashboards jvm_options: Xmx: 1g opensearch_security: From d05ac8259af2db5e119fd6962bb603d80c506cd1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Feb 2022 23:52:33 +0100 Subject: [PATCH 024/157] Migration to opensearch --- .../roles/upgrade/tasks/kibana-migration.yml | 2 +- .../roles/upgrade/tasks/odfe-migration.yml | 26 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml b/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml index bd783be1ec..0710084266 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml @@ -64,7 +64,7 @@ replace: "{{ item.2 }}" with_items: - { 1: 'elasticsearch', 2: 'opensearch' } - - { 1: 'kibana', 2: 'opensearchdashboards' } + - { 1: '/kibana', 2: '/opensearchdashboards' } - { 1: 'opendistro_security', 2: 'opensearch_security' } # OPS claims to not recognize these 3 following Kibana variables - { 1: 'newsfeed.enabled', 2: '#newsfeed.enabled' } diff --git a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml index 3d67b71252..37656a6117 100644 --- a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml @@ -170,7 +170,7 @@ delay: 1 run_once: true -- name: "ODFE migr | Set Opensearch admin password {{ specification.ops_password }}" +- name: ODFE migr | Set Opensearch admin passwor uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" method: PATCH @@ -193,7 +193,29 @@ retries: 5 delay: 1 run_once: true - # when: admin_check_response.status == 200 + when: admin_check_response.status == 200 + +- name: ODFE migr | Set kibanaserver user password + uri: + url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" + method: PATCH + status_code: [200] + body: + - op: "replace" + path: "/kibanaserver" + value: + password: "{{ specification.kibanaserver_password }}" + reserved: "true" + description: "kibanaserver user" + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key }}" + body_format: json + validate_certs: no + register: uri_response + until: uri_response is success + retries: 5 + delay: 1 + run_once: true - name: ODFE migr | Check the opensearch status command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k From d8bac1b16d014f0f45a64a84e54b9f00937772a4 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 9 Feb 2022 15:30:20 +0100 Subject: [PATCH 025/157] Migration to opensearch - docs update --- docs/home/howto/UPGRADE.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index ed13961377..43e0fe71ae 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -253,6 +253,15 @@ then start with the rest **one by one**. More detailed information about ZooKeeper you can find in [ZooKeeper documentation](https://cwiki.apache.org/confluence/display/ZOOKEEPER). +## Migration from Open Distro for Elasticsearch & Kibana to OpenSearch an OpenSearch Dashboards + +--- +**NOTE** + +Make sure you have a backup before proceeding to migration steps described below ! + +--- + ## Open Distro for Elasticsearch upgrade --- From 011470dd675df7044ceea154fd49ff8d9f532d5f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 9 Feb 2022 15:30:38 +0100 Subject: [PATCH 026/157] Migration to opensearch --- .../roles/upgrade/tasks/opensearch-01.yml | 4 +- .../roles/upgrade/tasks/opensearch-02.yml | 4 +- .../opensearch/migrate-from-demo-certs-01.yml | 2 +- .../opensearch/migrate-from-demo-certs-02.yml | 2 +- .../migrate-from-demo-certs-non-clustered.yml | 2 +- .../migrate-kibana.yml} | 2 +- .../migrate-odfe.yml} | 63 ++++++++++++++----- .../opensearch/upgrade-opensearch-02.yml | 16 ++--- .../opensearch/utils/assert-api-access.yml | 4 +- .../utils/assert-cert-files-exist.yml | 6 +- .../utils/create-dual-cert-file.yml | 4 +- .../utils/enable-shard-allocation.yml | 2 +- .../opensearch/utils/get-cluster-health.yml | 2 +- .../utils/get-config-from-files.yml | 6 +- .../prepare-cluster-for-node-restart.yml | 6 +- .../tasks/opensearch/utils/restart-node.yml | 6 +- .../utils/save-initial-cluster-status.yml | 4 +- .../opensearch/utils/test-api-access.yml | 2 +- .../utils/wait-for-cluster-status.yml | 2 +- .../utils/wait-for-node-to-join.yml | 2 +- .../utils/wait-for-shard-allocation.yml | 2 +- 21 files changed, 86 insertions(+), 57 deletions(-) rename ansible/playbooks/roles/upgrade/tasks/{kibana-migration.yml => opensearch/migrate-kibana.yml} (97%) rename ansible/playbooks/roles/upgrade/tasks/{odfe-migration.yml => opensearch/migrate-odfe.yml} (76%) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index 8f0d5afcec..3414cfbd3a 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -77,14 +77,14 @@ - include_role: name: upgrade - tasks_from: odfe-migration + tasks_from: opensearch/migrate-odfe when: ops_vars.specification.odfe_migration vars: current_group_name: logging - include_role: name: upgrade - tasks_from: kibana-migration + tasks_from: opensearch/migrate-kibana when: ops_vars.specification.odfe_migration vars: current_group_name: logging diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml index 1a34acdc31..06107ecde1 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml @@ -1,6 +1,6 @@ --- # If state file exists, it means upgrade has been started by the previous play and should be continued -- name: ODFE | Check if upgrade state file exists +- name: OPS | Check if upgrade state file exists stat: path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false @@ -8,6 +8,6 @@ get_mime: false register: stat_upgrade_state_file -- name: ODFE | Upgrade Elasticsearch and ODFE plugins (part 2/2) +- name: OPS | Upgrade Elasticsearch and ODFE plugins (part 2/2) include_tasks: opensearch/upgrade-opensearch-02.yml when: stat_upgrade_state_file.stat.exists diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml index 4d7b11a221..4ea4e8d24f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml @@ -22,7 +22,7 @@ - "{{ (certificates.dirs.certs, certificates.files.root_ca.cert.filename) | path_join }}" target_path: "{{ (certificates.dirs.certs, opensearch.certs_migration.dual_root_ca.filename) | path_join }}" -- name: ODFE | Load /etc/elasticsearch/elasticsearch.yml +- name: OPS | Load /etc/elasticsearch/elasticsearch.yml slurp: src: /etc/elasticsearch/elasticsearch.yml register: _elasticsearch_yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml index c93fb4c028..b6cccd4a6e 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml @@ -31,7 +31,7 @@ # Patch elasticsearch.yml to use Epiphany node cert (all hosts) - - name: ODFE | Load /etc/elasticsearch/elasticsearch.yml + - name: OPS | Load /etc/elasticsearch/elasticsearch.yml slurp: src: /etc/elasticsearch/elasticsearch.yml register: _elasticsearch_yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml index 273c8508c1..b3d376838d 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml @@ -1,5 +1,5 @@ --- -- name: ODFE | Load /etc/elasticsearch/elasticsearch.yml +- name: OPS | Load /etc/elasticsearch/elasticsearch.yml slurp: src: /etc/elasticsearch/elasticsearch.yml register: _elasticsearch_yml diff --git a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml similarity index 97% rename from ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 0710084266..a6196b82ee 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kibana-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -49,7 +49,7 @@ - name: Kibana migr | Clone kibana settings copy: - src: /etc/kibana/kibana.yml # Hardcoded for testing purposes only + src: /etc/kibana/kibana.yml dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" remote_src: yes owner: "{{ specification.opsd_user }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml similarity index 76% rename from ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml rename to ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 37656a6117..f78a32fa4f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/odfe-migration.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -52,8 +52,8 @@ vars: es_api: cert_type: Epiphany - cert_path: &epi_cert_path "{{ (certificates.dirs.certs, certificates.files.admin.cert) | path_join }}" - key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key) | path_join }}" + cert_path: "{{ (certificates.dirs.certs, certificates.files.admin.cert) | path_join }}" + key_path: "{{ (certificates.dirs.certs, certificates.files.admin.key) | path_join }}" url: https://{{ es_host }}:{{ es_http_port }} fail_msg: API access test failed. @@ -64,30 +64,26 @@ state: stopped register: elasticsearch_state -- name: ODFE migr | Include vars 1/2 for Opensearch binaries instalaltion +- name: ODFE migr | Include defaults Opensearch binaries instalaltion include_vars: file: roles/opensearch/defaults/main.yml -- name: ODFE migr | Include vars 2/2 for Opensearch binaries instalaltion +- name: ODFE migr | Include vars for Opensearch binaries instalaltion include_vars: file: roles/opensearch/vars/main.yml - name: ODFE migr | Inastall Opensearch binaries include_tasks: roles/opensearch/tasks/install-ops.yml -- name: ODFE migr | Copy data and logs directories +- name: ODFE migr | Copy ES directories to OPS directories copy: - src: "/var/lib/elasticsearch" # Hardcoded for testing purposes only - dest: "{{ specification.paths.ops_data }}" - remote_src: yes - directory_mode: yes - -- name: ODFE migr | Copy snapshots directories - copy: - src: "/var/lib/elasticsearch-snapshots" # Hardcoded for testing purposes only - dest: "{{ specification.paths.ops_repo }}" + src: "{{ item.1 }}" + dest: "{{ item.2 }}" remote_src: yes directory_mode: yes + with_items: + - { 1: "/var/lib/elasticsearch-snapshots", 2: "{{ specification.paths.ops_repo }}" } + - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.ops_data }}" } - name: ODFE migr | Prepare a list of certs and keys to OPS directories find: @@ -104,7 +100,7 @@ - name: ODFE migr | Clone JVM configuration file copy: - src: /etc/elasticsearch/jvm.options # Hardcoded for testing purposes only + src: /etc/elasticsearch/jvm.options dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" remote_src: yes owner: root @@ -123,7 +119,7 @@ - name: ODFE migr | Clone main configuration file copy: - src: /etc/elasticsearch/elasticsearch.yml # Hardcoded for testing purposes only + src: /etc/elasticsearch/elasticsearch.yml dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" remote_src: yes owner: root @@ -138,7 +134,7 @@ replace: "{{ item.2 }}" with_items: - { 1: 'elasticsearch', 2: 'opensearch' } - # - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } + - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } - { 1: 'opendistro_security.', 2: 'plugins.security.' } - name: ODFE migr | Start opensearch service @@ -195,6 +191,21 @@ run_once: true when: admin_check_response.status == 200 +- name: ODFE migr | Check if kibanaserver user exists + uri: + url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/kibanaserver" + method: GET + # 404 code is used there as someone can remove admin user on its own. + status_code: [200, 404] + validate_certs: no + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key }}" + register: kibanaserver_check_response + until: kibanaserver_check_response is success + retries: 60 + delay: 1 + run_once: true + - name: ODFE migr | Set kibanaserver user password uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" @@ -216,6 +227,7 @@ retries: 5 delay: 1 run_once: true + when: kibanaserver_check_response.status == 200 - name: ODFE migr | Check the opensearch status command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k @@ -225,3 +237,20 @@ debug: msg: "{{ ops_status.stdout }}" failed_when: "'number_of_nodes' not in ops_status.stdout" + +- name: ODFE migr | Reenable shard allocation for the cluster + include_tasks: + file: opensearch/utils/enable-shard-allocation.yml + apply: + delegate_to: "{{ target_inventory_hostname }}" + delegate_facts: true + loop: "{{ ansible_play_hosts_all }}" + loop_control: + loop_var: target_inventory_hostname + vars: + es_api: + cert_type: Epiphany + cert_path: "{{ (certificates.dirs.certs, certificates.files.admin.cert) | path_join }}" + key_path: "{{ (certificates.dirs.certs, certificates.files.admin.key) | path_join }}" + url: https://{{ es_host }}:{{ es_http_port }} + fail_msg: API access test failed. diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml index 1eea82b5d8..3730200eab 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml @@ -8,26 +8,26 @@ - es_api.cert_type == 'demo' - es_clustered # rolling upgrade only for clustered installation -- name: ODFE | Print API facts +- name: OPS | Print API facts debug: var: es_api tags: [ never, debug ] # only runs when debug or never tag requested -- name: ODFE | Prepare cluster for rolling upgrade +- name: OPS | Prepare cluster for rolling upgrade include_tasks: opensearch/utils/prepare-cluster-for-node-restart.yml when: es_clustered -- name: ODFE | Stop elasticsearch service +- name: OPS | Stop elasticsearch service systemd: name: opensearch state: stopped -- name: ODFE | Include Elasticsearch installation tasks +- name: OPS | Include Elasticsearch installation tasks include_role: name: opensearch tasks_from: install-ops.yml -- name: ODFE | Include Elasticsearch configuration tasks +- name: OPS | Include Elasticsearch configuration tasks include_role: name: opensearch tasks_from: configure-ops.yml @@ -65,7 +65,7 @@ existing_es_config: "{{ _old | combine(_updated_existing_config) }}" -- name: ODFE | Include upgrade plugins tasks +- name: OPS | Include upgrade plugins tasks include_tasks: opensearch/upgrade-plugins.yml # Restart elasticsearch service (unconditionally to ensure this task is not skipped in case of rerunning after interruption) @@ -97,13 +97,13 @@ initial_status: "{{ (slurp_upgrade_state_file.content | b64decode | from_json)['status'] }}" expected_status: "{{ [ initial_status, 'green'] | unique }}" -- name: ODFE | Remove dual root CA temporary file +- name: OPS | Remove dual root CA temporary file file: path: "{{ (certificates.dirs.certs, opensearch.certs_migration.dual_root_ca.filename) | path_join }}" state: absent when: es_api.cert_type == 'Epiphany' -- name: ODFE | Remove upgrade state file +- name: OPS | Remove upgrade state file file: path: "{{ opensearch.upgrade_state_file_path }}" state: absent diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml index b9d36e1d9f..9ae45b7ae8 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml @@ -1,5 +1,5 @@ --- -- name: ODFE | Assert input parameters +- name: OPS | Assert input parameters assert: that: - es_api.cert_path is defined @@ -13,7 +13,7 @@ # Sets 'test_api_access' - include_tasks: test-api-access.yml -- name: ODFE | Assert API access +- name: OPS | Assert API access assert: that: test_api_access.status == 200 fail_msg: diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml index a4ad4f4f60..b8dd104935 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml @@ -1,5 +1,5 @@ --- -- name: ODFE | Assert input parameters +- name: OPS | Assert input parameters assert: that: - es_api.cert_path is defined @@ -8,7 +8,7 @@ - es_api.key_path is defined quiet: true -- name: ODFE | Get info on files +- name: OPS | Get info on files stat: path: "{{ item }}" get_attributes: false @@ -20,7 +20,7 @@ - "{{ es_api.key_path }}" # Specific case for custom certificates (we don't know the paths so they have to be specified manually) -- name: ODFE | Assert files exist +- name: OPS | Assert files exist assert: that: stat_result.stat.exists fail_msg: "{{ _custom_cert_fail_msg if (es_api.cert_type == 'custom') else _common_fail_msg }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml index d655dc1887..40877c305c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml @@ -3,13 +3,13 @@ # - certs_to_concatenate # - target_path -- name: ODFE | Read certificates to concatenate +- name: OPS | Read certificates to concatenate slurp: src: "{{ item }}" register: _files loop: "{{ certs_to_concatenate }}" -- name: ODFE | Create dual root CA transitional file for migration +- name: OPS | Create dual root CA transitional file for migration copy: dest: "{{ target_path }}" content: "{{ _files.results | map(attribute='content') | map('b64decode') | join('') }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml index 8394d69fa2..52ec5744c2 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml @@ -4,7 +4,7 @@ # - es_api.cert_path # - es_api.key_path -- name: ODFE | Enable shard allocation for the cluster +- name: OPS | Enable shard allocation for the cluster uri: url: "{{ es_api.url }}/_cluster/settings" method: PUT diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml index 9c0079f468..12ad125baf 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml @@ -4,7 +4,7 @@ # - es_api.cert_path # - es_api.key_path -- name: ODFE | Get cluster health +- name: OPS | Get cluster health uri: url: "{{ es_api.url }}/_cluster/health" method: GET diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml index 814087368c..0aae6a6f97 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml @@ -1,17 +1,17 @@ --- # Sets facts on existing configuration -- name: ODFE | Load /etc/elasticsearch/elasticsearch.yml +- name: OPS | Load /etc/elasticsearch/elasticsearch.yml slurp: src: /etc/elasticsearch/elasticsearch.yml register: _elasticsearch_yml -- name: ODFE | Get Xmx value from /etc/elasticsearch/jvm.options +- name: OPS | Get Xmx value from /etc/elasticsearch/jvm.options command: grep -oP '(?<=^-Xmx)\d+[kKmMgG]?' /etc/elasticsearch/jvm.options register: _grep_xmx changed_when: false -- name: ODFE | Set existing configuration facts +- name: OPS | Set existing configuration facts set_fact: existing_config: main: "{{ _elasticsearch_yml.content | b64decode | from_yaml }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml index 34bebc59cb..d28bf90192 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml @@ -16,7 +16,7 @@ body_format: json block: # It's safe to run this task many times regardless of the state - - name: ODFE | Disable shard allocation for the cluster + - name: OPS | Disable shard allocation for the cluster uri: url: "{{ es_api.url }}/_cluster/settings" method: PUT @@ -35,7 +35,7 @@ # In epicli 0.7.x there is ES 7.3.2 but this step is optional. - name: Handle flush failure block: - - name: ODFE | Perform a synced flush (optional step) + - name: OPS | Perform a synced flush (optional step) uri: url: "{{ es_api.url }}/_flush" method: POST @@ -46,7 +46,7 @@ retries: 120 delay: 1 rescue: - - name: ODFE | Print warning + - name: OPS | Print warning debug: msg: - "WARNING: flush command failed" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml index 772d4e0390..dd88fe87bb 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml @@ -10,18 +10,18 @@ # - daemon_reload # - skip_waiting_for_status -- name: ODFE | Restart elasticsearch service +- name: OPS | Restart elasticsearch service systemd: name: opensearch state: restarted daemon_reload: "{{ daemon_reload | default(omit) }}" -- name: ODFE | Wait for Elasticsearch transport port to become available +- name: OPS | Wait for Elasticsearch transport port to become available wait_for: port: "{{ es_transport_port }}" host: "{{ hostvars[target_inventory_hostname].es_host }}" -- name: ODFE | Wait for Elasticsearch http port to become available +- name: OPS | Wait for Elasticsearch http port to become available wait_for: port: "{{ es_http_port }}" host: "{{ hostvars[target_inventory_hostname].es_host }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml index 8c37c863f4..87d01c58a6 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml @@ -1,5 +1,5 @@ --- -- name: ODFE | Get size of upgrade state file +- name: OPS | Get size of upgrade state file stat: path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false @@ -12,7 +12,7 @@ block: - include_tasks: get-cluster-health.yml - - name: ODFE | Save cluster health to upgrade state file + - name: OPS | Save cluster health to upgrade state file copy: content: "{{ cluster_health.json }}" dest: "{{ opensearch.upgrade_state_file_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml index 8d8495e525..0a620baa1e 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml @@ -5,7 +5,7 @@ # - es_api.key_path # - es_api.url -- name: ODFE | Test API access using {{ es_api.cert_type }} certificate +- name: OPS | Test API access using {{ es_api.cert_type }} certificate uri: client_cert: "{{ es_api.cert_path }}" client_key: "{{ es_api.key_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml index 496198a4a0..ef31a44613 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml @@ -5,7 +5,7 @@ # - es_api.key_path # - expected_status (type: list, e.g. [ 'green', 'yellow' ]) -- name: ODFE | Wait for '{{ expected_status | join("' or '") }}' cluster health status +- name: OPS | Wait for '{{ expected_status | join("' or '") }}' cluster health status uri: url: "{{ es_api.url }}/_cluster/health" client_cert: "{{ es_api.cert_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml index fcb039654c..1c3f1ae4f6 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml @@ -6,7 +6,7 @@ # - target_inventory_hostname # - hostvars[target_inventory_hostname].es_node_name -- name: ODFE | Wait for Elasticsearch node to join the cluster +- name: OPS | Wait for Elasticsearch node to join the cluster uri: url: "{{ es_api.url }}/_cat/nodes?h=name" method: GET diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml index 0175d1b2d5..bb294c0b1d 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml @@ -4,7 +4,7 @@ # - es_api.cert_path # - es_api.key_path -- name: ODFE | Wait for the cluster to finish shard allocation +- name: OPS | Wait for the cluster to finish shard allocation uri: url: "{{ es_api.url }}/_cluster/health" method: GET From 6b42ce738f5cdac0138c4d8e1fe292efd2a805b0 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 11 Feb 2022 09:23:00 +0100 Subject: [PATCH 027/157] Enable a cluster migration --- .../tasks/opensearch/migrate-odfe-serial.yml | 94 ++++++++++++++++ .../upgrade/tasks/opensearch/migrate-odfe.yml | 101 ++---------------- 2 files changed, 102 insertions(+), 93 deletions(-) create mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml new file mode 100644 index 0000000000..308981f830 --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -0,0 +1,94 @@ +--- +# Below tasks need to be run in serial +- name: ODFE migr | Stop elasticsearch service + systemd: + name: elasticsearch + enabled: yes + state: stopped + register: elasticsearch_state + +- name: ODFE migr | Include defaults Opensearch binaries instalaltion + include_vars: + file: roles/opensearch/defaults/main.yml + +- name: ODFE migr | Include vars for Opensearch binaries instalaltion + include_vars: + file: roles/opensearch/vars/main.yml + +- name: ODFE migr | Inastall Opensearch binaries + include_tasks: roles/opensearch/tasks/install-ops.yml + +- name: ODFE migr | Copy ES directories to OPS directories + copy: + src: "{{ item.1 }}" + dest: "{{ item.2 }}" + remote_src: yes + directory_mode: yes + with_items: + - { 1: "/var/lib/elasticsearch-snapshots", 2: "{{ specification.paths.ops_repo }}" } + - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.ops_data }}" } + +- name: ODFE migr | Prepare a list of ESS certs and keys + find: + paths: "/etc/elasticsearch/" + patterns: "*pem" + register: pem_files + +- name: ODFE migr | Copy a list of certs and keys to OPS directories + copy: + src: "{{ item.path }}" + dest: "{{ specification.paths.ops_conf_dir }}/" + remote_src: yes + with_items: "{{ pem_files.files }}" + +- name: ODFE migr | Clone JVM configuration file + copy: + src: /etc/elasticsearch/jvm.options + dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" + remote_src: yes + owner: root + group: opensearch + mode: ug=rw,o= + backup: yes + +- name: ODFE migr | Update JVM configuration file + replace: + path: "{{ specification.paths.ops_conf_dir }}/jvm.options" + regexp: "{{ item.1 }}" + replace: "{{ item.2 }}" + with_items: + - { 1: 'elasticsearch', 2: 'opensearch' } + - { 1: '\${ES_TMPDIR}', 2: '/tmp' } + +- name: ODFE migr | Clone main configuration file + copy: + src: /etc/elasticsearch/elasticsearch.yml + dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" + remote_src: yes + owner: root + group: opensearch + mode: ug=rw,o= + backup: yes + +- name: ODFE migr | Update main configuration file + replace: + path: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" + regexp: "{{ item.1 }}" + replace: "{{ item.2 }}" + with_items: + - { 1: 'elasticsearch', 2: 'opensearch' } + - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } + - { 1: 'opendistro_security.', 2: 'plugins.security.' } + +- name: ODFE migr | Start opensearch service + systemd: + name: opensearch + state: started + enabled: yes + register: restart_opensearch + +- name: ODFE migr | Wait for opensearch to startup + wait_for: + port: 9200 + host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" + sleep: 6 diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index f78a32fa4f..414045bceb 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -57,99 +57,14 @@ url: https://{{ es_host }}:{{ es_http_port }} fail_msg: API access test failed. -- name: ODFE migr | Stop elasticsearch service - systemd: - name: elasticsearch - enabled: yes - state: stopped - register: elasticsearch_state - -- name: ODFE migr | Include defaults Opensearch binaries instalaltion - include_vars: - file: roles/opensearch/defaults/main.yml - -- name: ODFE migr | Include vars for Opensearch binaries instalaltion - include_vars: - file: roles/opensearch/vars/main.yml - -- name: ODFE migr | Inastall Opensearch binaries - include_tasks: roles/opensearch/tasks/install-ops.yml - -- name: ODFE migr | Copy ES directories to OPS directories - copy: - src: "{{ item.1 }}" - dest: "{{ item.2 }}" - remote_src: yes - directory_mode: yes - with_items: - - { 1: "/var/lib/elasticsearch-snapshots", 2: "{{ specification.paths.ops_repo }}" } - - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.ops_data }}" } - -- name: ODFE migr | Prepare a list of certs and keys to OPS directories - find: - paths: "/etc/elasticsearch/" - patterns: "*pem" - register: pem_files - -- name: ODFE migr | Copy a list of certs and keys to OPS directories - copy: - src: "{{ item.path }}" - dest: "{{ specification.paths.ops_conf_dir }}/" - remote_src: yes - with_items: "{{ pem_files.files }}" - -- name: ODFE migr | Clone JVM configuration file - copy: - src: /etc/elasticsearch/jvm.options - dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" - remote_src: yes - owner: root - group: opensearch - mode: ug=rw,o= - backup: yes - -- name: ODFE migr | Update JVM configuration file - replace: - path: "{{ specification.paths.ops_conf_dir }}/jvm.options" - regexp: "{{ item.1 }}" - replace: "{{ item.2 }}" - with_items: - - { 1: 'elasticsearch', 2: 'opensearch' } - - { 1: '\${ES_TMPDIR}', 2: '/tmp' } - -- name: ODFE migr | Clone main configuration file - copy: - src: /etc/elasticsearch/elasticsearch.yml - dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" - remote_src: yes - owner: root - group: opensearch - mode: ug=rw,o= - backup: yes - -- name: ODFE migr | Update main configuration file - replace: - path: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" - regexp: "{{ item.1 }}" - replace: "{{ item.2 }}" - with_items: - - { 1: 'elasticsearch', 2: 'opensearch' } - - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } - - { 1: 'opendistro_security.', 2: 'plugins.security.' } - -- name: ODFE migr | Start opensearch service - systemd: - name: opensearch - state: started - enabled: yes - register: restart_opensearch - -- name: ODFE migr | Wait for opensearch to startup - wait_for: - port: 9200 - host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" - sleep: 6 - +- include_tasks: + file: opensearch/migrate-odfe-serial.yml + apply: + delegate_to: "{{ target_hostname }}" + delegate_facts: true + loop: "{{ groups.filebeat | default([]) }}" + loop_control: + loop_var: target_hostname - name: ODFE migr | Check if default admin user exists uri: From cd8932bf0c1ef25a3806c444420fec9b457c4d60 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 11 Feb 2022 14:11:16 +0100 Subject: [PATCH 028/157] Cluster migration --- .../playbooks/roles/upgrade/tasks/kafka.yml | 55 ------------------- .../tasks/opensearch/migrate-odfe-serial.yml | 6 +- .../upgrade/tasks/opensearch/migrate-odfe.yml | 21 ++++--- 3 files changed, 15 insertions(+), 67 deletions(-) delete mode 100644 ansible/playbooks/roles/upgrade/tasks/kafka.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/kafka.yml b/ansible/playbooks/roles/upgrade/tasks/kafka.yml deleted file mode 100644 index 68798a0953..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/kafka.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- -- name: Include defaults from kafka role - include_vars: - file: roles/kafka/defaults/main.yml - name: kafka_defaults - -- name: Include pre-flight checks - include_tasks: kafka/preflight-check.yml - -- name: Get installed Kafka version - shell: >- - set -o pipefail && - /opt/kafka/bin/kafka-server-start.sh --version | grep Commit | grep -oP '^\d+\.\d+\.\d+' - register: result - -- name: Set common facts - set_fact: - kafka_version: - old: "{{ result.stdout }}" - new: "{{ kafka_defaults.kafka_version }}" - scala_version: "{{ kafka_defaults.scala_version }}" - kafka_bin_filename: "{{ kafka_defaults.kafka_bin_filename }}" - -- name: Check for upgrade flag file - stat: - path: "{{ lock_file }}" - register: lock_file_status - -- name: Include upgrade tasks - block: - - name: Create upgrade flag file - file: - path: "{{ lock_file }}" - state: touch - - - name: Stop Kafka service - service: - name: kafka - state: stopped - - - name: Include update Kafka properties tasks - include_tasks: kafka/update-properties.yml - - - name: Include Kafka upgrade tasks - include_tasks: kafka/install-upgrade.yml - - - name: Include set Kafka version tasks - include_tasks: kafka/set-updated-version.yml - when: - - lock_file_status.stat.exists or kafka_version.old is version( kafka_version.new, '<' ) - -- name: Remove Kafka upgrade flag file - file: - path: "{{ lock_file }}" - state: absent diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index 308981f830..d343a98bb4 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -3,15 +3,15 @@ - name: ODFE migr | Stop elasticsearch service systemd: name: elasticsearch - enabled: yes + enabled: no state: stopped register: elasticsearch_state -- name: ODFE migr | Include defaults Opensearch binaries instalaltion +- name: ODFE migr | Include defaults Opensearch binaries installation include_vars: file: roles/opensearch/defaults/main.yml -- name: ODFE migr | Include vars for Opensearch binaries instalaltion +- name: ODFE migr | Include vars for Opensearch binaries installation include_vars: file: roles/opensearch/vars/main.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 414045bceb..508794b78f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -7,17 +7,18 @@ - name: OPS | Print elasticsearch ond opensearch versions debug: msg: - - "Elasticsearch version curently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" + - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - "Opensearch version to be installed: {{ ops_defaults.versions[ansible_os_family].ops_version }}" -- name: ODFE migr | Ensure elasticsearch service is running +- name: ODFE migr | Ensure elasticsearch cluster is up and running systemd: name: elasticsearch enabled: yes - state: started + state: restarted register: elasticsearch_state -- include_tasks: opensearch/utils/get-config-from-files.yml # Sets 'existing_config' fact +- name: ODFE migr | Set existing_config facts + include_tasks: opensearch/utils/get-config-from-files.yml - name: ODFE migr | Set common facts set_fact: @@ -28,7 +29,7 @@ es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" es_node_name: "{{ existing_config.main['node.name'] }}" -- name: ODFE migr | Assure Elastisearch files location will be used +- name: ODFE migr | Assure Elasticsearch files location will be used in following tasks set_fact: certificates: dirs: @@ -46,7 +47,7 @@ apply: delegate_to: "{{ target_inventory_hostname }}" delegate_facts: true - loop: "{{ ansible_play_hosts_all }}" + loop: "{{ groups.logging | default([]) }}" loop_control: loop_var: target_inventory_hostname vars: @@ -57,14 +58,16 @@ url: https://{{ es_host }}:{{ es_http_port }} fail_msg: API access test failed. -- include_tasks: +- name: ODFE migr | Run core migration tasks individually on each node + include_tasks: file: opensearch/migrate-odfe-serial.yml apply: delegate_to: "{{ target_hostname }}" delegate_facts: true - loop: "{{ groups.filebeat | default([]) }}" + loop: "{{ groups.logging | default([]) }}" loop_control: loop_var: target_hostname + run_once: true - name: ODFE migr | Check if default admin user exists uri: @@ -81,7 +84,7 @@ delay: 1 run_once: true -- name: ODFE migr | Set Opensearch admin passwor +- name: ODFE migr | Set Opensearch admin password uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" method: PATCH From 57479eed019998662fe1c53bb0774fb2b29801d3 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 11 Feb 2022 15:08:05 +0100 Subject: [PATCH 029/157] docs update --- docs/home/HOWTO.md | 1 + docs/home/howto/UPGRADE.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 48c61cb70c..54fb4015c8 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -59,6 +59,7 @@ - [Run apply after upgrade](./howto/UPGRADE.md#run-apply-after-upgrade) - [Kubernetes applications](./howto/UPGRADE.md#kubernetes-applications) - [Kafka upgrade](./howto/UPGRADE.md#how-to-upgrade-kafka) + - [Migration from Open Distro for Elasticsearch to OpenSearch](./howto/UPGRADE.md#open-distro-for-elasticsearch-upgrade) - [Open Distro for Elasticsearch upgrade](./howto/UPGRADE.md#open-distro-for-elasticsearch-upgrade) - [Node exporter upgrade](./howto/UPGRADE.md#node-exporter-upgrade) - [RabbitMQ upgrade](./howto/UPGRADE.md#rabbitmq-upgrade) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 43e0fe71ae..ee8ee72ea6 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -261,6 +261,27 @@ in [ZooKeeper documentation](https://cwiki.apache.org/confluence/display/ZOOKEE Make sure you have a backup before proceeding to migration steps described below ! --- +Following the decision of Elastic NV on ceasing open source options available for Elasticsearch and Kibana and releasing them under the Elastic license (more info [here](https://github.com/epiphany-platform/epiphany/issues/2870)) Epiphany team decided to implement a mechanism of autoamtic migration from Elasticsearch 7.10.2 to Opensearch 1.2.4. + +The migration can be fired by placing `odfe_migration` switch in your manifest file: +```yaml +[..] +--- +kind: configuration/logging +title: Logging Config +[..] +specification: + [..] + odfe_migration: true # <<------- + [..] +``` +and running the `upgrade` command against the logging component of your Epiphany installation, together with a `-f` option: +``` +epicli upgrade -b / --upgrade-components "logging" -f /.yml +``` +The default value of the `odfe_migration` parameter is set to _false_. + +
## Open Distro for Elasticsearch upgrade From 52d844e220c183d26cbd53d77c6d6ca64def101b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 11 Feb 2022 15:10:17 +0100 Subject: [PATCH 030/157] docs update - link correction --- docs/home/HOWTO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 54fb4015c8..38ec159c23 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -59,7 +59,7 @@ - [Run apply after upgrade](./howto/UPGRADE.md#run-apply-after-upgrade) - [Kubernetes applications](./howto/UPGRADE.md#kubernetes-applications) - [Kafka upgrade](./howto/UPGRADE.md#how-to-upgrade-kafka) - - [Migration from Open Distro for Elasticsearch to OpenSearch](./howto/UPGRADE.md#open-distro-for-elasticsearch-upgrade) + - [Migration from Open Distro for Elasticsearch to OpenSearch](./howto/UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-an-opensearch-dashboards) - [Open Distro for Elasticsearch upgrade](./howto/UPGRADE.md#open-distro-for-elasticsearch-upgrade) - [Node exporter upgrade](./howto/UPGRADE.md#node-exporter-upgrade) - [RabbitMQ upgrade](./howto/UPGRADE.md#rabbitmq-upgrade) From 3a3b0d164f9e985ae1069369b28f653931f07bde Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 11 Feb 2022 15:12:29 +0100 Subject: [PATCH 031/157] docs typo corrections --- docs/home/HOWTO.md | 2 +- docs/home/howto/UPGRADE.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index 38ec159c23..a8b546730b 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -59,7 +59,7 @@ - [Run apply after upgrade](./howto/UPGRADE.md#run-apply-after-upgrade) - [Kubernetes applications](./howto/UPGRADE.md#kubernetes-applications) - [Kafka upgrade](./howto/UPGRADE.md#how-to-upgrade-kafka) - - [Migration from Open Distro for Elasticsearch to OpenSearch](./howto/UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-an-opensearch-dashboards) + - [Migration from Open Distro for Elasticsearch to OpenSearch](./howto/UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) - [Open Distro for Elasticsearch upgrade](./howto/UPGRADE.md#open-distro-for-elasticsearch-upgrade) - [Node exporter upgrade](./howto/UPGRADE.md#node-exporter-upgrade) - [RabbitMQ upgrade](./howto/UPGRADE.md#rabbitmq-upgrade) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index ee8ee72ea6..3d84003ade 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -253,7 +253,7 @@ then start with the rest **one by one**. More detailed information about ZooKeeper you can find in [ZooKeeper documentation](https://cwiki.apache.org/confluence/display/ZOOKEEPER). -## Migration from Open Distro for Elasticsearch & Kibana to OpenSearch an OpenSearch Dashboards +## Migration from Open Distro for Elasticsearch & Kibana to OpenSearch and OpenSearch Dashboards --- **NOTE** @@ -261,7 +261,7 @@ in [ZooKeeper documentation](https://cwiki.apache.org/confluence/display/ZOOKEE Make sure you have a backup before proceeding to migration steps described below ! --- -Following the decision of Elastic NV on ceasing open source options available for Elasticsearch and Kibana and releasing them under the Elastic license (more info [here](https://github.com/epiphany-platform/epiphany/issues/2870)) Epiphany team decided to implement a mechanism of autoamtic migration from Elasticsearch 7.10.2 to Opensearch 1.2.4. +Following the decision of Elastic NV on ceasing open source options available for Elasticsearch and Kibana and releasing them under the Elastic license (more info [here](https://github.com/epiphany-platform/epiphany/issues/2870)) Epiphany team decided to implement a mechanism of automatic migration from Elasticsearch 7.10.2 to Opensearch 1.2.4. The migration can be fired by placing `odfe_migration` switch in your manifest file: ```yaml From 9c627ec1891ee4daf5c6cb6fb32a58256c71cff9 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 11 Feb 2022 15:17:37 +0100 Subject: [PATCH 032/157] docs suplement --- docs/home/howto/UPGRADE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 3d84003ade..3583d7a9e8 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -281,6 +281,8 @@ epicli upgrade -b / --upgrade-components "logging" ``` The default value of the `odfe_migration` parameter is set to _false_. +All described below remarks related to TLS certificates of the Open Distro upgrade stay valid. You should plan and test all your upgrade activities before proceeding on the production. +
## Open Distro for Elasticsearch upgrade From 44a4d034621eaf89ba0195c5b7add8fc51ec2d3d Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 14 Feb 2022 10:46:57 +0100 Subject: [PATCH 033/157] Resstore of accidetally removed file --- .../playbooks/roles/upgrade/tasks/kafka.yml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 ansible/playbooks/roles/upgrade/tasks/kafka.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/kafka.yml b/ansible/playbooks/roles/upgrade/tasks/kafka.yml new file mode 100644 index 0000000000..68798a0953 --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/kafka.yml @@ -0,0 +1,55 @@ +--- +- name: Include defaults from kafka role + include_vars: + file: roles/kafka/defaults/main.yml + name: kafka_defaults + +- name: Include pre-flight checks + include_tasks: kafka/preflight-check.yml + +- name: Get installed Kafka version + shell: >- + set -o pipefail && + /opt/kafka/bin/kafka-server-start.sh --version | grep Commit | grep -oP '^\d+\.\d+\.\d+' + register: result + +- name: Set common facts + set_fact: + kafka_version: + old: "{{ result.stdout }}" + new: "{{ kafka_defaults.kafka_version }}" + scala_version: "{{ kafka_defaults.scala_version }}" + kafka_bin_filename: "{{ kafka_defaults.kafka_bin_filename }}" + +- name: Check for upgrade flag file + stat: + path: "{{ lock_file }}" + register: lock_file_status + +- name: Include upgrade tasks + block: + - name: Create upgrade flag file + file: + path: "{{ lock_file }}" + state: touch + + - name: Stop Kafka service + service: + name: kafka + state: stopped + + - name: Include update Kafka properties tasks + include_tasks: kafka/update-properties.yml + + - name: Include Kafka upgrade tasks + include_tasks: kafka/install-upgrade.yml + + - name: Include set Kafka version tasks + include_tasks: kafka/set-updated-version.yml + when: + - lock_file_status.stat.exists or kafka_version.old is version( kafka_version.new, '<' ) + +- name: Remove Kafka upgrade flag file + file: + path: "{{ lock_file }}" + state: absent From c97c1d240406b04ab6539b41720905ab97643fcd Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 14 Feb 2022 10:53:11 +0100 Subject: [PATCH 034/157] Curator suppor info --- docs/home/howto/UPGRADE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 3583d7a9e8..a4b866da74 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -277,12 +277,14 @@ specification: ``` and running the `upgrade` command against the logging component of your Epiphany installation, together with a `-f` option: ``` -epicli upgrade -b / --upgrade-components "logging" -f /.yml +epicli upgrade -b / --upgrade-components "logging" -f /.yml ``` The default value of the `odfe_migration` parameter is set to _false_. All described below remarks related to TLS certificates of the Open Distro upgrade stay valid. You should plan and test all your upgrade activities before proceeding on the production. +Migration of Elasticsearch Curator is not supported. More info on use of Curator in OpenSearch environment can be found [here](https://github.com/opensearch-project/OpenSearch/issues/1352). +
## Open Distro for Elasticsearch upgrade From 1ed5c1fe8d11a4a107ce9d5169872b7d1f8d8b62 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 14 Feb 2022 15:45:27 +0100 Subject: [PATCH 035/157] File renaming --- .../tasks/logging_elasticsearch_snapshot.yml | 89 ------------------- .../roles/backup/tasks/logging_kibana_etc.yml | 27 ------ 2 files changed, 116 deletions(-) delete mode 100644 ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml delete mode 100644 ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml diff --git a/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml deleted file mode 100644 index 352fd9858c..0000000000 --- a/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml +++ /dev/null @@ -1,89 +0,0 @@ ---- -- name: Include default vars from opensearch role - include_vars: - file: roles/opensearch/defaults/main.yml - name: odfe - -- name: Set helper facts - set_fact: - elasticsearch_endpoint: >- - https://{{ ansible_default_ipv4.address }}:9200 - snapshot_name: >- - {{ ansible_date_time.iso8601_basic_short | replace('T','-') }} - vars: - uri_template: &uri - client_cert: "{{ odfe.certificates.dirs.certs }}/{{ odfe.certificates.files.admin.cert.filename }}" - client_key: "{{ odfe.certificates.dirs.certs }}/{{ odfe.certificates.files.admin.key.filename }}" - validate_certs: false - body_format: json - -- debug: var=snapshot_name - -- name: Check cluster health - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_cluster/health" - method: GET - register: uri_response - until: uri_response is success - retries: 12 - delay: 5 - -- name: Ensure snapshot repository is defined - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}" - method: PUT - body: - type: fs - settings: - location: "{{ elasticsearch_snapshot_repository_location }}" - compress: true - -- name: Trigger snapshot creation - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ snapshot_name }}" - method: PUT - -- name: Wait (up to 12h) for snapshot completion - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ snapshot_name }}" - method: GET - register: uri_response - until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS" - retries: "{{ (12 * 3600 // 10) | int }}" # 12h - delay: 10 - -- name: Find all snapshots - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/_all" - method: GET - register: uri_response - -- name: Delete old snapshots - uri: - <<: *uri - url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ item }}" - method: DELETE - loop: >- - {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }} - -- name: Create snapshot archive - import_tasks: common/create_snapshot_archive.yml - vars: - snapshot_prefix: "elasticsearch_snapshot" - dirs_to_archive: - - "{{ elasticsearch_snapshot_repository_location }}/" - -- name: Create snapshot checksum - import_tasks: common/create_snapshot_checksum.yml - -- name: Transfer artifacts via rsync - import_tasks: common/download_via_rsync.yml - vars: - artifacts: - - "{{ snapshot_path }}" - - "{{ snapshot_path }}.sha1" diff --git a/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml b/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml deleted file mode 100644 index 4b774e7d4f..0000000000 --- a/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Assert that the snapshot_name fact is defined and valid - assert: - that: - - snapshot_name is defined - - snapshot_name is string - - snapshot_name | length > 0 - fail_msg: The snapshot_name fact must be defined and must be a non-empty string. - -- debug: var=snapshot_name - -- name: Create snapshot archive - import_tasks: common/create_snapshot_archive.yml - vars: - snapshot_prefix: "kibana_etc" - dirs_to_archive: - - /etc/kibana/ - -- name: Create snapshot checksum - import_tasks: common/create_snapshot_checksum.yml - -- name: Transfer artifacts via rsync - import_tasks: common/download_via_rsync.yml - vars: - artifacts: - - "{{ snapshot_path }}" - - "{{ snapshot_path }}.sha1" From a1b849f346f2872468c6a49770c3121abfefa458 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 14 Feb 2022 15:46:09 +0100 Subject: [PATCH 036/157] File renaming --- .../logging_opensearch_dashboards_etc.yml | 32 +++++++ .../tasks/logging_opensearch_snapshot.yml | 91 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml create mode 100644 ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml new file mode 100644 index 0000000000..6205db72fa --- /dev/null +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml @@ -0,0 +1,32 @@ +--- +- name: Assert that the snapshot_name fact is defined and valid + assert: + that: + - snapshot_name is defined + - snapshot_name is string + - snapshot_name | length > 0 + fail_msg: The snapshot_name fact must be defined and must be a non-empty string. + +- debug: var=snapshot_name + +- name: Include default vars from opensearch_dashboards role + include_vars: + file: roles/opensearch_dashboards/defaults/main.yml + name: opsd_def + +- name: Create snapshot archive + import_tasks: common/create_snapshot_archive.yml + vars: + snapshot_prefix: "opsd_conf_dir" + dirs_to_archive: + - opsd_def.opsd_conf_dir + +- name: Create snapshot checksum + import_tasks: common/create_snapshot_checksum.yml + +- name: Transfer artifacts via rsync + import_tasks: common/download_via_rsync.yml + vars: + artifacts: + - "{{ snapshot_path }}" + - "{{ snapshot_path }}.sha1" diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml new file mode 100644 index 0000000000..c42af3240b --- /dev/null +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -0,0 +1,91 @@ +--- +- name: Include default vars from opensearch role + include_vars: + file: roles/opensearch/defaults/main.yml + name: ops_def + +- name: Set helper facts + set_fact: + opensearch_endpoint: >- + https://{{ ansible_default_ipv4.address }}:9200 + snapshot_name: >- + {{ ansible_date_time.iso8601_basic_short | replace('T','-') }} + vars: + uri_template: &uri + client_cert: "{{ ops_def.certificates.dirs.certs }}/{{ ops_def.certificates.files.admin.cert.filename }}" + client_key: "{{ ops_def.certificates.dirs.certs }}/{{ ops_def.certificates.files.admin.key.filename }}" + validate_certs: false + body_format: json + +- debug: var=snapshot_name + +- name: Check cluster health + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_cluster/health" + method: GET + return_content: yes + register: cluster_status + until: cluster_status.json.status + retries: 12 + delay: 5 + +- name: Ensure snapshot repository is defined + when: cluster_status.json.number_of_nodes == '1' # https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/BACKUP.md#logging + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}" + method: PUT + body: + type: fs + settings: + location: "{{ opensearch_snapshot_repository_location }}" + compress: true + +- name: Trigger snapshot creation + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}" + method: PUT + +- name: Wait (up to 12h) for snapshot completion + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}" + method: GET + register: uri_response + until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS" + retries: "{{ (12 * 3600 // 10) | int }}" # 12h + delay: 10 + +- name: Find all snapshots + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/_all" + method: GET + register: uri_response + +- name: Delete old snapshots + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ item }}" + method: DELETE + loop: >- + {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }} + +- name: Create snapshot archive + import_tasks: common/create_snapshot_archive.yml + vars: + snapshot_prefix: "elasticsearch_snapshot" + dirs_to_archive: + - "{{ opensearch_snapshot_repository_location }}/" + +- name: Create snapshot checksum + import_tasks: common/create_snapshot_checksum.yml + +- name: Transfer artifacts via rsync + import_tasks: common/download_via_rsync.yml + vars: + artifacts: + - "{{ snapshot_path }}" + - "{{ snapshot_path }}.sha1" From 8e85ffb8a9b797035c618ba92d8723058472fa47 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 14 Feb 2022 15:47:34 +0100 Subject: [PATCH 037/157] Coorrect permisions --- .../roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index d343a98bb4..19f812dff1 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -23,9 +23,12 @@ src: "{{ item.1 }}" dest: "{{ item.2 }}" remote_src: yes + owner: opensearch + group: root + mode: ug=rwx,o= directory_mode: yes with_items: - - { 1: "/var/lib/elasticsearch-snapshots", 2: "{{ specification.paths.ops_repo }}" } + - { 1: "/var/lib/elasticsearch-snapshots/", 2: "{{ specification.paths.ops_repo }}/" } - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.ops_data }}" } - name: ODFE migr | Prepare a list of ESS certs and keys From cffdbbc8daed8a39faf909f524189c8823939785 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 14 Feb 2022 15:47:57 +0100 Subject: [PATCH 038/157] ODFE to OPS migration --- ansible/playbooks/backup_logging.yml | 6 +++--- ansible/playbooks/roles/backup/defaults/main.yml | 4 ++-- .../playbooks/roles/opensearch/defaults/main.yml | 6 +++--- .../opensearch_dashboards/defaults/main.yml | 8 ++++---- .../playbooks/roles/recovery/defaults/main.yml | 4 ++-- .../tasks/logging_elasticsearch_snapshot.yml | 16 ++++++++-------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ansible/playbooks/backup_logging.yml b/ansible/playbooks/backup_logging.yml index c1252ec696..75e577a41b 100644 --- a/ansible/playbooks/backup_logging.yml +++ b/ansible/playbooks/backup_logging.yml @@ -14,7 +14,7 @@ name: component_vars - import_role: name: backup - tasks_from: logging_elasticsearch_snapshot + tasks_from: logging_opensearch_snapshot - import_role: name: backup tasks_from: logging_elasticsearch_etc @@ -28,10 +28,10 @@ - when: specification.components.logging.enabled | default(false) block: - include_vars: - file: roles/kibana/vars/main.yml + file: roles/opensearch_dashboards/vars/main.yml name: component_vars - import_role: name: backup - tasks_from: logging_kibana_etc + tasks_from: logging_opensearch_dashboards_etc vars: snapshot_name: "{{ hostvars[groups.logging.0].snapshot_name }}" diff --git a/ansible/playbooks/roles/backup/defaults/main.yml b/ansible/playbooks/roles/backup/defaults/main.yml index 770caa87ec..51cc26574b 100644 --- a/ansible/playbooks/roles/backup/defaults/main.yml +++ b/ansible/playbooks/roles/backup/defaults/main.yml @@ -2,5 +2,5 @@ backup_dir: /epibackup backup_destination_dir: "{{ backup_dir }}/mounted" backup_destination_host: "{{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}" -elasticsearch_snapshot_repository_name: epiphany -elasticsearch_snapshot_repository_location: /var/lib/opensearch-snapshots +opensearch_snapshot_repository_name: epiphany +opensearch_snapshot_repository_location: /var/lib/opensearch-snapshots diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 9aeebc06ed..ebf71aab64 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -8,9 +8,9 @@ versions: ops_version: "1.2.4" certificates: dirs: # must be under the config directory, specified using a relative path - certs: /usr/share/opensearch/config/cert - ca_key: /usr/share/opensearch/config/private - csr: /usr/share/opensearch/config/csr + certs: /usr/share/opensearch/config + ca_key: /usr/share/opensearch/config + csr: /usr/share/opensearch/config dn_attributes_order: ['CN', 'OU', 'O', 'L', 'S', 'C', 'DC'] files: demo: diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index acfb1c43d4..1dccc3b6d7 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -11,12 +11,12 @@ ops_nodes: |- populate_inventory_to_hosts_file: true -ops_dashboards_user: "{{ specification.ops_dashboards_user }}" -ops_dashboards_password: "{{ specification.ops_dashboards_password }}" +opsd_user: "{{ specification.ops_dashboards_user }}" +opsd_password: "{{ specification.ops_dashboards_password }}" ops_user: "{{ specification.ops_user }}" ops_download_url: https://artifacts.opensearch.org/releases/bundle/opensearch -ops_dashboards_home: "{{ specification.ops_dashboards_home }}" -ops_dashboards_conf_dir: "{{ specification.ops_dashboards_conf_dir }}" +opsd_home: "{{ specification.ops_dashboards_home }}" +opsd_conf_dir: "{{ specification.ops_dashboards_conf_dir }}" ops_plugin_bin_path: "{{ specification.ops_plugin_bin_path }}" ops_api_port: 9200 diff --git a/ansible/playbooks/roles/recovery/defaults/main.yml b/ansible/playbooks/roles/recovery/defaults/main.yml index d11dc98aef..e105375aa7 100644 --- a/ansible/playbooks/roles/recovery/defaults/main.yml +++ b/ansible/playbooks/roles/recovery/defaults/main.yml @@ -2,5 +2,5 @@ recovery_dir: /epibackup recovery_source_dir: "{{ recovery_dir }}/mounted" recovery_source_host: "{{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}" -elasticsearch_snapshot_repository_name: epiphany -elasticsearch_snapshot_repository_location: /var/lib/opensearch-snapshots +opensearch_snapshot_repository_name: epiphany +opensearch_snapshot_repository_location: /var/lib/opensearch-snapshots diff --git a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml index fa43b02982..d03dad0e26 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml @@ -6,7 +6,7 @@ - name: Set helper facts set_fact: - elasticsearch_endpoint: >- + opensearch_endpoint: >- https://{{ ansible_default_ipv4.address }}:9200 vars: uri_template: &uri @@ -18,7 +18,7 @@ - name: Check cluster health uri: <<: *uri - url: "{{ elasticsearch_endpoint }}/_cluster/health" + url: "{{ opensearch_endpoint }}/_cluster/health" method: GET register: uri_response until: uri_response is success @@ -45,17 +45,17 @@ import_tasks: common/clear_directories.yml vars: dirs_to_clear: - - "{{ elasticsearch_snapshot_repository_location }}/" + - "{{ opensearch_snapshot_repository_location }}/" - name: Extract the archive unarchive: - dest: "{{ elasticsearch_snapshot_repository_location }}/" + dest: "{{ opensearch_snapshot_repository_location }}/" src: "{{ recovery_dir }}/{{ snapshot_path | basename }}" remote_src: true - name: Change snapshot directory permissions file: - path: "{{ elasticsearch_snapshot_repository_location }}/" + path: "{{ opensearch_snapshot_repository_location }}/" owner: opensearch group: opensearch recurse: true @@ -89,19 +89,19 @@ - name: Close all indices uri: <<: *uri - url: "{{ elasticsearch_endpoint }}/_all/_close" + url: "{{ opensearch_endpoint }}/_all/_close" method: POST - name: Delete all indices uri: <<: *uri - url: "{{ elasticsearch_endpoint }}/_all" + url: "{{ opensearch_endpoint }}/_all" method: DELETE - name: Restore the snapshot uri: <<: *uri - url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ snapshot_name }}/_restore" + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}/_restore" method: POST always: From 76238291c0cd224388bca222743dc461af3f825e Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 08:28:02 +0100 Subject: [PATCH 039/157] vars housekeeping --- .../roles/opensearch_dashboards/defaults/main.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index 1dccc3b6d7..aa8a521a3d 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -11,14 +11,6 @@ ops_nodes: |- populate_inventory_to_hosts_file: true -opsd_user: "{{ specification.ops_dashboards_user }}" -opsd_password: "{{ specification.ops_dashboards_password }}" -ops_user: "{{ specification.ops_user }}" -ops_download_url: https://artifacts.opensearch.org/releases/bundle/opensearch -opsd_home: "{{ specification.ops_dashboards_home }}" -opsd_conf_dir: "{{ specification.ops_dashboards_conf_dir }}" -ops_plugin_bin_path: "{{ specification.ops_plugin_bin_path }}" - ops_api_port: 9200 ops_nodes_dashboards: |- {% for item in groups['opensearch_dashboards'] -%} From 279100f94361b45756ffd037edf9a1b9450696b1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 08:28:53 +0100 Subject: [PATCH 040/157] Renaming files --- ...csearch_etc.yml => logging_opensearch_conf.yml} | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) rename ansible/playbooks/roles/backup/tasks/{logging_elasticsearch_etc.yml => logging_opensearch_conf.yml} (66%) diff --git a/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml similarity index 66% rename from ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml rename to ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml index b9e2bf79db..db9f0655d5 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml @@ -1,4 +1,14 @@ --- +- name: Include default vars from opensearch role + include_vars: + file: roles/opensearch/defaults/main.yml + name: ops_def + +- name: Include vars from opensearch role + include_vars: + file: roles/opensearch/vars/main.yml + name: ops_vars + - name: Assert that the snapshot_name fact is defined and valid assert: that: @@ -12,9 +22,9 @@ - name: Create snapshot archive import_tasks: common/create_snapshot_archive.yml vars: - snapshot_prefix: "elasticsearch_etc" + snapshot_prefix: "opensearch_conf" dirs_to_archive: - - /etc/elasticsearch/ + - "{{ ops_vars.specification.paths.ops_conf_dir }}" - name: Create snapshot checksum import_tasks: common/create_snapshot_checksum.yml From 8c9406ca8c5d0750f0d2c90f7d1c1c95f0cd97a8 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 08:29:53 +0100 Subject: [PATCH 041/157] ODFE to OPS migration --- ansible/playbooks/backup_logging.yml | 2 +- .../tasks/logging_opensearch_snapshot.yml | 57 +++++++++++-------- docs/home/howto/BACKUP.md | 8 +-- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/ansible/playbooks/backup_logging.yml b/ansible/playbooks/backup_logging.yml index 75e577a41b..4e594fc736 100644 --- a/ansible/playbooks/backup_logging.yml +++ b/ansible/playbooks/backup_logging.yml @@ -17,7 +17,7 @@ tasks_from: logging_opensearch_snapshot - import_role: name: backup - tasks_from: logging_elasticsearch_etc + tasks_from: logging_opensearch_conf - hosts: kibana[0] gather_facts: true diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml index c42af3240b..f05f77d318 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -30,33 +30,40 @@ retries: 12 delay: 5 -- name: Ensure snapshot repository is defined - when: cluster_status.json.number_of_nodes == '1' # https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/BACKUP.md#logging - uri: - <<: *uri - url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}" - method: PUT - body: - type: fs - settings: - location: "{{ opensearch_snapshot_repository_location }}" - compress: true +- name: No backup warning + when: not cluster_status.json.number_of_nodes == 1 + debug: + msg: "[WARNING] No snapshot backup created as only single-node cluster backup is supported." -- name: Trigger snapshot creation - uri: - <<: *uri - url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}" - method: PUT +- name: Snapshot backup + when: cluster_status.json.number_of_nodes == 1 # https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/BACKUP.md#logging + block: + - name: Ensure snapshot repository is defined + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}" + method: PUT + body: + type: fs + settings: + location: "{{ opensearch_snapshot_repository_location }}" + compress: true -- name: Wait (up to 12h) for snapshot completion - uri: - <<: *uri - url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}" - method: GET - register: uri_response - until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS" - retries: "{{ (12 * 3600 // 10) | int }}" # 12h - delay: 10 + - name: Trigger snapshot creation + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}" + method: PUT + + - name: Wait (up to 12h) for snapshot completion + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}" + method: GET + register: uri_response + until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS" + retries: "{{ (12 * 3600 // 10) | int }}" # 12h + delay: 10 - name: Find all snapshots uri: diff --git a/docs/home/howto/BACKUP.md b/docs/home/howto/BACKUP.md index bc601be64f..568634ffb6 100644 --- a/docs/home/howto/BACKUP.md +++ b/docs/home/howto/BACKUP.md @@ -129,11 +129,11 @@ Recovery includes all backed up files Logging backup includes: -- Elasticsearch database snapshot -- Elasticsearch configuration ``/etc/elasticsearch/`` -- Kibana configuration ``/etc/kibana/`` +- OpenSearch database snapshot +- OpenSearch configuration ``/usr/share/opensearch/config/`` +- OpenSearch Dashboards configuration ``/usr/share/opensearch-dashboards/config/`` -Only single-node Elasticsearch backup is supported. Solution for multi-node Elasticsearch cluster will be added in +Only single-node OpenSearch backup is supported. Solution for multi-node OpenSearch cluster will be added in future release. ### Monitoring From 5b8d5b07ba1329426855cb32dc0019b0df3ee4e7 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 14:35:40 +0100 Subject: [PATCH 042/157] ODFE to OPS migration - bck/restore --- ansible/playbooks/recovery_logging.yml | 8 +-- .../tasks/logging_opensearch_snapshot.yml | 54 +++++++++---------- ...ch_etc.yml => logging_opensearch_conf.yml} | 13 +++-- ...=> logging_opensearch_dashboards_conf.yml} | 19 ++++--- ...ot.yml => logging_opensearch_snapshot.yml} | 8 +-- 5 files changed, 56 insertions(+), 46 deletions(-) rename ansible/playbooks/roles/recovery/tasks/{logging_elasticsearch_etc.yml => logging_opensearch_conf.yml} (71%) rename ansible/playbooks/roles/recovery/tasks/{logging_kibana_etc.yml => logging_opensearch_dashboards_conf.yml} (64%) rename ansible/playbooks/roles/recovery/tasks/{logging_elasticsearch_snapshot.yml => logging_opensearch_snapshot.yml} (92%) diff --git a/ansible/playbooks/recovery_logging.yml b/ansible/playbooks/recovery_logging.yml index 796d1c0bae..c7cd956778 100644 --- a/ansible/playbooks/recovery_logging.yml +++ b/ansible/playbooks/recovery_logging.yml @@ -13,10 +13,10 @@ name: component_vars - import_role: name: recovery - tasks_from: logging_elasticsearch_etc + tasks_from: logging_opensearch_conf - import_role: name: recovery - tasks_from: logging_elasticsearch_snapshot + tasks_from: logging_opensearch_snapshot - hosts: kibana[0] gather_facts: true @@ -27,8 +27,8 @@ - when: specification.components.logging.enabled | default(false) block: - include_vars: - file: roles/kibana/vars/main.yml + file: roles/opensearch_dashboards/vars/main.yml name: component_vars - import_role: name: recovery - tasks_from: logging_kibana_etc + tasks_from: logging_opensearch_dashboards_conf diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml index f05f77d318..a578ef764e 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -65,34 +65,34 @@ retries: "{{ (12 * 3600 // 10) | int }}" # 12h delay: 10 -- name: Find all snapshots - uri: - <<: *uri - url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/_all" - method: GET - register: uri_response + - name: Find all snapshots + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/_all" + method: GET + register: uri_response -- name: Delete old snapshots - uri: - <<: *uri - url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ item }}" - method: DELETE - loop: >- - {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }} + - name: Delete old snapshots + uri: + <<: *uri + url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ item }}" + method: DELETE + loop: >- + {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }} -- name: Create snapshot archive - import_tasks: common/create_snapshot_archive.yml - vars: - snapshot_prefix: "elasticsearch_snapshot" - dirs_to_archive: - - "{{ opensearch_snapshot_repository_location }}/" + - name: Create snapshot archive + import_tasks: common/create_snapshot_archive.yml + vars: + snapshot_prefix: "opensearch_snapshot" + dirs_to_archive: + - "{{ opensearch_snapshot_repository_location }}/" -- name: Create snapshot checksum - import_tasks: common/create_snapshot_checksum.yml + - name: Create snapshot checksum + import_tasks: common/create_snapshot_checksum.yml -- name: Transfer artifacts via rsync - import_tasks: common/download_via_rsync.yml - vars: - artifacts: - - "{{ snapshot_path }}" - - "{{ snapshot_path }}.sha1" + - name: Transfer artifacts via rsync + import_tasks: common/download_via_rsync.yml + vars: + artifacts: + - "{{ snapshot_path }}" + - "{{ snapshot_path }}.sha1" diff --git a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml similarity index 71% rename from ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml rename to ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml index 427dab580b..0980e97f48 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml @@ -1,8 +1,13 @@ --- +- name: Include vars from opensearch role + include_vars: + file: roles/opensearch/vars/main.yml + name: ops_vars + - name: Find snapshot archive import_tasks: common/find_snapshot_archive.yml vars: - snapshot_prefix: "elasticsearch_etc" + snapshot_prefix: "opensearch_conf" snapshot_name: "{{ specification.components.logging.snapshot_name }}" - name: Transfer the archive via rsync @@ -24,15 +29,15 @@ import_tasks: common/clear_directories.yml vars: dirs_to_clear: - - /etc/elasticsearch/ + - "{{ ops_vars.specification.paths.ops_conf_dir }}" - name: Extract the archive unarchive: - dest: /etc/elasticsearch/ + dest: "{{ ops_vars.specification.paths.ops_conf_dir }}" src: "{{ recovery_dir }}/{{ snapshot_path | basename }}" remote_src: true -- name: Start elasticsearch service +- name: Start OpenSearch service systemd: name: opensearch state: started diff --git a/ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml similarity index 64% rename from ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml rename to ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml index 3792303795..b4c8692bf2 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml @@ -1,8 +1,13 @@ --- +- name: Include vars from opensearch role + include_vars: + file: roles/opensearch_dashboards/vars/main.yml + name: opsd_vars + - name: Find snapshot archive import_tasks: common/find_snapshot_archive.yml vars: - snapshot_prefix: "kibana_etc" + snapshot_prefix: "opsd_conf_dir" snapshot_name: "{{ specification.components.logging.snapshot_name }}" - name: Transfer the archive via rsync @@ -15,24 +20,24 @@ - name: Verify snapshot checksum import_tasks: common/verify_snapshot_checksum.yml -- name: Stop kibana service +- name: Stop OPSD service systemd: - name: kibana + name: dashboards state: stopped - name: Clear directories import_tasks: common/clear_directories.yml vars: dirs_to_clear: - - /etc/kibana/ + - "{{ opsd_vars.specification.paths.ops_conf_dir }}" - name: Extract the archive unarchive: - dest: /etc/kibana/ + dest: "{{ opsd_vars.specification.paths.ops_conf_dir }}" src: "{{ recovery_dir }}/{{ snapshot_path | basename }}" remote_src: true -- name: Start kibana service +- name: Start OPSD service systemd: - name: kibana + name: dashboards state: started diff --git a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml similarity index 92% rename from ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml rename to ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml index d03dad0e26..588111e1aa 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml @@ -28,7 +28,7 @@ - name: Find snapshot archive import_tasks: common/find_snapshot_archive.yml vars: - snapshot_prefix: "elasticsearch_snapshot" + snapshot_prefix: "opensearch_snapshot" snapshot_name: "{{ specification.components.logging.snapshot_name }}" - name: Transfer the archive via rsync @@ -63,17 +63,17 @@ - name: Reconstruct the snapshot_name set_fact: snapshot_name: >- - {{ snapshot_path | basename | regex_replace('^elasticsearch_snapshot_(.*).tar.gz$', '\1') }} + {{ snapshot_path | basename | regex_replace('^opensearch_snapshot_(.*).tar.gz$', '\1') }} - debug: var=snapshot_name -- name: Ensure all kibana and filebeat instances are stopped, then restore the snapshot +- name: Ensure all OPSD and filebeat instances are stopped, then restore the snapshot block: - name: Stop all kibana instances delegate_to: "{{ item }}" systemd: - name: kibana + name: dashoboard state: stopped enabled: false loop: "{{ groups.kibana | default([]) }}" From 19b0ce63ea320d69fcc064f601ab2ebbff410f15 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 16:11:15 +0100 Subject: [PATCH 043/157] Workaround needed for this version of OPS with Filebeat --- ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index 32df1b338c..580b3c5abb 100644 --- a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -181,7 +181,7 @@ setup.template.settings: {% set dashboards_enabled = is_upgrade_run | ternary(existing_setup_dashboards.enabled, specification.kibana.dashboards.enabled) %} {% if dashboards_enabled | lower == 'auto' %} {% if group_names | intersect(['kibana', 'logging']) | count == 2 %} -setup.dashboards.enabled: true +setup.dashboards.enabled: false # A workaround. More info: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656#issuecomment-978036236 {% else %} setup.dashboards.enabled: false {% endif %} From e683c9949874979c34b401acb4b64593590d14c2 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 16:12:05 +0100 Subject: [PATCH 044/157] Assure logstash user pass is set --- .../upgrade/tasks/opensearch/migrate-odfe.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 508794b78f..33322e724c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -147,6 +147,44 @@ run_once: true when: kibanaserver_check_response.status == 200 +- name: ODFE migr | Check if logstash user exists + uri: + url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/logstash" + method: GET + # 404 code is used there as someone can remove admin user on its own. + status_code: [200, 404] + validate_certs: no + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key }}" + register: logstash_check_response + until: logstash_check_response is success + retries: 60 + delay: 1 + run_once: true + +- name: ODFE migr | Set logstash user password + uri: + url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" + method: PATCH + status_code: [200] + body: + - op: "replace" + path: "/logstash" + value: + password: "{{ specification.logstash_password }}" + reserved: "true" + description: "logstash user" + client_cert: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.cert }}" + client_key: "{{ certificates.dirs.certs }}/{{ certificates.files.admin.key }}" + body_format: json + validate_certs: no + register: uri_response + until: uri_response is success + retries: 5 + delay: 1 + run_once: true + when: logstash_check_response.status == 200 + - name: ODFE migr | Check the opensearch status command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k register: ops_status From 5da0bc815ecc1eb5acbf837a047045c941d62b89 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 16:51:11 +0100 Subject: [PATCH 045/157] ODFE to OPS migration - bck/restore --- .../roles/recovery/tasks/logging_opensearch_conf.yml | 2 +- .../roles/recovery/tasks/logging_opensearch_snapshot.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml index 0980e97f48..65b855cb61 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml @@ -20,7 +20,7 @@ - name: Verify snapshot checksum import_tasks: common/verify_snapshot_checksum.yml -- name: Stop elasticsearch service +- name: Stop OpenSearch service systemd: name: opensearch state: stopped diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml index 588111e1aa..15b360d45c 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml @@ -70,10 +70,10 @@ - name: Ensure all OPSD and filebeat instances are stopped, then restore the snapshot block: - - name: Stop all kibana instances + - name: Stop all OPS Dashboards instances delegate_to: "{{ item }}" systemd: - name: dashoboard + name: dashboards state: stopped enabled: false loop: "{{ groups.kibana | default([]) }}" @@ -105,10 +105,10 @@ method: POST always: - - name: Start all kibana instances + - name: Start all OPS Dashboards instances delegate_to: "{{ item }}" systemd: - name: kibana + name: dashboards state: started enabled: true loop: "{{ groups.kibana | default([]) }}" From cd954ae50529a619986cbad499a5b99b79ca488c Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 18:31:41 +0100 Subject: [PATCH 046/157] Migartion - File renaming --- ...oards_etc.yml => logging_opensearch_dashboards_conf.yml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename ansible/playbooks/roles/backup/tasks/{logging_opensearch_dashboards_etc.yml => logging_opensearch_dashboards_conf.yml} (81%) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml similarity index 81% rename from ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml rename to ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml index 6205db72fa..4017a61d68 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_etc.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml @@ -9,9 +9,9 @@ - debug: var=snapshot_name -- name: Include default vars from opensearch_dashboards role +- name: Include vars from opensearch_dashboards role include_vars: - file: roles/opensearch_dashboards/defaults/main.yml + file: roles/opensearch_dashboards/vars/main.yml name: opsd_def - name: Create snapshot archive @@ -19,7 +19,7 @@ vars: snapshot_prefix: "opsd_conf_dir" dirs_to_archive: - - opsd_def.opsd_conf_dir + - "{{ opsd_def.specification.paths.opsd_conf_dir }}" - name: Create snapshot checksum import_tasks: common/create_snapshot_checksum.yml From be98eb08e884db1f8bec359764b6698c0bf35884 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 15 Feb 2022 18:32:49 +0100 Subject: [PATCH 047/157] Corrrect var names --- ansible/playbooks/backup_logging.yml | 2 +- .../recovery/tasks/logging_opensearch_dashboards_conf.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/backup_logging.yml b/ansible/playbooks/backup_logging.yml index 4e594fc736..34c40726f4 100644 --- a/ansible/playbooks/backup_logging.yml +++ b/ansible/playbooks/backup_logging.yml @@ -32,6 +32,6 @@ name: component_vars - import_role: name: backup - tasks_from: logging_opensearch_dashboards_etc + tasks_from: logging_opensearch_dashboards_conf vars: snapshot_name: "{{ hostvars[groups.logging.0].snapshot_name }}" diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml index b4c8692bf2..05d69f6456 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml @@ -29,11 +29,11 @@ import_tasks: common/clear_directories.yml vars: dirs_to_clear: - - "{{ opsd_vars.specification.paths.ops_conf_dir }}" + - "{{ opsd_vars.specification.paths.opsd_conf_dir }}" - name: Extract the archive unarchive: - dest: "{{ opsd_vars.specification.paths.ops_conf_dir }}" + dest: "{{ opsd_vars.specification.paths.opsd_conf_dir }}" src: "{{ recovery_dir }}/{{ snapshot_path | basename }}" remote_src: true From e9b1d98bcbea8c522c0968dd0495dc32c908040a Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 08:38:06 +0100 Subject: [PATCH 048/157] Kibana API not available bug workaround --- .../roles/filebeat/templates/filebeat.yml.j2 | 23 +++++++++++-------- docs/home/howto/UPGRADE.md | 3 ++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index 580b3c5abb..95ec4ac418 100644 --- a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -178,16 +178,21 @@ setup.template.settings: # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. -{% set dashboards_enabled = is_upgrade_run | ternary(existing_setup_dashboards.enabled, specification.kibana.dashboards.enabled) %} -{% if dashboards_enabled | lower == 'auto' %} - {% if group_names | intersect(['kibana', 'logging']) | count == 2 %} -setup.dashboards.enabled: false # A workaround. More info: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656#issuecomment-978036236 - {% else %} +# +# Below logic commented out as a workaround for problem with filebeat till the time OPS team will resolve it. +# More info: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656#issuecomment-978036236 +# A static value is used instead: setup.dashboards.enabled: false - {% endif %} -{% else %} -setup.dashboards.enabled: {{ dashboards_enabled | lower }} -{% endif %} +# {% set dashboards_enabled = is_upgrade_run | ternary(existing_setup_dashboards.enabled, specification.kibana.dashboards.enabled) %} +# {% if dashboards_enabled | lower == 'auto' %} +# {% if group_names | intersect(['kibana', 'logging']) | count == 2 %} +# setup.dashboards.enabled: true +# {% else %} +#setup.dashboards.enabled: false +# {% endif %} +#{% else %} +#setup.dashboards.enabled: {{ dashboards_enabled | lower }} +#{% endif %} # The Elasticsearch index name. # This setting overwrites the index name defined in the dashboards and index pattern. diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index a4b866da74..63727c4d77 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -277,8 +277,9 @@ specification: ``` and running the `upgrade` command against the logging component of your Epiphany installation, together with a `-f` option: ``` -epicli upgrade -b / --upgrade-components "logging" -f /.yml +epicli upgrade -b / --upgrade-components "logging,filebeat" -f /.yml ``` +Keep in mind, that for the current version of OPS/OPSD it is necessary to include the `filebeat` component along with the loggging one in order to implement the workaround for _Kibana API not available_ [bug](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656#issuecomment-978036236). The default value of the `odfe_migration` parameter is set to _false_. All described below remarks related to TLS certificates of the Open Distro upgrade stay valid. You should plan and test all your upgrade activities before proceeding on the production. From d06ac967d08889ada8415d92cdcc1a755b8a02b1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 09:11:05 +0100 Subject: [PATCH 049/157] Adding some background of the migration to the docs --- docs/home/howto/UPGRADE.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 63727c4d77..64af4eb192 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -261,7 +261,10 @@ in [ZooKeeper documentation](https://cwiki.apache.org/confluence/display/ZOOKEE Make sure you have a backup before proceeding to migration steps described below ! --- -Following the decision of Elastic NV on ceasing open source options available for Elasticsearch and Kibana and releasing them under the Elastic license (more info [here](https://github.com/epiphany-platform/epiphany/issues/2870)) Epiphany team decided to implement a mechanism of automatic migration from Elasticsearch 7.10.2 to Opensearch 1.2.4. +Following the decision of Elastic NV[1] on ceasing open source options available for Elasticsearch and Kibana and releasing them under the Elastic license (more info [here](https://github.com/epiphany-platform/epiphany/issues/2870)) Epiphany team decided to implement a mechanism of automatic migration from ElasticSearch 7.10.2 to OpenSearch 1.2.4. + +It is important to remember that while the new platform makes an effort to continue to support a broad set of third party tools (ie. Beats tools) however there can be some drawbacks or even malfunctions came across all over the way as not everything have been tested or have explicitly been added to OpenSearch compatibility scope[2]. +Additionally some of the components (ie. ElasticSearch Curator) or some embedded service accounts ( ie. _kibanaserver_) can be still found in OpenSearch environment but they will be successfully phased out. The migration can be fired by placing `odfe_migration` switch in your manifest file: ```yaml @@ -286,6 +289,11 @@ All described below remarks related to TLS certificates of the Open Distro upgr Migration of Elasticsearch Curator is not supported. More info on use of Curator in OpenSearch environment can be found [here](https://github.com/opensearch-project/OpenSearch/issues/1352). +[1] https://www.elastic.co/pricing/faq/licensing#what-are-the-key-changes-being-made-to-the-elastic-license + +[2] https://opensearch.org/docs/latest/clients/agents-and-ingestion-tools/index/ + +
## Open Distro for Elasticsearch upgrade From 52e47a19a674bbaad553ba00be5b55d475d050cc Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 09:55:38 +0100 Subject: [PATCH 050/157] Docs update --- docs/changelogs/CHANGELOG-2.0.md | 2 ++ docs/home/COMPONENTS.md | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 7aaea839b2..0e729d80cc 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -3,6 +3,7 @@ ## [2.0.0] YYYY-MM-DD ### Added +- [#2870](https://github.com/epiphany-platform/epiphany/issues/2870) - OpenDistro for ElasticSearch project migrated to OpenSearch ### Fixed @@ -24,6 +25,7 @@ - [#2833](https://github.com/epiphany-platform/epiphany/issues/2833) - Removal of Logstash component - [#2836](https://github.com/epiphany-platform/epiphany/issues/2836) - Removal of Istio component - [#2837](https://github.com/epiphany-platform/epiphany/issues/2837) - Removal of Apache Ignite component +- [#2870](https://github.com/epiphany-platform/epiphany/issues/2870) - Migration of OpenDistro for ElasticSearch ### Deprecated diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index 08afbee7a1..aeec3a1923 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -18,10 +18,6 @@ Note that versions are default versions and can be changed in certain cases thro | RabbitMQ | 3.8.9 | https://github.com/rabbitmq/rabbitmq-server | [Mozilla Public License](https://www.mozilla.org/en-US/MPL/) | | Docker CE | 20.10.8 | https://docs.docker.com/engine/release-notes/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | KeyCloak | 14.0.0 | https://github.com/keycloak/keycloak | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Elasticsearch OSS | 7.10.2 | https://github.com/elastic/elasticsearch | https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt | -| Elasticsearch Curator OSS | 5.8.3 | https://github.com/elastic/curator | https://github.com/elastic/curator/blob/master/LICENSE.txt | -| Opendistro for Elasticsearch | 1.13.x | https://opendistro.github.io/for-elasticsearch/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Opendistro for Elasticsearch Kibana | 1.13.1 | https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Filebeat | 7.9.2 | https://github.com/elastic/beats | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Filebeat Helm Chart | 7.9.2 | https://github.com/elastic/helm-charts | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Prometheus | 2.31.1 | https://github.com/prometheus/prometheus | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | @@ -181,6 +177,8 @@ Note that versions are default versions and can be changed in certain cases thro | msrest | 0.6.21 | https://github.com/Azure/msrest-for-python | [MIT License](https://api.github.com/repos/azure/msrest-for-python/license) | | msrestazure | 0.6.4 | https://github.com/Azure/msrestazure-for-python | [MIT License](https://api.github.com/repos/azure/msrestazure-for-python/license) | | oauthlib | 3.1.1 | https://github.com/oauthlib/oauthlib | [BSD 3-Clause "New" or "Revised" License](https://api.github.com/repos/oauthlib/oauthlib/license) | +| OpenSearch | 1.2.4 | https://github.com/opensearch-project/OpenSearch | [Apache License 2.0](https://www.apache.org/licenses/) | +| OpenSearch Dashboards | 1.2.0 | https://github.com/opensearch-project/OpenSearch-Dashboards | [Apache License 2.0](https://www.apache.org/licenses/) | | packaging | 20.9 | https://github.com/pypa/packaging | [Other](https://api.github.com/repos/pypa/packaging/license) | | paramiko | 2.9.1 | https://paramiko.org | LGPL | | pathlib2 | 2.3.6 | https://github.com/mcmtroffaes/pathlib2 | [MIT License](https://api.github.com/repos/mcmtroffaes/pathlib2/license) | From 0f935ddb8fac303d07381b371dfe613a6b8a82ef Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 12:34:50 +0100 Subject: [PATCH 051/157] Correcting inconsitences --- .../playbooks/roles/elasticsearch_curator/tasks/main.yml | 2 +- .../roles/opensearch/templates/opensearch.service.j2 | 2 +- .../opensearch_dashboards/templates/dashboards.service | 2 +- ansible/playbooks/roles/preflight/defaults/main.yml | 2 -- docs/home/howto/DATABASES.md | 6 +++--- schema/common/defaults/configuration/feature-mapping.yml | 4 ++-- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml b/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml index 1d03db3046..d743ae642f 100644 --- a/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml +++ b/ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml @@ -3,4 +3,4 @@ include_tasks: install-ops-curator-{{ ansible_os_family }}.yml - name: Include configuration tasks - include_tasks: configure-cron-jobs.yml \ No newline at end of file + include_tasks: configure-cron-jobs.yml diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 index 2978f29c49..f4f8ae2fc2 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 @@ -48,4 +48,4 @@ SendSIGKILL=no SuccessExitStatus=143 [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service index 7961a744ac..f161a6576c 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service @@ -45,4 +45,4 @@ SendSIGKILL=no SuccessExitStatus=143 [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index 5972f161b4..cc27378795 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -39,7 +39,6 @@ unsupported_roles: - logging - elasticsearch_curator - opensearch - - opensearch - kibana - filebeat - prometheus @@ -76,7 +75,6 @@ unsupported_roles: - logging - elasticsearch_curator - opensearch - - opensearch - kibana - filebeat - prometheus diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index adc929ea8c..864fe221c0 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -455,7 +455,7 @@ Properly configured application (kubernetes service) to use fully HA configurati PostgreSQL native replication is now deprecated and removed. Use [PostgreSQL HA replication with repmgr](#how-to-set-up-postgresql-ha-replication-with-repmgr-cluster) instead. -## How to start working with OpenDistro for opensearch +## How to start working with OpenDistro for ElasticSearch OpenDistro for opensearch is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). @@ -482,7 +482,7 @@ specification: **Installation with more than one node will always be clustered** - Option to configure the non-clustered installation of more than one node for Open Distro is not supported. ```yaml -kind: configuration/opensearch-for-opensearch +kind: configuration/opendistro-for-elasticsearch title: OpenDistro for Elasticsearch Config name: default specification: @@ -499,7 +499,7 @@ name: default specification: roles_mapping: opensearch: - - opensearch-for-opensearch + - opendistro-for-elasticsearch - node-exporter - filebeat - firewall diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index e4c05ca6c4..3b99b4a2a0 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -17,7 +17,7 @@ specification: enabled: true - name: logging enabled: true - - name: opensearch-for-opensearch + - name: opensearch enabled: true - name: elasticsearch-curator enabled: true @@ -122,7 +122,7 @@ specification: - filebeat - firewall opensearch: - - opensearch-for-opensearch + - opensearch - node-exporter - filebeat - firewall From 2a40af48b088679bcc2f489049bb19b7b43ca99e Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 12:41:46 +0100 Subject: [PATCH 052/157] Correction of renames which were not needed --- docs/changelogs/CHANGELOG-0.5.md | 2 +- docs/changelogs/CHANGELOG-1.3.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelogs/CHANGELOG-0.5.md b/docs/changelogs/CHANGELOG-0.5.md index 198400405d..86fa021746 100644 --- a/docs/changelogs/CHANGELOG-0.5.md +++ b/docs/changelogs/CHANGELOG-0.5.md @@ -82,7 +82,7 @@ - [#381](https://github.com/epiphany-platform/epiphany/issues/381) - Add AWS EC2 Root Volume encryption - [#782](https://github.com/epiphany-platform/epiphany/issues/781) - All disks encryption documentation - AWS - [#782](https://github.com/epiphany-platform/epiphany/issues/782) - All disks encryption documentation - Azure -- [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for opensearch +- [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for ElasticSearch - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch) - [Centralized logging](/docs/home/howto/LOGGING.md#centralized-logging-setup) diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index 3c5cfc39be..70490de0b2 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -68,7 +68,7 @@ - [#2748](https://github.com/epiphany-platform/epiphany/issues/2748) - Upgrade Kafka exporter to the version 1.4.0 - [#2750](https://github.com/epiphany-platform/epiphany/issues/2750) - Upgrade JMX exporter to the newest version - [#2699](https://github.com/epiphany-platform/epiphany/issues/2699) - Upgrade Grafana to 8.3.2 -- [#2788](https://github.com/epiphany-platform/epiphany/issues/2788) - Upgrade Log4j in Open Distro for opensearch +- [#2788](https://github.com/epiphany-platform/epiphany/issues/2788) - Upgrade Log4j in Open Distro for ElasticSearch - [#2661](https://github.com/epiphany-platform/epiphany/issues/2661) - Update K8s documentation according to the latest version Epiphany supports - [#2752](https://github.com/epiphany-platform/epiphany/issues/2752) - Upgrade postgresql exporter to the version 0.10.0 - [#2856](https://github.com/epiphany-platform/epiphany/issues/2856) - Update cloud OS images to the latest From 5858dc2c9f77568e2022598d59488330760bd35f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 12:50:20 +0100 Subject: [PATCH 053/157] ToC added for ease of nav --- docs/home/howto/UPGRADE.md | 40 +++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 64af4eb192..35e9687112 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -1,5 +1,43 @@ # Upgrade - +- [Upgrade](#upgrade) + - [Introduction](#introduction) + - [Online upgrade](#online-upgrade) + - [Online prerequisites](#online-prerequisites) + - [Start the online upgrade](#start-the-online-upgrade) + - [Offline upgrade](#offline-upgrade) + - [Offline prerequisites](#offline-prerequisites) + - [Start the offline upgrade](#start-the-offline-upgrade) + - [Additional parameters](#additional-parameters) + - [Run *apply* after *upgrade*](#run-apply-after-upgrade) + - [Kubernetes applications](#kubernetes-applications) + - [How to upgrade Kafka](#how-to-upgrade-kafka) + - [Kafka upgrade](#kafka-upgrade) + - [ZooKeeper upgrade](#zookeeper-upgrade) + - [Migration from Open Distro for Elasticsearch & Kibana to OpenSearch and OpenSearch Dashboards](#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) + - [Open Distro for Elasticsearch upgrade](#open-distro-for-elasticsearch-upgrade) + - [Node exporter upgrade](#node-exporter-upgrade) + - [RabbitMQ upgrade](#rabbitmq-upgrade) + - [Kubernetes upgrade](#kubernetes-upgrade) + - [Prerequisites](#prerequisites) + - [PostgreSQL upgrade](#postgresql-upgrade) + - [Versions](#versions) + - [Prerequisites](#prerequisites-1) + - [Upgrade](#upgrade-1) + - [Manual actions](#manual-actions) + - [Post-upgrade processing](#post-upgrade-processing) + - [Statistics](#statistics) + - [Delete old cluster](#delete-old-cluster) + - [Terraform upgrade from Epiphany 1.x to 2.x](#terraform-upgrade-from-epiphany-1x-to-2x) + - [Azure](#azure) + - [v0.12.6 => v0.13.x](#v0126--v013x) + - [v0.13.x => v0.14.x](#v013x--v014x) + - [v0.14.x => v1.0.x](#v014x--v10x) + - [v1.0.x => v1.1.3](#v10x--v113) + - [AWS](#aws) + - [v0.12.6 => v0.13.x](#v0126--v013x-1) + - [v0.13.x => v0.14.x](#v013x--v014x-1) + - [v0.14.x => v1.0.x](#v014x--v10x-1) + - [v1.0.x => v1.1.3](#v10x--v113-1) ## Introduction From Epicli 0.4.2 and up the CLI has the ability to perform upgrades on certain components on a cluster. The components From fb9d1f666904fa33cd45d4dd0f4ba431e79b8a6d Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 16 Feb 2022 22:03:25 +0100 Subject: [PATCH 054/157] Correction of renames which were not needed --- docs/home/howto/MONITORING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index c717c1f52b..acdb8dfe3b 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -307,7 +307,7 @@ To change `admin` user's password, change value for `admin_password` key. For `k and `logstash_password` keys respectively. ```yaml -kind: configuration/opensearch-for-opensearch +kind: configuration/opendistro-for-elasticsearch title: OpenSearch Config name: default specification: From c6b68acfcb745a92e805211a172991860c5f5415 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 17 Feb 2022 12:01:46 +0100 Subject: [PATCH 055/157] Docs update --- docs/changelogs/CHANGELOG-0.10.md | 2 +- docs/changelogs/CHANGELOG-0.5.md | 2 +- docs/changelogs/CHANGELOG-0.9.md | 2 +- docs/home/HOWTO.md | 4 +- docs/home/howto/DATABASES.md | 18 ++++---- docs/home/howto/LOGGING.md | 45 +++++++++++-------- docs/home/howto/MONITORING.md | 72 +++++++++++++++++-------------- 7 files changed, 79 insertions(+), 66 deletions(-) diff --git a/docs/changelogs/CHANGELOG-0.10.md b/docs/changelogs/CHANGELOG-0.10.md index 30201c9086..3c51518385 100644 --- a/docs/changelogs/CHANGELOG-0.10.md +++ b/docs/changelogs/CHANGELOG-0.10.md @@ -50,7 +50,7 @@ Version 0.10 won't be supported anymore. Instead, we introduced version 1.0 whic ### Breaking changes -- Feature `elasticsearch` removed in favor of `opensearch-for-elasticsearch`. +- Feature `elasticsearch` removed in favor of `opendistro-for-elasticsearch`. ### Known issues diff --git a/docs/changelogs/CHANGELOG-0.5.md b/docs/changelogs/CHANGELOG-0.5.md index 86fa021746..82308f6dca 100644 --- a/docs/changelogs/CHANGELOG-0.5.md +++ b/docs/changelogs/CHANGELOG-0.5.md @@ -83,7 +83,7 @@ - [#782](https://github.com/epiphany-platform/epiphany/issues/781) - All disks encryption documentation - AWS - [#782](https://github.com/epiphany-platform/epiphany/issues/782) - All disks encryption documentation - Azure - [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for ElasticSearch - - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch) + - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) - [Centralized logging](/docs/home/howto/LOGGING.md#centralized-logging-setup) - [#755](https://github.com/epiphany-platform/epiphany/issues/755) - Create Ansible playbook to install Apache Ignite as a service on VM diff --git a/docs/changelogs/CHANGELOG-0.9.md b/docs/changelogs/CHANGELOG-0.9.md index 1321890269..d0d420d3c2 100644 --- a/docs/changelogs/CHANGELOG-0.9.md +++ b/docs/changelogs/CHANGELOG-0.9.md @@ -38,7 +38,7 @@ ### Deprecated -- Elasticsearch OSS v6 (feature name: `elasticsearch`), succesor: Elasticsearch OSS v7 (feature name: `opensearch-for-elasticsearch`). It may be removed in the next major release. +- Elasticsearch OSS v6 (feature name: `elasticsearch`), succesor: Elasticsearch OSS v7 (feature name: `opendistro-for-elasticsearch`). It may be removed in the next major release. ### Breaking changes diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index a8b546730b..fe9ddacbf6 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -91,8 +91,8 @@ - [How to switchover database nodes](./howto/DATABASES.md#how-to-switchover-database-nodes) - [How to set up PGBouncer, PgPool and PostgreSQL parameters](./howto/DATABASES.md#how-to-set-up-pgbouncer-pgpool-and-postgresql-parameters) - [How to set up PostgreSQL audit logging](./howto/DATABASES.md#how-to-set-up-postgresql-audit-logging) - - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch) - - [How to manage Opendistro for Elasticsearch data](./howto/LOGGING.md#how-to-manage-opensearch-for-elasticsearch-data) + - [How to start working with OpenDistro for Elasticsearch](./howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch) + - [How to manage Opendistro for Elasticsearch data](./howto/LOGGING.md#how-to-manage-opendistro-for-elasticsearch-data) - [Backup and Recovery](./howto/BACKUP.md) - [Epiphany backup and restore](./howto/BACKUP.md#epiphany-backup-and-restore) diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 864fe221c0..30e2796119 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -455,11 +455,10 @@ Properly configured application (kubernetes service) to use fully HA configurati PostgreSQL native replication is now deprecated and removed. Use [PostgreSQL HA replication with repmgr](#how-to-set-up-postgresql-ha-replication-with-repmgr-cluster) instead. -## How to start working with OpenDistro for ElasticSearch +## How to start working with Opensearch -OpenDistro for opensearch -is [an Apache 2.0-licensed distribution of Elasticsearch enhanced with enterprise security, alerting, SQL](https://opendistro.github.io/for-elasticsearch/). -In order to start working with OpenDistro change machines count to value greater than 0 in your cluster configuration: +OpenSearch is the [successor](https://opendistro.github.io/for-elasticsearch-docs/) of OpenDistro for ElasticSearch project. Epipahny is providing an [automated solution](https://github.com/romsok24/epiphany/blob/feature/migr-ODFE-OpenSearch-2870/docs/home/howto/UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) for migrating your existing ODFE installation to OpenSearch. +On the other hand, if you plan to just start working with OpenSearch - change machines count to value greater than 0 in your cluster configuration: ```yaml kind: epiphany-cluster @@ -479,18 +478,18 @@ specification: count: 2 ``` -**Installation with more than one node will always be clustered** - Option to configure the non-clustered installation of more than one node for Open Distro is not supported. +**Installation with more than one node will always be clustered** - Option to configure the non-clustered installation of more than one node for OpenSearch is not supported. ```yaml kind: configuration/opendistro-for-elasticsearch -title: OpenDistro for Elasticsearch Config +title: OpenSearch Config name: default specification: cluster_name: EpiphanyOpensearch ``` -By default, Kibana is deployed only for `logging` component. If you want to deploy Kibana -for `opensearch` you have to modify feature mapping. Use below configuration in your manifest. +By default, OpenSearch Dashboards ( previously Kibana component ) is deployed only for `logging` component. If you want to deploy it +for `opensearch` component you have to modify feature mapping. Use below configuration in your manifest: ```yaml kind: configuration/feature-mapping @@ -499,11 +498,10 @@ name: default specification: roles_mapping: opensearch: - - opendistro-for-elasticsearch - node-exporter - filebeat - firewall - kibana ``` -Filebeat running on `opensearch` hosts will always point to centralized logging hosts (./LOGGING.md). +Filebeat running on `opensearch` hosts will always point to centralized logging hosts ( [more info](./LOGGING.md) ). diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index fba7555a4d..97d1104814 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -1,46 +1,50 @@ # Centralized logging setup -For centralized logging Epiphany uses [OpenDistro for Elasticsearch](https://opendistro.github.io/for-elasticsearch/). -In order to enable centralized logging, be sure that `count` property for `logging` feature is greater than 0 in your +For centralized logging Epiphany uses [OpenSearch](https://opensearch.org/) stack - an opensource successor[1] of Elasticsearch & Kibana projects. + +In order to enable centralized logging, be sure to set `count` property for `logging` feature to the value greater than 0 in your configuration manifest. ```yaml kind: epiphany-cluster -... +[...] specification: - ... + [...] components: kubernetes_master: count: 1 kubernetes_node: count: 0 - ... + [...] logging: - count: 1 - ... + count: 1 # <<------ + [...] ``` ## Default feature mapping for logging +Below example shows a default feature mapping for logging: ```yaml -... -logging: - - logging - - kibana - - node-exporter - - filebeat - - firewall +[...] +roles_mapping: +[...] + logging: + - logging + - kibana + - node-exporter + - filebeat + - firewall ... ``` -The `logging` role replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data +The `logging` role has replaced `elasticsearch` role. This change was done to enable Elasticsearch usage also for data storage - not only for logs as it was till 0.5.0. -Default configuration of `logging` and `opensearch` roles is identical ( -./DATABASES.md#how-to-start-working-with-opensearch-for-elasticsearch). To modify configuration of centralized logging -adjust and use the following defaults in your manifest: +Default configuration of `logging` and `opensearch` roles is identical ( more info [here](./DATABASES.md#how-to-start-working-with-opensearch) ). To modify configuration of centralized logging +adjust to your needs the following default values in your manifest: ```yaml +[...] kind: configuration/logging title: Logging Config name: default @@ -269,3 +273,8 @@ specification: ``` Note: Setting `specification.kibana.dashboards.enabled` to `true` not providing Kibana will result in a Filebeat crash. + +
+ +--- +[1] More information about migrating from Elasticsearch & Kibana to OpenSearch & OpenSearch Dashboards can be found [here](./UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards). \ No newline at end of file diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index acdb8dfe3b..6f8b902dcd 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -231,50 +231,50 @@ When dashboard creation or import succeeds you will see it on your dashboard lis *Note: For some dashboards, there is no data to visualize until there is traffic activity for the monitored component.* -# Kibana +# OpenSearch Dashboard -Kibana is an free and open frontend application that sits on top of the Elastic Stack, providing search and data visualization capabilities for data indexed in Elasticsearch. For more informations about Kibana please refer to [the official website](https://www.elastic.co/what-is/kibana). +OpenSearch Dashboards ( a Kibana successor ) is an open source search and analytics visualization layer. It also serves as a user interface for many OpenSearch project plugins. For more information please refer to [the official website](https://opensearch.org/docs/latest/dashboards/index/). -## How to configure Kibana - Open Distro +## How to configure OpenSearch Dashboards -In order to start viewing and analyzing logs with Kibana, you first need to add an index pattern for Filebeat according to the following steps: +In order to start viewing and analyzing logs with Dashboards tool, you first need to add an index pattern for Filebeat according to the following procedure: -1. Goto the `Management` tab -2. Select `Index Patterns` -3. On the first step define as index pattern: +1. Goto the `Stack Management` tab +2. Select `Index Patterns` --> `Create index pattern` +3. Define an index pattern: `filebeat-*` - Click next. + and click next. 4. Configure the time filter field if desired by selecting `@timestamp`. This field represents the time that events occurred or were processed. You can choose not to have a time field, but you will not be able to narrow down your data by a time range. -This filter pattern can now be used to query the Elasticsearch indices. +This filter pattern can now be used to query the OpenSsearch indices. -By default Kibana adjusts the UTC time in `@timestamp` to the browser's local timezone. This can be changed in `Management` > `Advanced Settings` > `Timezone for date formatting`. +By default OpenSearch Dashoboards adjusts the UTC time in `@timestamp` to the browser's local timezone. This can be changed in `Stack Management` > `Advanced Settings` > `Timezone for date formatting`. -## How to configure default user passwords for Kibana - Open Distro, Open Distro for Elasticsearch and Filebeat +## How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat -To configure admin password for Kibana - Open Distro and Open Distro for Elasticsearch you need to follow the procedure below. -There are separate procedures for `logging` and `opensearch-for-elasticsearch` roles since most of the times for `opensearch-for-elasticsearch`, `kibanaserver` and `logstash` users are not required to be present. +To configure admin password for OpenSearch Dashoboards ( previously Kibana ) and OpenSearch you need to follow the procedure below. +There are separate procedures for `logging` and `opensearch` roles since for most of the time `opensearch`, `kibanaserver` and `logstash` users are not required to be present. ### Logging component -#### - Logging role +#### Logging role -By default Epiphany removes users that are listed in `demo_users_to_remove` section of `configuration/logging` doc. -By default, `kibanaserver` user (needed by default Epiphany installation of Kibana) and `logstash` (needed by default Epiphany +By default Epiphany removes users that are listed in `demo_users_to_remove` section of `configuration/logging` manifest document. +Additionally, `kibanaserver`[1] user (needed by default Epiphany installation of Dashooards) and `logstash` user (needed by default Epiphany installation of Filebeat) are not removed. If you want to perform configuration by Epiphany, set `kibanaserver_user_active` to `true` -for `kibanaserver` user or `logstash_user_active` for `logstash` user. For `logging` role, those settings are already set to `true` by default. +for `kibanaserver` user and/or `logstash_user_active` to `true` for `logstash` user. For `logging` role, those settings are already set to `true` by default. We strongly advice to set different password for each user. -To change `admin` user's password, change value for `admin_password` key. For `kibanaserver` and `logstash`, change values -for `kibanaserver_password` and `logstash_password` keys respectively. Changes from logging role will be propagated to Kibana -and Filebeat configuration. +To change `admin` user's password, you need to change the value for `admin_password` key ( see the example below ). For `kibanaserver` and `logstash`, you need to change values +for `kibanaserver_password` and `logstash_password` keys respectively. Changes from logging role will be propagated to OpenSearch Dashboards +and Filebeat configuration accordingly. ```yaml kind: configuration/logging title: Logging Config name: default specification: - ... + [...] admin_password: YOUR_PASSWORD kibanaserver_password: YOUR_PASSWORD kibanaserver_user_active: true @@ -286,32 +286,32 @@ specification: - snapshotrestore ``` -#### - Kibana role +#### OpenSearch Dashboards ( Kibana ) role -To set password of `kibanaserver` user, which is used by Kibana for communication with Open Distro Elasticsearch backend follow the procedure -described in [Logging role](#-logging-role). +To set password for `kibanaserver` user, which is used by Dashboards for communication with OpenSearch backend follow the procedure +described in [Logging role](#logging-role). -#### - Filebeat role +#### Filebeat role To set password of `logstash` user, which is used by Filebeat for communication with Open Distro Elasticsearch backend follow the procedure described in [Logging role](#-logging-role). -### Open Distro for Elasticsearch component +### OpenSearch component By default Epiphany removes all demo users except `admin` user. Those users are listed in `demo_users_to_remove` section -of `configuration/opensearch-for-elasticsearch` doc. If you want to keep `kibanaserver` user (needed by default Epiphany installation of Kibana), -you need to remove it from `demo_users_to_remove` list and set `kibanaserver_user_active` to `true` in order to change the default password. +of `configuration/opensearch` manifest doc ( see example below ). If you want to keep `kibanaserver` user (needed by default Epiphany installation of OpenSearch Dashboards), +you need to exclude it from `demo_users_to_remove` list and set `kibanaserver_user_active` to `true` in order to change the default password. We strongly advice to set different password for each user. -To change `admin` user's password, change value for `admin_password` key. For `kibanaserver` and `logstash`, change values for `kibanaserver_password` +To change `admin` user's password, change value for the `admin_password` key. For `kibanaserver` and `logstash`, change values for `kibanaserver_password` and `logstash_password` keys respectively. ```yaml -kind: configuration/opendistro-for-elasticsearch +kind: configuration/opensearch title: OpenSearch Config name: default specification: - ... + [...] admin_password: YOUR_PASSWORD kibanaserver_password: YOUR_PASSWORD kibanaserver_user_active: false @@ -325,9 +325,15 @@ specification: - kibanaserver ``` -### Upgrade of Elasticsearch, Kibana and Filebeat +### Upgrade of OpenSearch, OpenSearch Dashboards and Filebeat + +Keep in mind taht during the upgrade process Epiphany takes `kibanaserver` (for Dashboards) and `logstash` (for Filebeat) user passwords and re-applies them to upgraded configuration of Filebeat and Kibana. So if these password phrases differ from what was setup before upgrade, you should reflect these changes upon next login process. + + Epiphany upgrade of OpenSearch, OpenSearch Dashboards or Filebeat components will fail if `kibanaserver` or `logstash` usernames were changed in configuration of OpenSearch, OpenSearch Dashboards or Filebeat before. + +
-During upgrade Epiphany takes `kibanaserver` (for Kibana) and `logstash` (for Filebeat) user passwords and re-applies them to upgraded configuration of Filebeat and Kibana. Epiphany upgrade of Open Distro, Kibana or Filebeat will fail if `kibanaserver` or `logstash` usernames were changed in configuration of Kibana, Filebeat or Open Distro for Elasticsearch. +[1] For the backward compatibility needs, some naming conventions ( ie. kibanaserver user name ) are still present within the new ( OpenSearch ) platform though they will be suppresed in the future. In aftermath, Epiphany stack is also still using these names. # HAProxy From 0113f43d25329446a2065b60aceff12bed9d96dd Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 17 Feb 2022 15:30:22 +0100 Subject: [PATCH 056/157] Docs update --- docs/home/howto/LOGGING.md | 105 ++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 97d1104814..942a2caf5b 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -57,67 +57,69 @@ specification: logs: /var/log/opensearch ``` -## How to manage Opendistro for Elasticsearch data - +## How to manage OpenSearch data +OpenSearch Elasticsearch stores data using JSON documents, and an Index is a collection of documents. As in every database, it's crucial to correctly maintain data in this one. It's almost impossible to deliver database configuration which will fit -to every type of project and data stored in. Epiphany deploys preconfigured Opendistro Elasticsearch, but this -configuration may not meet user requirements. Before going to production, configuration should be tailored to the +to every type of project and data stored in. Epiphany deploys preconfigured OpenSearch instance but this +configuration may not meet any single user requirements. That's why, before going to production, stack configuration should be tailored to the project needs. All configuration tips and tricks are available -in [official documentation](https://opendistro.github.io/for-elasticsearch-docs/). +in [official documentation](https://opensearch.org/docs/latest). -The main and most important decisions to take before you deploy cluster are: +The main and most important decisions to take before you deploy the cluster are: -1) How many Nodes are needed -2) How big machines and/or storage data disks need to be used + - how many nodes are needed + - how big machines and/or storage data disks need to be used -These parameters are defined in yaml file, and it's important to create a big enough cluster. +These parameters can be defined in manifest yaml file. It is important to create a big enough cluster. ```yaml specification: + [..] components: logging: - count: 1 # Choose number of nodes + count: 1 # Choose number of nodes that suits your needs + machines: + - logging-machine-n + [..] --- kind: infrastructure/virtual-machine title: "Virtual Machine Infra" -name: logging-machine +name: logging-machine-n specification: - size: Standard_DS2_v2 # Choose machine size + size: Standard_DS2_v2 # Choose a VM size that suits your needs ``` -If it's required to have Elasticsearch which works in cluster formation configuration, except setting up more than one +If it's required to have OpenSearch instance which works in cluster formation configuration, except setting up more than one machine in yaml config file please acquaint dedicated support [article](https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/cluster/) and adjust Elasticsearch configuration file. -At this moment Opendistro for Elasticsearch does not support plugin similar -to [ILM](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html), log rotation -is possible only by configuration created in Index State Management. +We also want to strongly encourage you to get familiar with a bunch of plugins and policies available along with OpenSearch with the following ones among them: -`ISM - Index State Management` - is a plugin that provides users and administrative panel to monitor the indices and +`ISM - Index State Management` - is a plugin that allows users and administrative panel to monitor the indices and apply policies at different index stages. ISM lets users automate periodic, administrative operations by triggering them based on index age, size, or number of documents. Using the ISM plugin, can define policies that automatically handle -index rollovers or deletions. ISM is installed with Opendistro by default - user does not have to enable this. Official +index rollovers or deletions. Official plugin documentation is available -in [Opendistro for Elasticsearch website](https://opendistro.github.io/for-elasticsearch-docs/docs/im/ism/). +[here](https://opensearch.org/docs/latest/im-plugin/ism/index/). To reduce the consumption of disk resources, every index you created should use -well-designed [policy](https://opendistro.github.io/for-elasticsearch-docs/docs/im/ism/policies/). +well-designed [policy](https://opensearch.org/docs/latest/im-plugin/ism/policies/). Among others these two index actions might save machine from filling up disk space: -[`Index Rollover`](https://opendistro.github.io/for-elasticsearch-docs/docs/im/ism/policies/#rollover) - rolls an alias +[`Index Rollover`](https://opensearch.org/docs/latest/im-plugin/ism/policies/#rollover) - rolls an alias to a new index. Set up correctly max index size / age or minimum number of documents to keep index size in requirements framework. -[`Index Deletion`](https://opendistro.github.io/for-elasticsearch-docs/docs/im/ism/policies/#delete) - deletes indexes +[`Index Deletion`](https://opensearch.org/docs/latest/im-plugin/ism/policies/#delete) - deletes indexes managed by policy -Combining these actions, adapting them to data amount and specification users are able to create policy which will -maintain data in cluster for example: to secure node from fulfilling disk space. +Combining these actions and adapting them to data amount and specification, users are able to create policy which will +maintain their data in cluster for example to secure node from fulfilling disk space. -There is example of policy below. Be aware that this is only example, and it needs to be adjusted to environment needs. +There is an example of such policy below. Be aware that this is only example and as avery example it needs to be adjusted to actual environment needs. ```json { @@ -185,64 +187,66 @@ There is example of policy below. Be aware that this is only example, and it nee } ``` -Example above shows configuration with rollover daily or when index achieve 1 GB size. Indexes older than 14 days will +Example above shows configuration with rollover index policy on a daily basis or when the index achieve 1 GB size. Indexes older than 14 days will be deleted. States and conditionals could be combined. Please -see [policies](https://opendistro.github.io/for-elasticsearch-docs/docs/im/ism/policies/) documentation for more +see [policies](https://opensearch.org/docs/latest/im-plugin/ism/policies//) documentation for more details. -`Apply Policy` +
+ +#### Apply Policy -To apply policy use similar API request as presented below: +To apply a policy you can use similar API request as presented below: ``` -PUT _template/template_01 +PUT _index_template/ism_rollover ``` ```json { "index_patterns": ["filebeat*"], "settings": { - "opendistro.index_state_management.rollover_alias": "filebeat" - "opendistro.index_state_management.policy_id": "epi_policy" + "plugins.index_state_management.rollover_alias": "filebeat" + "plugins.index_state_management.policy_id": "epi_policy" } } ``` After applying this policy, every new index created under this one will apply to it. There is also possibility to apply -policy to already existing policies by assigning them to policy in Index Management Kibana panel. +policy to already existing policies by assigning them to policy in dashboard Index Management panel. -## How to export Kibana reports to CSV format +## How to export Dashboards reports -Since v1.0 Epiphany provides the possibility to export reports from Kibana to CSV, PNG or PDF using the Open Distro for -Elasticsearch Kibana reports feature. +Since v1.0 Epiphany provides the possibility to export reports from Kibana to CSV, PNG or PDF using the Open Distro for Elasticsearch Kibana reports feature. And after migrating from Elastic stack to OpenSearch stack you can make use of the OpenSearch Reporting feature a choieve this and more. -Check more details about the plugin and how to export reports in the -[documentation](https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/reporting) +Check more details about the OpenSearch Reports plugin and how to export reports in the +[documentation](https://github.com/opensearch-project/dashboards-reports/blob/main/README.md#opensearch-dashboards-reports). -`Note: Currently in Open Distro for Elasticsearch Kibana the following plugins are installed and enabled by default: security, alerting, anomaly detection, index management, query workbench, notebooks, reports, alerting, gantt chart plugins.` +Notice: Currently in the OpenSearch stack the following plugins are installed and enabled by default: security, alerting, anomaly detection, index management, query workbench, notebooks, reports, alerting, gantt chart plugins. -You can easily check enabled default plugins for Kibana using the following command on the logging machine: -`./bin/kibana-plugin list` in Kibana directory. +You can easily check enabled default plugins for Dashboards component using the following command on the logging machine: +`./bin/opensearch-dashboards-plugin list` in directory where you've installed _opensearch-dashboards_. --- ## How to add multiline support for Filebeat logs -In order to properly handle multilines in files harvested by Filebeat you have to provide `multiline` definition in the -configuration manifest. Using the following code you will be able to specify which lines are part of a single event. +In order to properly handle multiline outputs in files harvested by Filebeat you have to provide `multiline` definition in the cluster configuration manifest. Using the following code you will be able to specify which lines are part of a single event. By default, postgresql block is provided, you can use it as example: ```yaml +[..] postgresql_input: multiline: pattern: >- '^\d{4}-\d{2}-\d{2} ' negate: true match: after +[..] ``` -Supported inputs: `common_input`,`postgresql_input`,`container_input` +Supported inputs: `common_input`,`postgresql_input`,`container_input`. More details about multiline options you can find in the [official documentation](https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html) @@ -257,22 +261,27 @@ specification: k8s_as_cloud_service: true ``` -## How to use default Kibana dashboards +## How to use default OpenSearch dashboards + +--- +This feature is not working in current version of OpenSearch and so the `setup.dashboards.enabled` is set with value _false_ as a workaround. +--- It is possible to configure `setup.dashboards.enabled` and `setup.dashboards.index` Filebeat settings using `specification.kibana.dashboards` key in `configuration/filebeat` doc. -When `specification.kibana.dashboards.enabled` is set to `auto`, the corresponding setting in Filebeat configuration file will be set to `true` only if Kibana is configured to be present on the host. +When `specification.kibana.dashboards.enabled` is set to `auto`, the corresponding setting in Filebeat configuration file will be set to `true` only if OpenSearch Dashboards component is configured to be present on the host. Other possible values are `true` and `false`. Default configuration: -``` +```yaml specification: +[..] kibana: dashboards: enabled: auto index: filebeat-* ``` -Note: Setting `specification.kibana.dashboards.enabled` to `true` not providing Kibana will result in a Filebeat crash. +Notice: Setting `specification.kibana.dashboards.enabled` to `true` not providing Kibana will result in a Filebeat crash.
From f484e30f02df0944bc14b7bd1085bb7230dfbb80 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 17 Feb 2022 15:48:18 +0100 Subject: [PATCH 057/157] Docs update-correcting the docs links and names --- .../roles/opensearch/tasks/install-ops-plugins.yml | 1 - docs/home/HOWTO.md | 4 ++-- docs/home/howto/MONITORING.md | 12 ++++++------ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml index c5802ac6c6..4be7b41e84 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml @@ -1,5 +1,4 @@ --- -# NOTE: Keep in mind compatibility matrix for Open Distro https://opendistro.github.io/for-elasticsearch-docs/docs/install/plugins/#plugin-compatibility - name: Install opensearch-* packages package: name: "{{ _packages[ansible_os_family] }}" diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index fe9ddacbf6..f3767ce8e5 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -34,8 +34,8 @@ - [How to configure scalable Prometheus setup](./howto/MONITORING.md#how-to-configure-scalable-prometheus-setup) - [Import and create Grafana dashboards](./howto/MONITORING.md#import-and-create-grafana-dashboards) - [How to setup default admin password and user in Grafana](./howto/MONITORING.md#how-to-setup-default-admin-password-and-user-in-grafana) - - [How to configure Kibana - Open Distro](./howto/MONITORING.md#how-to-configure-kibana---open-distro) - - [How to configure default user passwords for Kibana - Open Distro, Open Distro for Elasticsearch and Filebeat](./howto/MONITORING.md#how-to-configure-default-user-passwords-for-kibana---open-distro-open-distro-for-elasticsearch-and-filebeat) + - [How to configure Dashboards](./howto/MONITORING.md#how-to-configure-opensearch-dashboards) + - [How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat](./howto/MONITORING.md#how-to-configure-default-passwords-for-service-users-in-opensearch-dashboards-opensearch-and-filebeat) - [How to configure scalable Prometheus setup](./howto/MONITORING.md#how-to-configure-scalable-prometheus-setup) - [How to configure Azure additional monitoring and alerting](./howto/MONITORING.md#how-to-configure-azure-additional-monitoring-and-alerting) - [How to configure AWS additional monitoring and alerting](./howto/MONITORING.md#how-to-configure-aws-additional-monitoring-and-alerting) diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index 6f8b902dcd..b2ba767f4b 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -11,10 +11,10 @@ Grafana: - [How to setup default admin password and user in Grafana](#how-to-setup-default-admin-password-and-user-in-grafana) - [Import and create Grafana dashboards](#import-and-create-grafana-dashboards) -Kibana: +OpenSearch Dashboards: -- [How to configure Kibana](#how-to-configure-kibana) -- [How to configure default user password in Kibana](#how-to-configure-default-user-password-in-kibana) +- [How to configure Dashboards](#how-to-configure-opensearch-dashboards) +- [How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat](#how-to-configure-default-passwords-for-service-users-in-opensearch-dashboards-opensearch-and-filebeat) RabbitMQ: @@ -231,7 +231,7 @@ When dashboard creation or import succeeds you will see it on your dashboard lis *Note: For some dashboards, there is no data to visualize until there is traffic activity for the monitored component.* -# OpenSearch Dashboard +# OpenSearch Dashboards OpenSearch Dashboards ( a Kibana successor ) is an open source search and analytics visualization layer. It also serves as a user interface for many OpenSearch project plugins. For more information please refer to [the official website](https://opensearch.org/docs/latest/dashboards/index/). @@ -288,12 +288,12 @@ specification: #### OpenSearch Dashboards ( Kibana ) role -To set password for `kibanaserver` user, which is used by Dashboards for communication with OpenSearch backend follow the procedure +To set password for `kibanaserver` user, which is used by Dashboards for communication with OpenSearch Dashboards backend follow the procedure described in [Logging role](#logging-role). #### Filebeat role -To set password of `logstash` user, which is used by Filebeat for communication with Open Distro Elasticsearch backend follow the procedure described +To set password of `logstash` user, which is used by Filebeat for communication with OpenSearch Dashboards backend follow the procedure described in [Logging role](#-logging-role). ### OpenSearch component From 47f0212e20e6f3725b7b4e01a2952ae35851a57b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 17 Feb 2022 17:37:28 +0100 Subject: [PATCH 058/157] Docs update --- README.md | 4 ++-- docs/architecture/logical-view.md | 12 ++++++------ docs/architecture/process-view.md | 4 ++-- docs/home/RESOURCES.md | 4 ++-- docs/home/SECURITY.md | 18 +++++++++++------- docs/home/howto/CLUSTER.md | 2 +- docs/home/howto/DATABASES.md | 2 +- docs/home/howto/LOGGING.md | 7 +++---- docs/home/howto/MAINTENANCE.md | 4 ++-- 9 files changed, 30 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index b996c4b75d..7fe94fdcc5 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ Epiphany at its core is a full automation of Kubernetes and Docker plus addition - Kafka or RabbitMQ for high speed messaging/events - Prometheus and Alertmanager for monitoring with Graphana for visualization -- Elasticsearch and Kibana for centralized logging (OpenDistro) +- OpenSearch for centralized logging - HAProxy for loadbalancing -- Postgres and Elasticsearch for data storage +- Postgres and OpenSearch for data storage - KeyCloak for authentication - Helm as package manager for Kubernetes diff --git a/docs/architecture/logical-view.md b/docs/architecture/logical-view.md index 7a33aca8be..0aabcaa0ea 100644 --- a/docs/architecture/logical-view.md +++ b/docs/architecture/logical-view.md @@ -51,14 +51,14 @@ Source | Purpose /var/log/zookeeper/version-2/* | Zookeeper's logs Docker containers | Kubernetes components that run in a container -`Filebeat`, unlike `Grafana`, pushes data to database (`Elasticsearch`) instead of pulling them. +`Filebeat`, unlike `Grafana`, pushes data to database (`OpenSearch`) instead of pulling them. [Read more](https://www.elastic.co/products/beats/filebeat) about `Filebeat`. -### opensearch +### OpenSearch -`Elasticsearch` is highly scalable and full-text search enabled analytics engine. Epiphany Platform uses it for storage and analysis of logs. +`OpenSearch` is highly scalable and full-text search enabled analytics engine. Epiphany Platform uses it for storage and analysis of logs. -[Read more](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/index.html) +[Read more](https://opensearch.org/docs/latest) ### Elasticsearch Curator @@ -66,9 +66,9 @@ Docker containers | Kubernetes components that run in a container [Read more](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.8/index.html) -### Kibana +### OpenSearch Dashboards -`Kibana` like `Grafana` is used in Epiphany for visualization, in addition it has full text search capabilities. `Kibana` uses `Elasticsearch` as datasource for logs, it allows to create full text queries, dashboards and analytics that are performed on logs. +`OpenSearch Dashboards` like `Grafana` is used in Epiphany for visualization,It uses `OpenSearch` as datasource for logs, it allows to create full text queries, dashboards and analytics that are performed on logs. [Read more](https://www.elastic.co/products/kibana) diff --git a/docs/architecture/process-view.md b/docs/architecture/process-view.md index 366bb2ee83..a124c7fd16 100644 --- a/docs/architecture/process-view.md +++ b/docs/architecture/process-view.md @@ -24,8 +24,8 @@ metrics from different kinds of exporters. ## Logging -Epiphany uses `Elasticsearch` as key-value database with `Filebeat` for gathering logs and `Kibana` as user interface to write queries and analyze logs. +Epiphany uses `OpenSearch` as key-value database with `Filebeat` for gathering logs and `OpenSearch Dashboards` as user interface to write queries and analyze logs. ![Logging process view](diagrams/process-view/logging-process-view.svg) -`Filebeat` gathers OS and application logs and ships them to `Elasticsearch`. Queries from `Kibana` are run against `Elasticsearch` key-value database. \ No newline at end of file +`Filebeat` gathers OS and application logs and ships them to `OpenSearch`. Queries from `Kibana` are run against `OpenSearch` key-value database. \ No newline at end of file diff --git a/docs/home/RESOURCES.md b/docs/home/RESOURCES.md index 03dac4c716..75adb34694 100644 --- a/docs/home/RESOURCES.md +++ b/docs/home/RESOURCES.md @@ -42,8 +42,8 @@ Here are some materials concerning Epiphany tooling and cluster components - bot 2. [RabbitMQ](https://www.rabbitmq.com/) - [RabbitMQ Getting started](https://www.rabbitmq.com/getstarted.html) 5. Central logging - 1. [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) - 2. [Kibana](https://www.elastic.co/guide/en/kibana/current/index.html) + 1. [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/index/) + 2. [OpenSearch](https://opensearch.org/docs/latest) 3. [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/index.html) - Beats platform reference(https://www.elastic.co/guide/en/beats/libbeat/current/index.html) 6. Load Balancing diff --git a/docs/home/SECURITY.md b/docs/home/SECURITY.md index e66633969e..df2fe12407 100644 --- a/docs/home/SECURITY.md +++ b/docs/home/SECURITY.md @@ -11,8 +11,12 @@ changes made in settings of your antivirus/antimalware solution. ## Contents -- [Users and roles created by Epiphany](#users-and-roles-created-by-epiphany) -- [Ports used by components in Epiphany](#ports-used-by-components-in-epiphany) +- [Security related information](#security-related-information) + - [Contents](#contents) + - [Users and roles created by epiphany](#users-and-roles-created-by-epiphany) + - [Ports used by components in Epiphany](#ports-used-by-components-in-epiphany) + - [Connection protocols and ciphers used by components in Epiphany](#connection-protocols-and-ciphers-used-by-components-in-epiphany) + - [Notes](#notes) ### Users and roles created by epiphany @@ -61,15 +65,15 @@ different values. The list does not include ports that are bound to the loopback - 9093 - encrypted communication (if TLS/SSL is enabled) - unconfigurable random port from ephemeral range - JMX (for local access only), see note [[1]](#notes) -5. Elasticsearch: +5. OpenSearch: - - 9200 - Elasticsearch REST communication - - 9300 - Elasticsearch nodes communication + - 9200 - OpenSearch REST communication + - 9300 - OpenSearch nodes communication - 9600 - Performance Analyzer (REST API) -6. Kibana: +6. OpenSearch Dashboards: - - 5601 - Kibana web UI + - 5601 - OpenSearch Dashboards web UI 7. Prometheus: diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index a0be06e2ff..dcba84f4c3 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -820,7 +820,7 @@ Kubernetes master | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check Kubernetes node | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | [#1580](https://github.com/epiphany-platform/epiphany/issues/1580) Kafka | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Load Balancer | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | --- -Opendistro for elasticsearch | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- +OpenSearch | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | --- Postgresql | :x: | :x: | :heavy_check_mark: | :heavy_check_mark: | [#1577](https://github.com/epiphany-platform/epiphany/issues/1577) RabbitMQ | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | [#1578](https://github.com/epiphany-platform/epiphany/issues/1578), [#1309](https://github.com/epiphany-platform/epiphany/issues/1309) RabbitMQ K8s | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | [#1486](https://github.com/epiphany-platform/epiphany/issues/1486) diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 30e2796119..5ea1228b63 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -481,7 +481,7 @@ specification: **Installation with more than one node will always be clustered** - Option to configure the non-clustered installation of more than one node for OpenSearch is not supported. ```yaml -kind: configuration/opendistro-for-elasticsearch +kind: configuration/opensearch title: OpenSearch Config name: default specification: diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 942a2caf5b..0749920d5c 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -58,8 +58,7 @@ specification: ``` ## How to manage OpenSearch data -OpenSearch -Elasticsearch stores data using JSON documents, and an Index is a collection of documents. As in every database, it's +OpenSearch stores data using JSON documents, and an Index is a collection of documents. As in every database, it's crucial to correctly maintain data in this one. It's almost impossible to deliver database configuration which will fit to every type of project and data stored in. Epiphany deploys preconfigured OpenSearch instance but this configuration may not meet any single user requirements. That's why, before going to production, stack configuration should be tailored to the @@ -92,8 +91,8 @@ specification: If it's required to have OpenSearch instance which works in cluster formation configuration, except setting up more than one machine in yaml config file please acquaint dedicated -support [article](https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/cluster/) and adjust -Elasticsearch configuration file. +support [article](https://opensearch.org/docs/latest/troubleshoot/index/) and adjust +OpenSearch configuration file. We also want to strongly encourage you to get familiar with a bunch of plugins and policies available along with OpenSearch with the following ones among them: diff --git a/docs/home/howto/MAINTENANCE.md b/docs/home/howto/MAINTENANCE.md index 765ac61097..c80c20f150 100644 --- a/docs/home/howto/MAINTENANCE.md +++ b/docs/home/howto/MAINTENANCE.md @@ -114,7 +114,7 @@ status prometheus-node-exporter #### - opensearch -To check status of Elasticsearch we can use the command: +To check status of OpenSearch we can use the command: ```shell systemct status opensearch @@ -132,7 +132,7 @@ We can also check if service is listening on 9300 (nodes communication port): netstat -antup | grep 9300 ``` -We can also check status of Elasticsearch cluster: +We can also check status of OpenSearch cluster: ```shell :9200/_cluster/health From 63c19a2eb165c368b7778947aad5e13caf679a34 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 17 Feb 2022 22:57:24 +0100 Subject: [PATCH 059/157] Removed as target OPS ver includes log4j patch https://discuss.opendistrocommunity.dev/t/log4j-patch-for-cve-2021-45105/8119 --- .../roles/opensearch/tasks/configure-ops.yml | 3 --- .../roles/upgrade/tasks/opensearch-01.yml | 14 -------------- 2 files changed, 17 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index bd451f3995..fe573c732d 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -107,9 +107,6 @@ file: "{{ is_upgrade_run | ternary('remove-known-demo-certs.yml', 'remove-demo-certs.yml') }}" when: not certificates.files.demo.opensearch_security.allow_unsafe_democertificates -# - name: Include log4j patch -# include_tasks: patch-log4j.yml - - name: Restart opensearch service systemd: name: opensearch diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index 3414cfbd3a..ce091650b9 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -38,20 +38,6 @@ - "{{ ops_vars.specification.paths.ops_conf_dir }}" - "{{ ops_defaults.certificates.dirs.certs }}" -# TODO: Remove this part of code -# - name: OPS | Patch log4j -# include_role: -# name: opensearch -# tasks_from: patch-log4j -# when: ops_defaults.log4j_file_name is defined - -# - name: Restart elasticsearch service -# systemd: -# name: elasticsearch -# state: restarted -# register: restart_opensearch -# when: ops_defaults.log4j_file_name is defined and log4j_patch.changed - - name: OPS | Print elasticsearch ond opensearch versions debug: msg: From 01392dd6a01dff632cb501911c2f6f40aba00795 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 13:44:36 +0100 Subject: [PATCH 060/157] Naming corrected --- .../opensearch/tasks/install-ops-plugins.yml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml index 4be7b41e84..3cedf80d3d 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml @@ -6,19 +6,19 @@ vars: _packages: Debian: - - opensearch-alerting={{ versions[ansible_os_family].opendistro }} - - opensearch-index-management={{ versions[ansible_os_family].opendistro }} - - opensearch-job-scheduler={{ versions[ansible_os_family].opendistro }} - - opensearch-performance-analyzer={{ versions[ansible_os_family].opendistro }} - - opensearch-security={{ versions[ansible_os_family].opendistro }} - - opensearch-sql={{ versions[ansible_os_family].opendistro }} + - opensearch-alerting={{ versions[ansible_os_family].ops_version }} + - opensearch-index-management={{ versions[ansible_os_family].ops_version }} + - opensearch-job-scheduler={{ versions[ansible_os_family].ops_version }} + - opensearch-performance-analyzer={{ versions[ansible_os_family].ops_version }} + - opensearch-security={{ versions[ansible_os_family].ops_version }} + - opensearch-sql={{ versions[ansible_os_family].ops_version }} RedHat: - - opensearch-alerting-{{ versions[ansible_os_family].opendistro }} - - opensearch-index-management-{{ versions[ansible_os_family].opendistro }} - - opensearch-job-scheduler-{{ versions[ansible_os_family].opendistro }} - - opensearch-performance-analyzer-{{ versions[ansible_os_family].opendistro }} - - opensearch-security-{{ versions[ansible_os_family].opendistro }} - - opensearch-sql-{{ versions[ansible_os_family].opendistro }} - register: install_opendistro_packages + - opensearch-alerting-{{ versions[ansible_os_family].ops_version }} + - opensearch-index-management-{{ versions[ansible_os_family].ops_version }} + - opensearch-job-scheduler-{{ versions[ansible_os_family].ops_version }} + - opensearch-performance-analyzer-{{ versions[ansible_os_family].ops_version }} + - opensearch-security-{{ versions[ansible_os_family].ops_version }} + - opensearch-sql-{{ versions[ansible_os_family].ops_version }} + register: install_opensearch_packages module_defaults: yum: { lock_timeout: "{{ yum_lock_timeout }}" } From e18882537f72fe34a9703c91bac44f1fe16c5532 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 13:46:35 +0100 Subject: [PATCH 061/157] Reenabling ater test --- ansible/playbooks/roles/opensearch/tasks/main.yml | 2 +- .../playbooks/roles/opensearch/tasks/patch-log4j.yml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index 8d73a431da..979cd2fd9d 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -18,7 +18,7 @@ - include_tasks: install-ops.yml -# - include_tasks: install-ops-plugins.yml +- include_tasks: install-ops-plugins.yml - name: Include configuration tasks include_tasks: configure-ops.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml index 01578552f6..1947ae27b8 100644 --- a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml +++ b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml @@ -61,9 +61,8 @@ _archive_root_dir: >- {{ unarchive_list_files.files | first | dirname }} -# TODO: add instllation of PerfAn to opensearch role -# - name: Restart opensearch-performance-analyzer service -# systemd: -# name: opensearch-performance-analyzer -# state: restarted -# when: log4j_patch.changed +- name: Restart opensearch-performance-analyzer service + systemd: + name: opensearch-performance-analyzer + state: restarted + when: log4j_patch.changed From d24d7313b612a260d93254c6a109df4eebab78db Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 13:48:43 +0100 Subject: [PATCH 062/157] OPS PerfTop function added --- .../playbooks/roles/opensearch/defaults/main.yml | 4 +++- .../roles/opensearch/tasks/install-ops.yml | 15 +++++++++++++-- .../common/defaults/configuration/opensearch.yml | 1 + 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index ebf71aab64..3ef54213b8 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -4,10 +4,12 @@ versions: RedHat: ops_version: "1.2.4" + ops_perftop_version: "1.1.0.0" Debian: ops_version: "1.2.4" + ops_perftop_version: "1.1.0.0" certificates: - dirs: # must be under the config directory, specified using a relative path + dirs: certs: /usr/share/opensearch/config ca_key: /usr/share/opensearch/config csr: /usr/share/opensearch/config diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index f496cbd6ba..98c35c670f 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -4,7 +4,10 @@ name: download tasks_from: download_file vars: - file_name: "opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" + file_name: "{{ item }}" + with_items: + - "opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" + - "opensearch-perf-top-{{ versions[ansible_os_family].ops_perftop_version }}-linux-x64.zip" - name: Ensure Opensearch service user exists user: @@ -20,13 +23,14 @@ group: "{{ specification.ops_user }}" with_items: - "{{ specification.paths.ops_home }}" + - "{{ specification.paths.ops_perftop_home }}" - "{{ specification.paths.ops_log_dir }}" - "{{ specification.paths.ops_conf_dir }}" - "{{ specification.paths.ops_data }}" - "{{ specification.paths.ops_logs }}" - "{{ certificates.dirs.certs }}" -- name: Extract the tar file +- name: Extract the OPS tar file unarchive: src: "/tmp/opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" dest: "{{ specification.paths.ops_home }}" @@ -35,6 +39,13 @@ extra_opts: - --strip-components=1 +- name: Extract OPS PerfTop the tar file + unarchive: + src: "/tmp/opensearch-perf-top-{{ versions[ansible_os_family].ops_perftop_version }}-linux-x64.zip" + dest: "{{ specification.paths.ops_perftop_home }}" + owner: "{{ specification.ops_user }}" + remote_src: yes + - name: Create systemd service template: src: roles/opensearch/templates/opensearch.service.j2 diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 7eadc7620c..139f047486 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -25,6 +25,7 @@ specification: ops_repo: /var/lib/opensearch-snapshots ops_data: /var/lib/opensearch ops_logs: /var/log/opensearch + ops_perftop_home: /usr/share/opensearch/perftop opsd_home: /usr/share/opensearch-dashboards jvm_options: Xmx: 1g From 26f9cc2a8c6a3c64e9f269c2f472f91db960292b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 14:09:12 +0100 Subject: [PATCH 063/157] Mistakes correction --- docs/changelogs/CHANGELOG-0.10.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelogs/CHANGELOG-0.10.md b/docs/changelogs/CHANGELOG-0.10.md index 3c51518385..b5e772c48b 100644 --- a/docs/changelogs/CHANGELOG-0.10.md +++ b/docs/changelogs/CHANGELOG-0.10.md @@ -12,12 +12,12 @@ Version 0.10 won't be supported anymore. Instead, we introduced version 1.0 whic - [#1798](https://github.com/epiphany-platform/epiphany/issues/1798) - Additional alerts for Prometheus - [#1355](https://github.com/epiphany-platform/epiphany/issues/1355) - Updating cloud based OS images - configuration required for Azure RHEL LVM images - [#2081](https://github.com/epiphany-platform/epiphany/issues/2081) - Replace Skopeo with Crane -- [#1323](https://github.com/epiphany-platform/epiphany/issues/1323) - Documentation how to handle data in Opendistro for opensearch +- [#1323](https://github.com/epiphany-platform/epiphany/issues/1323) - Documentation how to handle data in Opendistro for Elasticsearch - [#1789](https://github.com/epiphany-platform/epiphany/issues/1789) - [Ubuntu] Add retry feature for downloading packages in download-requirements.sh ### Fixed -- [#1870](https://github.com/epiphany-platform/epiphany/issues/1870) - Do not install Filebeat when there is no opensearch +- [#1870](https://github.com/epiphany-platform/epiphany/issues/1870) - Do not install Filebeat when there is no Elasticsearch - [#1881](https://github.com/epiphany-platform/epiphany/issues/1881) - epicli: wrong informations in help messages - [#1959](https://github.com/epiphany-platform/epiphany/issues/1959) - Network traffic not allowed from load balancer's subnet to Kubernetes's subnet in AWS - [#1991](https://github.com/epiphany-platform/epiphany/issues/1991) - When custom repo is used backup/recovery stops working From effbfee1f00cbb10758169a5bf848e8ca7fe17f0 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 21:13:22 +0100 Subject: [PATCH 064/157] This compoennt is now part of OPS --- .../opensearch/tasks/install-ops-plugins.yml | 24 ------------------- .../tasks/opensearch/upgrade-plugins.yml | 18 -------------- 2 files changed, 42 deletions(-) delete mode 100644 ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml deleted file mode 100644 index 3cedf80d3d..0000000000 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops-plugins.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Install opensearch-* packages - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: - - opensearch-alerting={{ versions[ansible_os_family].ops_version }} - - opensearch-index-management={{ versions[ansible_os_family].ops_version }} - - opensearch-job-scheduler={{ versions[ansible_os_family].ops_version }} - - opensearch-performance-analyzer={{ versions[ansible_os_family].ops_version }} - - opensearch-security={{ versions[ansible_os_family].ops_version }} - - opensearch-sql={{ versions[ansible_os_family].ops_version }} - RedHat: - - opensearch-alerting-{{ versions[ansible_os_family].ops_version }} - - opensearch-index-management-{{ versions[ansible_os_family].ops_version }} - - opensearch-job-scheduler-{{ versions[ansible_os_family].ops_version }} - - opensearch-performance-analyzer-{{ versions[ansible_os_family].ops_version }} - - opensearch-security-{{ versions[ansible_os_family].ops_version }} - - opensearch-sql-{{ versions[ansible_os_family].ops_version }} - register: install_opensearch_packages - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml deleted file mode 100644 index 0f714a75fb..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-plugins.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: ODFE plugins | Assert that opensearch-* packages are installed - assert: - that: ansible_facts.packages['{{ item }}'] is defined - fail_msg: "Missing package to upgrade: {{ item }}" - quiet: true - loop: - - opensearch-alerting - - opensearch-index-management - - opensearch-job-scheduler - - opensearch-performance-analyzer - - opensearch-security - - opensearch-sql - -- name: ODFE plugins | Upgrade opensearch-* packages - include_role: - name: opensearch - tasks_from: install-ops-plugins.yml From 822b79b8be8c3a6281edf2b3f9b31ba3ee1477d9 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 21:15:18 +0100 Subject: [PATCH 065/157] Update the FS right to the OPS needs --- .../roles/opensearch/tasks/configure-ops.yml | 9 +++--- .../roles/opensearch/tasks/generate-certs.yml | 31 +++++++++---------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index fe573c732d..fdb98aea32 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -14,8 +14,8 @@ backup: yes src: jvm.options.j2 dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" - owner: root - group: opensearch + owner: "{{ specification.ops_user }}" + group: "{{ specification.ops_user }}" mode: ug=rw,o= register: change_jvm_config vars: @@ -36,8 +36,8 @@ backup: yes src: opensearch.yml.j2 dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" - owner: root - group: opensearch + owner: "{{ specification.ops_user }}" + group: "{{ specification.ops_user }}" mode: ug=rw,o= register: change_config vars: @@ -111,6 +111,7 @@ systemd: name: opensearch state: restarted + enabled: yes register: restart_opensearch when: change_config.changed or change_jvm_config.changed diff --git a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml index 6edf0585ef..6d6f9269ef 100644 --- a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml +++ b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml @@ -5,18 +5,18 @@ file: state: directory path: "{{ certificates.dirs.ca_key }}" - owner: root + owner: opensearch group: opensearch - mode: u=rwx,g=rx,o= # elasticsearch.service requires 'rx' for group + mode: u=rwx,g=rwx,o= # csr files are kept only for idempotency - name: Create directory for CSR files file: state: directory path: "{{ certificates.dirs.csr }}" - owner: root + owner: opensearch group: opensearch - mode: u=rwx,g=rx,o= # CSR file doesn't contain private key + mode: u=rwx,g=rwx,o= # CSR file doesn't contain private key - name: Generate keys and certificates on first node when: inventory_hostname == ansible_play_hosts_all[0] @@ -25,20 +25,17 @@ size: 2048 # based on ODFE docs type: RSA mode: u=rw,go= - owner: root - group: opensearch + owner: opensearch format: pkcs8 community.crypto.openssl_csr: mode: u=rw,g=r,o= - owner: root - group: opensearch + owner: opensearch use_common_name_for_san: false community.crypto.x509_certificate: selfsigned_digest: sha256 ownca_digest: sha256 mode: u=rw,g=r,o= - owner: root - group: opensearch + owner: opensearch block: # --- Generate CA root certificate --- @@ -171,9 +168,9 @@ format: pkcs8 size: 2048 type: RSA - mode: u=rw,g=r,o= # elasticsearch.service requires 'r' for group - owner: root - group: opensearch + mode: u=rw,g=r,o= + owner: opensearch + # group: opensearch return_content: false register: node_key @@ -199,8 +196,8 @@ subjectAltName: "{{ _dns_list + [ 'IP:' + ansible_default_ipv4.address ] }}" use_common_name_for_san: false mode: u=rw,g=r,o= - owner: root - group: opensearch + owner: opensearch + # group: opensearch register: node_csr vars: _unique_hostnames: "{{ [ansible_hostname, ansible_nodename, ansible_fqdn] | unique }}" @@ -217,5 +214,5 @@ ownca_not_after: "{{ certificates.files.node.cert.ownca_not_after }}" ownca_digest: sha256 mode: u=rw,go=r - owner: root - group: opensearch + owner: opensearch + # group: opensearch From f92aa58525c3052d8522af2f17b419fdb8807a67 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 21:16:32 +0100 Subject: [PATCH 066/157] This compoennt is now part of OPS --- ansible/playbooks/roles/opensearch/tasks/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index 979cd2fd9d..f4161ce63c 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -18,7 +18,5 @@ - include_tasks: install-ops.yml -- include_tasks: install-ops-plugins.yml - - name: Include configuration tasks include_tasks: configure-ops.yml From d8da2f1fb12ce3cb3ccd7e81365215ead8ed8571 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 21:18:02 +0100 Subject: [PATCH 067/157] Workaround for: opensearch ES_TMPDIR does not exist --- ansible/playbooks/roles/opensearch/templates/jvm.options.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 index d60c44fb1f..43ba467b7a 100644 --- a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 +++ b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 @@ -51,7 +51,9 @@ 14-:-XX:InitiatingHeapOccupancyPercent=30 ## JVM temporary directory --Djava.io.tmpdir=${ES_TMPDIR} +# Workaround for: opensearch[38222]: ERROR: Temporary file directory [${ES_TMPDIR}] does not exist or is not accessible +# -Djava.io.tmpdir=${ES_TMPDIR} +-Djava.io.tmpdir=/tmp ## heap dumps From ebaf52e14766d9d56f07de9f789a5e2f45b5ce89 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 21:19:17 +0100 Subject: [PATCH 068/157] Vars naming unification --- .../tasks/dashboards.yml | 30 +++++++++++-------- .../opensearch_dashboards/tasks/main.yml | 2 +- .../templates/opensearch_dashboards.yml.j2 | 4 +-- .../common/defaults/configuration/logging.yml | 3 +- .../defaults/configuration/opensearch.yml | 3 +- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index 58d723542e..01ecbe09e4 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -1,33 +1,39 @@ --- -- name: Download Opensearch dashbaords {{ versions[ansible_os_family].ops_dashboards_version }} +- name: Download Opensearch dashbaords include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ versions[ansible_os_family].ops_dashboards_version }}-linux-x64.tar.gz" + file_name: "opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" -- name: Create opensearch Dashboards user +- name: Create OpenSearch Dashboards user user: - name: "{{ ops_dashboards_user }}" + name: "{{ specification.opsd_user }}" state: present shell: /bin/bash - name: Create home directory file: - path: "{{ ops_dashboards_home }}" + path: "{{ specification.paths.opsd_home }}" state: directory - owner: "{{ ops_dashboards_user }}" - group: "{{ ops_dashboards_user }}" + owner: "{{ specification.opsd_user }}" + group: "{{ specification.opsd_user }}" -- name: Extract the tar file - command: "chdir=/tmp/ tar -xvzf opensearch-dashboards-{{ versions[ansible_os_family].ops_dashboards_version }}-linux-x64.tar.gz -C {{ ops_dashboards_home }} --strip-components=1" +- name: Extract the OPSD tar file + unarchive: + src: "/tmp/opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + dest: "{{ specification.paths.opsd_home }}" + owner: "{{ specification.opsd_user }}" + remote_src: yes + extra_opts: + - --strip-components=1 - name: Copy Configuration File template: src: opensearch_dashboards.yml.j2 - dest: "{{ ops_dashboards_conf_dir }}/opensearch_dashboards.yml" - owner: "{{ ops_dashboards_user }}" - group: "{{ ops_dashboards_user }}" + dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" + owner: "{{ specification.opsd_user }}" + group: "{{ specification.opsd_user }}" mode: 0644 backup: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 0cb5e1ce22..9fa364121b 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -22,7 +22,7 @@ enabled: yes - name: Get all the installed dashboards plugins - command: "sudo -u {{ ops_dashboards_user }} {{ ops_plugin_bin_path }} list" + command: "sudo -u {{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" register: list_plugins - name: Show all the installed dashboards plugins diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index 5b997e313e..cce08f11d1 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -2,8 +2,8 @@ server.port: 5601 server.host: "{{ hostvars[inventory_hostname]['ansible_host'] }}" opensearch.hosts: ["{{ ops_nodes_dashboards }}"] opensearch.ssl.verificationMode: none -opensearch.username: "{{ ops_dashboards_user }}" -opensearch.password: "{{ ops_dashboards_password }}" +opensearch.username: "{{ specification.opsd_user }}" +opensearch.password: "{{ specification.opsd_password }}" opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] opensearch_security.multitenancy.enabled: true diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index e7b9b914c6..7d0e1f06ad 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -24,9 +24,10 @@ specification: ops_repo: /var/lib/opensearch-snapshots ops_data: /var/lib/opensearch ops_logs: /var/log/opensearch + ops_perftop_home: /usr/share/opensearch/perftop + opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin opsd_home: /usr/share/opensearch-dashboards opsd_conf_dir: /usr/share/opensearch-dashboards/config - opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin jvm_options: Xmx: 1g opensearch_security: diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 139f047486..29779825f3 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -21,12 +21,13 @@ specification: ops_home: /usr/share/opensearch ops_conf_dir: /usr/share/opensearch/config ops_log_dir: /var/log/opensearch - ops_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin ops_repo: /var/lib/opensearch-snapshots ops_data: /var/lib/opensearch ops_logs: /var/log/opensearch ops_perftop_home: /usr/share/opensearch/perftop + opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin opsd_home: /usr/share/opensearch-dashboards + opsd_conf_dir: /usr/share/opensearch-dashboards/config jvm_options: Xmx: 1g opensearch_security: From 2fb3998e6c8645708077940df236268f96a27f9b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 18 Feb 2022 21:27:22 +0100 Subject: [PATCH 069/157] A mistak in naming --- ansible/playbooks/roles/logging/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/logging/tasks/main.yml b/ansible/playbooks/roles/logging/tasks/main.yml index 62dd2319fc..90f22a2fef 100644 --- a/ansible/playbooks/roles/logging/tasks/main.yml +++ b/ansible/playbooks/roles/logging/tasks/main.yml @@ -10,7 +10,7 @@ run_once: true no_log: true # contains sensitive data -- name: Install and configure OpenDistro for opensearch +- name: Install and configure OpenSearch import_role: name: opensearch vars: From 05b01aa7de6e094a08d2ddb0832ae7b515a9179c Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 08:07:11 +0100 Subject: [PATCH 070/157] Correcting names for upperc when aplicable and names changed by mistake previously --- ansible/playbooks/opensearch.yml | 2 +- .../backup/tasks/logging_opensearch_conf.yml | 2 +- .../filebeat/tasks/configure-filebeat.yml | 2 +- .../roles/filebeat/templates/filebeat.yml.j2 | 4 ++-- .../playbooks/roles/logging/tasks/main.yml | 2 +- .../roles/opensearch/defaults/main.yml | 2 +- .../roles/opensearch/tasks/configure-ops.yml | 8 ++++---- .../roles/opensearch/tasks/patch-log4j.yml | 20 +++++++++---------- .../templates/opensearch.service.j2 | 2 +- .../opensearch_dashboards/tasks/main.yml | 2 +- .../templates/dashboards.service | 2 +- .../roles/opensearch_dashboards/vars/main.yml | 2 +- .../roles/upgrade/tasks/opensearch-01.yml | 4 ++-- .../tasks/opensearch/migrate-odfe-serial.yml | 2 +- .../upgrade/tasks/opensearch/migrate-odfe.yml | 6 +++--- docs/changelogs/CHANGELOG-0.5.md | 2 +- docs/changelogs/CHANGELOG-0.8.md | 2 +- docs/changelogs/CHANGELOG-0.9.md | 2 +- docs/design-docs/arm/centos-arm-analysis.md | 12 +++++------ docs/design-docs/arm/redhat-arm-analysis.md | 12 +++++------ docs/design-docs/arm/ubuntu-arm-analysis.md | 12 +++++------ .../health-monitor/health-monitor.md | 2 +- docs/home/howto/MAINTENANCE.md | 2 +- docs/home/howto/RETENTION.md | 2 +- .../configuration/opensearch-dashboards.yml | 2 +- .../validation/configuration/opensearch.yml | 2 +- .../ip_change/elasticsearch/2_config_files.sh | 2 +- 27 files changed, 58 insertions(+), 58 deletions(-) diff --git a/ansible/playbooks/opensearch.yml b/ansible/playbooks/opensearch.yml index 89f753b35a..b4a6e188df 100644 --- a/ansible/playbooks/opensearch.yml +++ b/ansible/playbooks/opensearch.yml @@ -1,5 +1,5 @@ --- -# Ansible playbook for installing opensearch +# Ansible playbook for installing OpenSearch - hosts: opensearch become: true diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml index db9f0655d5..57957c3787 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml @@ -1,5 +1,5 @@ --- -- name: Include default vars from opensearch role +- name: Include default vars from OpenSearch role include_vars: file: roles/opensearch/defaults/main.yml name: ops_def diff --git a/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml b/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml index 6951e11c1f..09d27464db 100644 --- a/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml +++ b/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml @@ -7,7 +7,7 @@ name: postgresql_defaults when: "'postgresql' in group_names" -# Do not select Kibana configured to use ES deployed by 'opensearch' role +# Do not select OPSD configured host to use OPS deployed by 'opensearch' role - name: Set value for setup.kibana.host set_fact: setup_kibana_host: >- diff --git a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index 95ec4ac418..c644398c5f 100644 --- a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -383,14 +383,14 @@ processors: #monitoring.enabled: false # Sets the UUID of the Elasticsearch cluster under which monitoring data for this -# Filebeat instance will appear in the Stack Monitoring UI. If output.opensearch +# Filebeat instance will appear in the Stack Monitoring UI. If output.OpenSearch # is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. #monitoring.cluster_uuid: # Uncomment to send the metrics to Elasticsearch. Most settings from the # Elasticsearch output are accepted here as well. # Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the opensearch +# Any setting that is not set is automatically inherited from the OpenSearch # output configuration, so if you have the Elasticsearch output configured such # that it is pointing to your Elasticsearch monitoring cluster, you can simply # uncomment the following line. diff --git a/ansible/playbooks/roles/logging/tasks/main.yml b/ansible/playbooks/roles/logging/tasks/main.yml index 90f22a2fef..4c615900a2 100644 --- a/ansible/playbooks/roles/logging/tasks/main.yml +++ b/ansible/playbooks/roles/logging/tasks/main.yml @@ -14,4 +14,4 @@ import_role: name: opensearch vars: - specification: "{{ logging_vars.specification }}" # to override opensearch specification + specification: "{{ logging_vars.specification }}" # to override OpenSearch specification diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 3ef54213b8..39d7f4a658 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -25,7 +25,7 @@ certificates: cert: esnode.pem key: esnode-key.pem opensearch_security: - allow_unsafe_democertificates: false # if 'false' all demo files must be removed to start opensearch + allow_unsafe_democertificates: false # if 'false' all demo files must be removed to start OpenSearch common: subject: &common-subject O: Epiphany diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index fdb98aea32..3414eec5cb 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -31,7 +31,7 @@ - include_tasks: generate-certs.yml -- name: Provide opensearch configuration file +- name: Provide OpenSearch configuration file template: backup: yes src: opensearch.yml.j2 @@ -98,7 +98,7 @@ transport_port: "{{ is_upgrade_run | ternary(existing_es_config['transport.port'], ports.transport) }}" # When 'opensearch_security.allow_unsafe_democertificates' is set to 'false' all demo certificate files must be removed, -# otherwise opensearch service doesn't start. +# otherwise OpenSearch service doesn't start. # For apply mode, demo certificate files are removed based only on their names. For upgrade mode, # public key fingerprints are checked to protect against unintentional deletion (what takes additional time). @@ -107,7 +107,7 @@ file: "{{ is_upgrade_run | ternary('remove-known-demo-certs.yml', 'remove-demo-certs.yml') }}" when: not certificates.files.demo.opensearch_security.allow_unsafe_democertificates -- name: Restart opensearch service +- name: Restart OpenSearch service systemd: name: opensearch state: restarted @@ -119,7 +119,7 @@ - name: Change default users when: not is_upgrade_run block: - - name: Wait for opensearch service to start up + - name: Wait for OpenSearch service to start up when: restart_opensearch.changed wait_for: port: 9200 diff --git a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml index 1947ae27b8..b500e2160c 100644 --- a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml +++ b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml @@ -1,7 +1,7 @@ --- - name: Log4j patch block: - - name: "opensearch : Log4j patch | Get archive" + - name: "elasticsearch : Log4j patch | Get archive" include_role: name: download tasks_from: download_file @@ -21,18 +21,18 @@ copy: src: "{{ item.src }}" dest: "{{ item.dest }}" - owner: opensearch + owner: elasticsearch group: root mode: u=rw,g=r,o= remote_src: true loop: - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/lib/ } - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/ } + - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/ } - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/lib/ } - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_slfj_impl }}", dest: /usr/share/elasticsearch/plugins/opensearch_security/ } + - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/ } + - { src: "{{ download_directory }}/{{ log4j_slfj_impl }}", dest: /usr/share/elasticsearch/plugins/elasticsearch_security/ } vars: log4j_api: "{{ unarchive_list_files.files | select('contains', 'log4j-api-2.17.1.jar') | first }}" log4j_core: "{{ unarchive_list_files.files | select('contains', 'log4j-core-2.17.1.jar') | first }}" @@ -45,13 +45,13 @@ state: absent path: "{{ item }}" loop: - - /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/log4j-api-2.13.0.jar - - /usr/share/elasticsearch/plugins/opensearch-performance-analyzer/performance-analyzer-rca/lib/log4j-core-2.13.0.jar + - /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/log4j-api-2.13.0.jar + - /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-api-2.13.0.jar - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - /usr/share/elasticsearch/lib/log4j-api-2.11.1.jar - /usr/share/elasticsearch/lib/log4j-core-2.11.1.jar - - /usr/share/elasticsearch/plugins/opensearch_security/log4j-slf4j-impl-2.11.1.jar + - /usr/share/elasticsearch/plugins/elasticsearch_security/log4j-slf4j-impl-2.11.1.jar - name: Log4j patch | Delete temporary dir file: @@ -61,8 +61,8 @@ _archive_root_dir: >- {{ unarchive_list_files.files | first | dirname }} -- name: Restart opensearch-performance-analyzer service +- name: Restart elasticsearch-performance-analyzer service systemd: - name: opensearch-performance-analyzer + name: elasticsearch-performance-analyzer state: restarted when: log4j_patch.changed diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 index f4f8ae2fc2..5d4dba7016 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 @@ -1,5 +1,5 @@ [Unit] -Description=opensearch +Description=OpenSearch Wants=network-online.target After=network-online.target diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 9fa364121b..b250592a3a 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -15,7 +15,7 @@ - name: include dashboards installation include: dashboards.yml -- name: Make sure opensearch dashboards is started +- name: Make sure OpenSearch Ddashboards is started service: name: dashboards state: started diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service index f161a6576c..47ef120956 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service @@ -1,5 +1,5 @@ [Unit] -Description=opensearch-dashboards +Description=OpenSearch Dashboards Wants=network-online.target After=network-online.target diff --git a/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml b/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml index 3b120cca51..017318905e 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml @@ -1,3 +1,3 @@ --- -# vars file for opensearch +# vars file for openseOpenSearcharch java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index ce091650b9..84a87a9ed5 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -10,7 +10,7 @@ fail_msg: elasticsearch-oss package not found, nothing to upgrade quiet: true -- name: OPS | Include defaults from opensearch role +- name: OPS | Include defaults from OpenSearch role include_vars: file: roles/opensearch/defaults/main.yml name: ops_defaults @@ -38,7 +38,7 @@ - "{{ ops_vars.specification.paths.ops_conf_dir }}" - "{{ ops_defaults.certificates.dirs.certs }}" -- name: OPS | Print elasticsearch ond opensearch versions +- name: OPS | Print ElasticSearch ond OpenSearch versions debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index 19f812dff1..62005acf03 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -83,7 +83,7 @@ - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } - { 1: 'opendistro_security.', 2: 'plugins.security.' } -- name: ODFE migr | Start opensearch service +- name: ODFE migr | Start OpenSearch service systemd: name: opensearch state: started diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 33322e724c..937cb7c5a6 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -4,7 +4,7 @@ manager: auto when: ansible_facts.packages is undefined -- name: OPS | Print elasticsearch ond opensearch versions +- name: OPS | Print ElasticSearch ond OpenSearch versions debug: msg: - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" @@ -185,11 +185,11 @@ run_once: true when: logstash_check_response.status == 200 -- name: ODFE migr | Check the opensearch status +- name: ODFE migr | Check the OpenSearch status command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k register: ops_status -- name: ODFE migr | Show the opensearch status +- name: ODFE migr | Show the OpenSearch status debug: msg: "{{ ops_status.stdout }}" failed_when: "'number_of_nodes' not in ops_status.stdout" diff --git a/docs/changelogs/CHANGELOG-0.5.md b/docs/changelogs/CHANGELOG-0.5.md index 82308f6dca..fbb2bb0c7a 100644 --- a/docs/changelogs/CHANGELOG-0.5.md +++ b/docs/changelogs/CHANGELOG-0.5.md @@ -50,7 +50,7 @@ - [#854](https://github.com/epiphany-platform/epiphany/issues/854) - PostgreSQL: PGBouncer implementation - [#905](https://github.com/epiphany-platform/epiphany/pull/905) - PostgreSQL: pgAudit extension for audit logging -- PostgreSQL: Send logs to opensearch +- PostgreSQL: Send logs to elasticsearch - [#915](https://github.com/epiphany-platform/epiphany/pull/915) - PostgreSQL: Add logrotate configuration ### Fixed diff --git a/docs/changelogs/CHANGELOG-0.8.md b/docs/changelogs/CHANGELOG-0.8.md index 5e169f719b..5cc2b64178 100644 --- a/docs/changelogs/CHANGELOG-0.8.md +++ b/docs/changelogs/CHANGELOG-0.8.md @@ -66,4 +66,4 @@ ### Known issues -- [1647](https://github.com/epiphany-platform/epiphany/issues/1647) - `epicli upgrade` fails on `[opensearch : Provide jvm configuration file]` task +- [1647](https://github.com/epiphany-platform/epiphany/issues/1647) - `epicli upgrade` fails on `[elasticsearch : Provide jvm configuration file]` task diff --git a/docs/changelogs/CHANGELOG-0.9.md b/docs/changelogs/CHANGELOG-0.9.md index d0d420d3c2..d2395ecc3a 100644 --- a/docs/changelogs/CHANGELOG-0.9.md +++ b/docs/changelogs/CHANGELOG-0.9.md @@ -23,7 +23,7 @@ - [#1888](https://github.com/epiphany-platform/epiphany/issues/1888) - epicli upgrade of cluster created by Epiphany v0.5 may fail - [#1884](https://github.com/epiphany-platform/epiphany/issues/1884) - Prometheus is not able to scrape metrics from AKS/EKS nodes - [#1887](https://github.com/epiphany-platform/epiphany/issues/1887) - epicli upgrade of cluster created by Epiphany v0.6 fails on "Store preflight facts" task -- [#1866](https://github.com/epiphany-platform/epiphany/issues/1866) - No logs from K8s apps in opensearch +- [#1866](https://github.com/epiphany-platform/epiphany/issues/1866) - No logs from K8s apps in elasticsearch ### Updated diff --git a/docs/design-docs/arm/centos-arm-analysis.md b/docs/design-docs/arm/centos-arm-analysis.md index 8d37ff8df7..10bc8019e2 100644 --- a/docs/design-docs/arm/centos-arm-analysis.md +++ b/docs/design-docs/arm/centos-arm-analysis.md @@ -61,12 +61,12 @@ | net-tools | + | | + | | nfs-utils | + | | + | | nmap-ncat | + | | ? | -| opensearch-alerting-1.10.1* | + | | + | -| opensearch-index-management-1.10.1* | + | | + | -| opensearch-job-scheduler-1.10.1* | + | | + | -| opensearch-performance-analyzer-1.10.1* | + | | + | -| opensearch-security-1.10.1* | + | | + | -| opensearch-sql-1.10.1* | + | | + | +| elasticsearch-alerting-1.10.1* | + | | + | +| elasticsearch-index-management-1.10.1* | + | | + | +| elasticsearch-job-scheduler-1.10.1* | + | | + | +| elasticsearch-performance-analyzer-1.10.1* | + | | + | +| elasticsearch-security-1.10.1* | + | | + | +| elasticsearch-sql-1.10.1* | + | | + | | opendistroforelasticsearch-kibana-1.10.1* | --- | opendistroforelasticsearch-kibana-1.13.0 | + | | openssl | + | | + | | perl | + | | + | diff --git a/docs/design-docs/arm/redhat-arm-analysis.md b/docs/design-docs/arm/redhat-arm-analysis.md index 32b37f5e5c..3c339a065c 100644 --- a/docs/design-docs/arm/redhat-arm-analysis.md +++ b/docs/design-docs/arm/redhat-arm-analysis.md @@ -60,12 +60,12 @@ | net-tools | + | | + | | nfs-utils | + | | + | | nmap-ncat | + | | ? | -| opensearch-alerting-1.13.1* | + | | + | -| opensearch-index-management-1.13.1* | + | | + | -| opensearch-job-scheduler-1.13.1* | + | | + | -| opensearch-performance-analyzer-1.13.1* | + | | + | -| opensearch-security-1.13.1* | + | | + | -| opensearch-sql-1.13.1* | + | | + | +| elasticsearch-alerting-1.13.1* | + | | + | +| elasticsearch-index-management-1.13.1* | + | | + | +| elasticsearch-job-scheduler-1.13.1* | + | | + | +| elasticsearch-performance-analyzer-1.13.1* | + | | + | +| elasticsearch-security-1.13.1* | + | | + | +| elasticsearch-sql-1.13.1* | + | | + | | opendistroforelasticsearch-kibana-1.13.1* | + | | + | | unixODBC | + | | + | | openssl | + | | + | diff --git a/docs/design-docs/arm/ubuntu-arm-analysis.md b/docs/design-docs/arm/ubuntu-arm-analysis.md index 8db31ea512..0c0d24e2b3 100644 --- a/docs/design-docs/arm/ubuntu-arm-analysis.md +++ b/docs/design-docs/arm/ubuntu-arm-analysis.md @@ -52,12 +52,12 @@ | netcat | + | | + | | net-tools | + | | + | | nfs-common | + | | + | -| opensearch-alerting | + | | + | -| opensearch-index-management | + | | + | -| opensearch-job-scheduler | + | | + | -| opensearch-performance-analyzer | + | | + | -| opensearch-security | + | | + | -| opensearch-sql | + | | + | +| elasticsearch-alerting | + | | + | +| elasticsearch-index-management | + | | + | +| elasticsearch-job-scheduler | + | | + | +| elasticsearch-performance-analyzer | + | | + | +| elasticsearch-security | + | | + | +| elasticsearch-sql | + | | + | | opendistroforelasticsearch-kibana | + | | + | | openjdk-8-jre-headless | + | | + | | openssl | + | | + | diff --git a/docs/design-docs/health-monitor/health-monitor.md b/docs/design-docs/health-monitor/health-monitor.md index bdb0ec8c70..1c51f91072 100644 --- a/docs/design-docs/health-monitor/health-monitor.md +++ b/docs/design-docs/health-monitor/health-monitor.md @@ -21,7 +21,7 @@ Components that Health Monitor should check: - Prometheus - Kafka - ZooKeeper -- opensearch +- OpenSearch - RabbitMQ `*` means MVP version. diff --git a/docs/home/howto/MAINTENANCE.md b/docs/home/howto/MAINTENANCE.md index c80c20f150..805a9486db 100644 --- a/docs/home/howto/MAINTENANCE.md +++ b/docs/home/howto/MAINTENANCE.md @@ -112,7 +112,7 @@ To check status of Node Exporter we can use the command: status prometheus-node-exporter ``` -#### - opensearch +#### - OpenSearch To check status of OpenSearch we can use the command: diff --git a/docs/home/howto/RETENTION.md b/docs/home/howto/RETENTION.md index 3ba3a3dc5e..841681c000 100644 --- a/docs/home/howto/RETENTION.md +++ b/docs/home/howto/RETENTION.md @@ -1,7 +1,7 @@ An Epiphany cluster has a number of components which log, collect and retain data. To make sure that these do not exceed the usable storage of the machines they running on, the following configurations are available. -## opensearch +## OpenSearch TODO diff --git a/schema/common/validation/configuration/opensearch-dashboards.yml b/schema/common/validation/configuration/opensearch-dashboards.yml index 23cfbfa7b5..ddba5a2283 100644 --- a/schema/common/validation/configuration/opensearch-dashboards.yml +++ b/schema/common/validation/configuration/opensearch-dashboards.yml @@ -1,5 +1,5 @@ kind: configuration/opensearch-dashboards -title: "OpenSearch-dashboards" +title: "OpenSearch Dashboards specification schema" name: default specification: ops_dashboards_log_dir: /var/log/opensearchdashboards diff --git a/schema/common/validation/configuration/opensearch.yml b/schema/common/validation/configuration/opensearch.yml index e31f942979..a33004c8de 100644 --- a/schema/common/validation/configuration/opensearch.yml +++ b/schema/common/validation/configuration/opensearch.yml @@ -1,6 +1,6 @@ "$id": "#/specification" title: "opensearch schema" -description: "opensearch specification schema" +description: "OpenSearch specification schema" type: object properties: cluster_name: diff --git a/tools/ip_change/elasticsearch/2_config_files.sh b/tools/ip_change/elasticsearch/2_config_files.sh index 30b6d229ef..6f09052de5 100644 --- a/tools/ip_change/elasticsearch/2_config_files.sh +++ b/tools/ip_change/elasticsearch/2_config_files.sh @@ -42,5 +42,5 @@ systemctl restart kibana echo "==== Kibana restarted ====" echo "==== Restarting Elasticsearch ====" -systemctl restart opensearch +systemctl restart elasticsearch echo "==== Elasticsearch restarted ====" From 67c6d49fcd6922945f2c8eddd333b80eb71a28a5 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 11:43:11 +0100 Subject: [PATCH 071/157] Unify odfe_migration parameter location and schema validation added --- schema/common/defaults/configuration/logging.yml | 1 - schema/common/validation/configuration/opensearch.yml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 7d0e1f06ad..fd217b7d3e 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -3,7 +3,6 @@ title: Logging Config name: default specification: cluster_name: EpiphanyOpensearch - odfe_migration: false ops_user: opensearch opsd_user: opensearchboard opsd_password: PASSWORD_TO_CHANGE diff --git a/schema/common/validation/configuration/opensearch.yml b/schema/common/validation/configuration/opensearch.yml index a33004c8de..4ccecb9bcf 100644 --- a/schema/common/validation/configuration/opensearch.yml +++ b/schema/common/validation/configuration/opensearch.yml @@ -5,6 +5,8 @@ type: object properties: cluster_name: type: string + odfe_migration: + type: boolean clustered: type: boolean admin_password: From e32dfe9e864d4f0af65513064abae1cbde037a86 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 14:57:57 +0100 Subject: [PATCH 072/157] Not standard versions of ODFE/OPS are not supported --- docs/home/howto/UPGRADE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 35e9687112..875bf2af86 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -325,6 +325,8 @@ The default value of the `odfe_migration` parameter is set to _false_. All described below remarks related to TLS certificates of the Open Distro upgrade stay valid. You should plan and test all your upgrade activities before proceeding on the production. +Upgrade of the ESS/ODFE versions not shipped with the previous Epiphany releases is not supported. If your environment is customized it needs to be standardized ( as described in [this](https://opensearch.org/docs/latest/upgrade-to/upgrade-to/#upgrade-paths) table ) prior to running the subject migration. + Migration of Elasticsearch Curator is not supported. More info on use of Curator in OpenSearch environment can be found [here](https://github.com/opensearch-project/OpenSearch/issues/1352). [1] https://www.elastic.co/pricing/faq/licensing#what-are-the-key-changes-being-made-to-the-elastic-license From 24c99c50685565436061d1fe512464d443c45bf7 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 15:05:32 +0100 Subject: [PATCH 073/157] OPS Perf. analyzer download path added --- .../download-requirements/centos-7/requirements.x86_64.txt | 2 ++ .../download-requirements/redhat-7/requirements.x86_64.txt | 2 ++ .../download-requirements/ubuntu-20.04/requirements.x86_64.txt | 2 ++ 3 files changed, 6 insertions(+) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 5a8d3db0fa..d1022a181e 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -190,6 +190,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz # OpenSearch Dashboards https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz +# OpenSearch PerfTop +https://github.com/opensearch-project/perftop/releases/download/1.1.0.0/opensearch-perf-top-1.1.0.0-linux-x64.zip [images] diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index 6f050d61df..245242764a 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -188,6 +188,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz # OpenSearch Dashboards https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz +# OpenSearch PerfTop +https://github.com/opensearch-project/perftop/releases/download/1.1.0.0/opensearch-perf-top-1.1.0.0-linux-x64.zip [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index 64b5c89646..39633ad5f4 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -225,6 +225,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz # OpenSearch Dashboards https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz +# OpenSearch PerfTop +https://github.com/opensearch-project/perftop/releases/download/1.1.0.0/opensearch-perf-top-1.1.0.0-linux-x64.zip [images] haproxy:2.2.2-alpine From 463ee90b3f219551291f5495e9caa5a1fdd95bec Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 15:24:54 +0100 Subject: [PATCH 074/157] Make it more precize which dashoboards we talk about --- docs/home/HOWTO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md index f3767ce8e5..30fd1d73c4 100644 --- a/docs/home/HOWTO.md +++ b/docs/home/HOWTO.md @@ -34,7 +34,7 @@ - [How to configure scalable Prometheus setup](./howto/MONITORING.md#how-to-configure-scalable-prometheus-setup) - [Import and create Grafana dashboards](./howto/MONITORING.md#import-and-create-grafana-dashboards) - [How to setup default admin password and user in Grafana](./howto/MONITORING.md#how-to-setup-default-admin-password-and-user-in-grafana) - - [How to configure Dashboards](./howto/MONITORING.md#how-to-configure-opensearch-dashboards) + - [How to configure OpenSearch Dashboards](./howto/MONITORING.md#how-to-configure-opensearch-dashboards) - [How to configure default passwords for service users in OpenSearch Dashboards, OpenSearch and Filebeat](./howto/MONITORING.md#how-to-configure-default-passwords-for-service-users-in-opensearch-dashboards-opensearch-and-filebeat) - [How to configure scalable Prometheus setup](./howto/MONITORING.md#how-to-configure-scalable-prometheus-setup) - [How to configure Azure additional monitoring and alerting](./howto/MONITORING.md#how-to-configure-azure-additional-monitoring-and-alerting) From bf3be0d1ac4512c8a2238ef80ce2fa8feff6f322 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 15:27:08 +0100 Subject: [PATCH 075/157] Correcting the link --- docs/home/howto/DATABASES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 5ea1228b63..50ff5345fe 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -457,7 +457,7 @@ Use [PostgreSQL HA replication with repmgr](#how-to-set-up-postgresql-ha-replica ## How to start working with Opensearch -OpenSearch is the [successor](https://opendistro.github.io/for-elasticsearch-docs/) of OpenDistro for ElasticSearch project. Epipahny is providing an [automated solution](https://github.com/romsok24/epiphany/blob/feature/migr-ODFE-OpenSearch-2870/docs/home/howto/UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) for migrating your existing ODFE installation to OpenSearch. +OpenSearch is the [successor](https://opendistro.github.io/for-elasticsearch-docs/) of OpenDistro for ElasticSearch project. Epipahny is providing an [automated solution](./UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) for migrating your existing ODFE installation to OpenSearch. On the other hand, if you plan to just start working with OpenSearch - change machines count to value greater than 0 in your cluster configuration: ```yaml From 1d47066116010d6fd2485e2de3a34cfbbc59f722 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 15:28:03 +0100 Subject: [PATCH 076/157] Withdrawing incorect chanes --- docs/changelogs/CHANGELOG-0.5.md | 2 +- docs/changelogs/CHANGELOG-0.8.md | 2 +- docs/changelogs/CHANGELOG-0.9.md | 2 +- docs/changelogs/CHANGELOG-1.3.md | 2 +- docs/design-docs/arm/centos-arm-analysis.md | 12 ++++++------ docs/design-docs/arm/redhat-arm-analysis.md | 12 ++++++------ docs/design-docs/arm/ubuntu-arm-analysis.md | 12 ++++++------ 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/changelogs/CHANGELOG-0.5.md b/docs/changelogs/CHANGELOG-0.5.md index fbb2bb0c7a..9acb3929a2 100644 --- a/docs/changelogs/CHANGELOG-0.5.md +++ b/docs/changelogs/CHANGELOG-0.5.md @@ -50,7 +50,7 @@ - [#854](https://github.com/epiphany-platform/epiphany/issues/854) - PostgreSQL: PGBouncer implementation - [#905](https://github.com/epiphany-platform/epiphany/pull/905) - PostgreSQL: pgAudit extension for audit logging -- PostgreSQL: Send logs to elasticsearch +- PostgreSQL: Send logs to Elasticsearch - [#915](https://github.com/epiphany-platform/epiphany/pull/915) - PostgreSQL: Add logrotate configuration ### Fixed diff --git a/docs/changelogs/CHANGELOG-0.8.md b/docs/changelogs/CHANGELOG-0.8.md index 5cc2b64178..2b77d60cda 100644 --- a/docs/changelogs/CHANGELOG-0.8.md +++ b/docs/changelogs/CHANGELOG-0.8.md @@ -66,4 +66,4 @@ ### Known issues -- [1647](https://github.com/epiphany-platform/epiphany/issues/1647) - `epicli upgrade` fails on `[elasticsearch : Provide jvm configuration file]` task +- [1647](https://github.com/epiphany-platform/epiphany/issues/1647) - `epicli upgrade` fails on `[opendistro_for_elasticsearch : Provide jvm configuration file]` task diff --git a/docs/changelogs/CHANGELOG-0.9.md b/docs/changelogs/CHANGELOG-0.9.md index d2395ecc3a..eda84f658b 100644 --- a/docs/changelogs/CHANGELOG-0.9.md +++ b/docs/changelogs/CHANGELOG-0.9.md @@ -23,7 +23,7 @@ - [#1888](https://github.com/epiphany-platform/epiphany/issues/1888) - epicli upgrade of cluster created by Epiphany v0.5 may fail - [#1884](https://github.com/epiphany-platform/epiphany/issues/1884) - Prometheus is not able to scrape metrics from AKS/EKS nodes - [#1887](https://github.com/epiphany-platform/epiphany/issues/1887) - epicli upgrade of cluster created by Epiphany v0.6 fails on "Store preflight facts" task -- [#1866](https://github.com/epiphany-platform/epiphany/issues/1866) - No logs from K8s apps in elasticsearch +- [#1866](https://github.com/epiphany-platform/epiphany/issues/1866) - No logs from K8s apps in Elasticsearch ### Updated diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index 70490de0b2..f083683e6e 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -68,7 +68,7 @@ - [#2748](https://github.com/epiphany-platform/epiphany/issues/2748) - Upgrade Kafka exporter to the version 1.4.0 - [#2750](https://github.com/epiphany-platform/epiphany/issues/2750) - Upgrade JMX exporter to the newest version - [#2699](https://github.com/epiphany-platform/epiphany/issues/2699) - Upgrade Grafana to 8.3.2 -- [#2788](https://github.com/epiphany-platform/epiphany/issues/2788) - Upgrade Log4j in Open Distro for ElasticSearch +- [#2788](https://github.com/epiphany-platform/epiphany/issues/2788) - Upgrade Log4j in Open Distro for Elasticsearch - [#2661](https://github.com/epiphany-platform/epiphany/issues/2661) - Update K8s documentation according to the latest version Epiphany supports - [#2752](https://github.com/epiphany-platform/epiphany/issues/2752) - Upgrade postgresql exporter to the version 0.10.0 - [#2856](https://github.com/epiphany-platform/epiphany/issues/2856) - Update cloud OS images to the latest diff --git a/docs/design-docs/arm/centos-arm-analysis.md b/docs/design-docs/arm/centos-arm-analysis.md index 10bc8019e2..b7211606dc 100644 --- a/docs/design-docs/arm/centos-arm-analysis.md +++ b/docs/design-docs/arm/centos-arm-analysis.md @@ -61,12 +61,12 @@ | net-tools | + | | + | | nfs-utils | + | | + | | nmap-ncat | + | | ? | -| elasticsearch-alerting-1.10.1* | + | | + | -| elasticsearch-index-management-1.10.1* | + | | + | -| elasticsearch-job-scheduler-1.10.1* | + | | + | -| elasticsearch-performance-analyzer-1.10.1* | + | | + | -| elasticsearch-security-1.10.1* | + | | + | -| elasticsearch-sql-1.10.1* | + | | + | +| opendistro-alerting-1.10.1* | + | | + | +| opendistro-index-management-1.10.1* | + | | + | +| opendistro-job-scheduler-1.10.1* | + | | + | +| opendistro-performance-analyzer-1.10.1* | + | | + | +| opendistro-security-1.10.1* | + | | + | +| opendistro-sql-1.10.1* | + | | + | | opendistroforelasticsearch-kibana-1.10.1* | --- | opendistroforelasticsearch-kibana-1.13.0 | + | | openssl | + | | + | | perl | + | | + | diff --git a/docs/design-docs/arm/redhat-arm-analysis.md b/docs/design-docs/arm/redhat-arm-analysis.md index 3c339a065c..17c13a5f1f 100644 --- a/docs/design-docs/arm/redhat-arm-analysis.md +++ b/docs/design-docs/arm/redhat-arm-analysis.md @@ -60,12 +60,12 @@ | net-tools | + | | + | | nfs-utils | + | | + | | nmap-ncat | + | | ? | -| elasticsearch-alerting-1.13.1* | + | | + | -| elasticsearch-index-management-1.13.1* | + | | + | -| elasticsearch-job-scheduler-1.13.1* | + | | + | -| elasticsearch-performance-analyzer-1.13.1* | + | | + | -| elasticsearch-security-1.13.1* | + | | + | -| elasticsearch-sql-1.13.1* | + | | + | +| opendistro-alerting-1.13.1* | + | | + | +| opendistro-index-management-1.13.1* | + | | + | +| opendistro-job-scheduler-1.13.1* | + | | + | +| opendistro-performance-analyzer-1.13.1* | + | | + | +| opendistro-security-1.13.1* | + | | + | +| opendistro-sql-1.13.1* | + | | + | | opendistroforelasticsearch-kibana-1.13.1* | + | | + | | unixODBC | + | | + | | openssl | + | | + | diff --git a/docs/design-docs/arm/ubuntu-arm-analysis.md b/docs/design-docs/arm/ubuntu-arm-analysis.md index 0c0d24e2b3..0a5c635ead 100644 --- a/docs/design-docs/arm/ubuntu-arm-analysis.md +++ b/docs/design-docs/arm/ubuntu-arm-analysis.md @@ -52,12 +52,12 @@ | netcat | + | | + | | net-tools | + | | + | | nfs-common | + | | + | -| elasticsearch-alerting | + | | + | -| elasticsearch-index-management | + | | + | -| elasticsearch-job-scheduler | + | | + | -| elasticsearch-performance-analyzer | + | | + | -| elasticsearch-security | + | | + | -| elasticsearch-sql | + | | + | +| opendistro-alerting | + | | + | +| opendistro-index-management | + | | + | +| opendistro-job-scheduler | + | | + | +| opendistro-performance-analyzer | + | | + | +| opendistro-security | + | | + | +| opendistro-sql | + | | + | | opendistroforelasticsearch-kibana | + | | + | | openjdk-8-jre-headless | + | | + | | openssl | + | | + | From b15b7b5e39a1bdbb48de4cbda5703f8659b8ca33 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 21 Feb 2022 15:36:33 +0100 Subject: [PATCH 077/157] Old to new name migration --- docs/home/howto/DATABASES.md | 2 +- docs/home/howto/LOGGING.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 50ff5345fe..09c864f957 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -501,7 +501,7 @@ specification: - node-exporter - filebeat - firewall - - kibana + - opensearch-dashboards ``` Filebeat running on `opensearch` hosts will always point to centralized logging hosts ( [more info](./LOGGING.md) ). diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 0749920d5c..73e6e432d8 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -30,7 +30,7 @@ roles_mapping: [...] logging: - logging - - kibana + - opensearch-dashboards - node-exporter - filebeat - firewall From af359c0b1f3a3ab11ef662f45e67c724cf8e7c27 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 22 Feb 2022 14:49:47 +0100 Subject: [PATCH 078/157] Removal of not used component --- ansible/playbooks/kibana.yml | 12 ----- .../playbooks/roles/upgrade/tasks/kibana.yml | 47 ------------------- .../common/defaults/configuration/kibana.yml | 5 -- .../validation/configuration/kibana.yml | 7 --- 4 files changed, 71 deletions(-) delete mode 100644 ansible/playbooks/kibana.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/kibana.yml delete mode 100644 schema/common/defaults/configuration/kibana.yml delete mode 100644 schema/common/validation/configuration/kibana.yml diff --git a/ansible/playbooks/kibana.yml b/ansible/playbooks/kibana.yml deleted file mode 100644 index 882d4c66ff..0000000000 --- a/ansible/playbooks/kibana.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Ansible playbook that makes sure the base items for all nodes are installed - -- hosts: all - gather_facts: yes - tasks: [ ] - -- hosts: kibana - become: true - become_method: sudo - roles: - - kibana diff --git a/ansible/playbooks/roles/upgrade/tasks/kibana.yml b/ansible/playbooks/roles/upgrade/tasks/kibana.yml deleted file mode 100644 index c8e3baab72..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/kibana.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -- name: Kibana | Get information about installed packages as facts - package_facts: - manager: auto - when: ansible_facts.packages is undefined - -# Kibana is upgraded only when there is no 'kibana-oss' package (replaced by 'opendistroforelasticsearch-kibana' since v0.5). -# This condition has been added to not fail when 'epicli upgrade' is run for Epiphany v0.4 cluster. -# We cannot upgrade Kibana to v7 having Elasticsearch v6. -- name: Upgrade Kibana - when: ansible_facts.packages['kibana-oss'] is undefined - block: - - name: Kibana | Assert that opendistroforelasticsearch-kibana package is installed - assert: - that: ansible_facts.packages['opendistroforelasticsearch-kibana'] is defined - fail_msg: opendistroforelasticsearch-kibana package not found, nothing to upgrade - quiet: true - - - name: Kibana | Load defaults from kibana role - include_vars: - file: roles/kibana/defaults/main.yml - name: kibana_defaults - - - name: Kibana | Print versions - debug: - msg: - - "Installed version: {{ ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version }}" - - "Target version: {{ kibana_defaults.kibana_version[ansible_os_family] }}" - - - name: Upgrade Kibana - when: - - kibana_defaults.kibana_version[ansible_os_family] - is version(ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version, '>=') - block: - - name: Kibana | Slurp /etc/kibana/kibana.yml - slurp: - src: /etc/kibana/kibana.yml - register: _kibana_config_yml - no_log: true - - - name: Kibana | Upgrade - import_role: - name: kibana - vars: - context: upgrade - existing_es_password: >- - {{ (_kibana_config_yml.content | b64decode | from_yaml)['elasticsearch.password'] }} diff --git a/schema/common/defaults/configuration/kibana.yml b/schema/common/defaults/configuration/kibana.yml deleted file mode 100644 index bea9fbb13b..0000000000 --- a/schema/common/defaults/configuration/kibana.yml +++ /dev/null @@ -1,5 +0,0 @@ -kind: configuration/kibana -title: "Kibana" -name: default -specification: - kibana_log_dir: /var/log/kibana diff --git a/schema/common/validation/configuration/kibana.yml b/schema/common/validation/configuration/kibana.yml deleted file mode 100644 index 17b77c2e15..0000000000 --- a/schema/common/validation/configuration/kibana.yml +++ /dev/null @@ -1,7 +0,0 @@ -"$id": "#/specification" -title: "Kibana specification schema" -description: "Kibana specification schema" -type: object -properties: - kibana_log_dir: - type: string From 1afb6420f4adc60a11b463b2d6701a4ed9b2e6d1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 07:56:46 +0100 Subject: [PATCH 079/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../playbooks/roles/backup/tasks/logging_opensearch_conf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml index 57957c3787..db9f0655d5 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml @@ -1,5 +1,5 @@ --- -- name: Include default vars from OpenSearch role +- name: Include default vars from opensearch role include_vars: file: roles/opensearch/defaults/main.yml name: ops_def From b44f43864a768ee038b2b0402b206607f5d5ef3e Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 08:02:03 +0100 Subject: [PATCH 080/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../playbooks/roles/backup/tasks/logging_opensearch_conf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml index db9f0655d5..ceca36f5e0 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml @@ -2,7 +2,7 @@ - name: Include default vars from opensearch role include_vars: file: roles/opensearch/defaults/main.yml - name: ops_def + name: opensearch_defaults - name: Include vars from opensearch role include_vars: From ee9caf5e651a5e3a0842c0714497415184139454 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 08:04:20 +0100 Subject: [PATCH 081/157] Vars naming change + correcting var meaning as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../roles/backup/tasks/logging_opensearch_dashboards_conf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml index 4017a61d68..61bc5d05e2 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml @@ -12,7 +12,7 @@ - name: Include vars from opensearch_dashboards role include_vars: file: roles/opensearch_dashboards/vars/main.yml - name: opsd_def + name: opensearch_dashboards_vars - name: Create snapshot archive import_tasks: common/create_snapshot_archive.yml From c9b98d3766ddbed2c7bd6cadb77017408ba92eb1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 08:05:35 +0100 Subject: [PATCH 082/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../roles/backup/tasks/logging_opensearch_dashboards_conf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml index 61bc5d05e2..ca41debf4c 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml @@ -17,7 +17,7 @@ - name: Create snapshot archive import_tasks: common/create_snapshot_archive.yml vars: - snapshot_prefix: "opsd_conf_dir" + snapshot_prefix: "opensearch_dashboards_conf_dir" dirs_to_archive: - "{{ opsd_def.specification.paths.opsd_conf_dir }}" From d428903718024a0c996c1115d87fe3d69bee150c Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 08:05:56 +0100 Subject: [PATCH 083/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../roles/backup/tasks/logging_opensearch_dashboards_conf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml index ca41debf4c..c0a045bf4d 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml @@ -19,7 +19,7 @@ vars: snapshot_prefix: "opensearch_dashboards_conf_dir" dirs_to_archive: - - "{{ opsd_def.specification.paths.opsd_conf_dir }}" + - "{{ opensearch_dashboards_vars.specification.paths.opensearch_dashboards_conf_dir }}" - name: Create snapshot checksum import_tasks: common/create_snapshot_checksum.yml From e6d95d897ce1d505862aceabd3ad006dbccb462d Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 08:07:05 +0100 Subject: [PATCH 084/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../roles/backup/tasks/logging_opensearch_snapshot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml index a578ef764e..1156cfd345 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -2,7 +2,7 @@ - name: Include default vars from opensearch role include_vars: file: roles/opensearch/defaults/main.yml - name: ops_def + name: opensearch_defaults - name: Set helper facts set_fact: From 42e3e43c83b38fd4aa354015e2ac5a7ad4d2eff5 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 08:07:41 +0100 Subject: [PATCH 085/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../roles/backup/tasks/logging_opensearch_snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml index 1156cfd345..44d74c972a 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -12,8 +12,8 @@ {{ ansible_date_time.iso8601_basic_short | replace('T','-') }} vars: uri_template: &uri - client_cert: "{{ ops_def.certificates.dirs.certs }}/{{ ops_def.certificates.files.admin.cert.filename }}" - client_key: "{{ ops_def.certificates.dirs.certs }}/{{ ops_def.certificates.files.admin.key.filename }}" + client_cert: "{{ opensearch_defaults.certificates.dirs.certs }}/{{ opensearch_defaults.certificates.files.admin.cert.filename }}" + client_key: "{{ opensearch_defaults.certificates.dirs.certs }}/{{ opensearch_defaults.certificates.files.admin.key.filename }}" validate_certs: false body_format: json From 803efeb66b676d691373450a46eeb3b8e8099e99 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 10:17:08 +0100 Subject: [PATCH 086/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml b/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml index 09d27464db..06be00633b 100644 --- a/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml +++ b/ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml @@ -7,7 +7,7 @@ name: postgresql_defaults when: "'postgresql' in group_names" -# Do not select OPSD configured host to use OPS deployed by 'opensearch' role +# Do not select OpenSearch Dashboards configured host to use OpenSearch deployed by 'opensearch' role - name: Set value for setup.kibana.host set_fact: setup_kibana_host: >- From f60507158a4e15620e6d81860b07976cdaf52636 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 10:19:16 +0100 Subject: [PATCH 087/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/configure-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index 3414eec5cb..30d285e9e1 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -98,7 +98,7 @@ transport_port: "{{ is_upgrade_run | ternary(existing_es_config['transport.port'], ports.transport) }}" # When 'opensearch_security.allow_unsafe_democertificates' is set to 'false' all demo certificate files must be removed, -# otherwise OpenSearch service doesn't start. +# otherwise opensearch service doesn't start. # For apply mode, demo certificate files are removed based only on their names. For upgrade mode, # public key fingerprints are checked to protect against unintentional deletion (what takes additional time). From 184938e18dae732f7e0b991f3d9c62e6e1c7a5bc Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 10:20:29 +0100 Subject: [PATCH 088/157] Vars naming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/configure-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index 30d285e9e1..971e540f30 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -119,7 +119,7 @@ - name: Change default users when: not is_upgrade_run block: - - name: Wait for OpenSearch service to start up + - name: Wait for opensearch service to start up when: restart_opensearch.changed wait_for: port: 9200 From 762dbbd5627f6a185d12e6fb5f3fdd37d90ab51f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 10:44:15 +0100 Subject: [PATCH 089/157] Suggest potential problem solution to user --- .../upgrade/tasks/opensearch/migrate-odfe.yml | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 937cb7c5a6..691074923c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -11,11 +11,25 @@ - "Opensearch version to be installed: {{ ops_defaults.versions[ansible_os_family].ops_version }}" - name: ODFE migr | Ensure elasticsearch cluster is up and running - systemd: - name: elasticsearch - enabled: yes - state: restarted - register: elasticsearch_state + block: + - name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml + include_vars: + file: roles/opensearch/vars/main.yml + name: ops_vars + + - name: ODFE migr | Ensure elasticsearch cluster is up and running + systemd: + name: elasticsearch + enabled: yes + state: restarted + register: elasticsearch_state + rescue: + - name: ODFE migr | Suggest potential problem solution and fail + fail: + msg: "Are you trying to migrate from ODFE ( ops_vars.specification.odfe_migration: true ) on an already migrated server?" + when: + - ops_vars.specification.odfe_migration == true + - name: ODFE migr | Set existing_config facts include_tasks: opensearch/utils/get-config-from-files.yml From 8c6102ba12b084625fe34beeb96be914c4b72605 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 10:45:02 +0100 Subject: [PATCH 090/157] Vars naming change as requested during PR rev. --- ansible/playbooks/roles/opensearch/defaults/main.yml | 8 ++++---- ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 8 ++++---- ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml | 4 ++-- .../roles/upgrade/tasks/opensearch/migrate-odfe.yml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 39d7f4a658..e6f5d0060a 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -3,11 +3,11 @@ versions: RedHat: - ops_version: "1.2.4" - ops_perftop_version: "1.1.0.0" + opensearch: "1.2.4" + opensearch_perftop: "1.1.0.0" Debian: - ops_version: "1.2.4" - ops_perftop_version: "1.1.0.0" + opensearch: "1.2.4" + opensearch_perftop: "1.1.0.0" certificates: dirs: certs: /usr/share/opensearch/config diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 98c35c670f..0eb55cff5c 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -6,8 +6,8 @@ vars: file_name: "{{ item }}" with_items: - - "opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" - - "opensearch-perf-top-{{ versions[ansible_os_family].ops_perftop_version }}-linux-x64.zip" + - "opensearch-{{ versions[ansible_os_family].opensearch }}-linux-x64.tar.gz" + - "opensearch-perf-top-{{ versions[ansible_os_family].opensearch_perftop }}-linux-x64.zip" - name: Ensure Opensearch service user exists user: @@ -32,7 +32,7 @@ - name: Extract the OPS tar file unarchive: - src: "/tmp/opensearch-{{ versions[ansible_os_family].ops_version }}-linux-x64.tar.gz" + src: "/tmp/opensearch-{{ versions[ansible_os_family].opensearch }}-linux-x64.tar.gz" dest: "{{ specification.paths.ops_home }}" owner: "{{ specification.ops_user }}" remote_src: yes @@ -41,7 +41,7 @@ - name: Extract OPS PerfTop the tar file unarchive: - src: "/tmp/opensearch-perf-top-{{ versions[ansible_os_family].ops_perftop_version }}-linux-x64.zip" + src: "/tmp/opensearch-perf-top-{{ versions[ansible_os_family].opensearch_perftop }}-linux-x64.zip" dest: "{{ specification.paths.ops_perftop_home }}" owner: "{{ specification.ops_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index 84a87a9ed5..6e6ef2d7d9 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -42,7 +42,7 @@ debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Target version: {{ ops_defaults.versions[ansible_os_family].ops_version }}" + - "Target version: {{ ops_defaults.versions[ansible_os_family].opensearch }}" # If state file exists it means the previous run failed - name: OPS | Check if upgrade state file exists @@ -59,7 +59,7 @@ or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') and stat_upgrade_state_file.stat.exists) vars: - _target_version: "{{ ops_defaults.versions[ansible_os_family].ops_version }}" + _target_version: "{{ ops_defaults.versions[ansible_os_family].opensearch }}" - include_role: name: upgrade diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 691074923c..fc394b56ad 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -8,7 +8,7 @@ debug: msg: - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Opensearch version to be installed: {{ ops_defaults.versions[ansible_os_family].ops_version }}" + - "Opensearch version to be installed: {{ ops_defaults.versions[ansible_os_family].opensearch }}" - name: ODFE migr | Ensure elasticsearch cluster is up and running block: From f51a2e36d5f80b41998dd87f33dacf732a3ce6a4 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 10:57:01 +0100 Subject: [PATCH 091/157] Cleaning up the code --- .../roles/backup/tasks/logging_opensearch_snapshot.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml index 44d74c972a..9665caacf5 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -17,8 +17,6 @@ validate_certs: false body_format: json -- debug: var=snapshot_name - - name: Check cluster health uri: <<: *uri From 51b48bddbf4156349eacd1e23d106c961e6a904a Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 11:01:06 +0100 Subject: [PATCH 092/157] Removing incorect commenting --- ansible/playbooks/roles/opensearch/tasks/generate-certs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml index 6d6f9269ef..0c0dca8d55 100644 --- a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml +++ b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml @@ -170,7 +170,7 @@ type: RSA mode: u=rw,g=r,o= owner: opensearch - # group: opensearch + group: opensearch return_content: false register: node_key @@ -197,7 +197,7 @@ use_common_name_for_san: false mode: u=rw,g=r,o= owner: opensearch - # group: opensearch + group: opensearch register: node_csr vars: _unique_hostnames: "{{ [ansible_hostname, ansible_nodename, ansible_fqdn] | unique }}" @@ -215,4 +215,4 @@ ownca_digest: sha256 mode: u=rw,go=r owner: opensearch - # group: opensearch + group: opensearch From 2c908935d63ed035ae87f43a85819334cd1b0310 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 11:36:16 +0100 Subject: [PATCH 093/157] Vars naming change as requested during PR rev. --- .../backup/tasks/logging_opensearch_conf.yml | 4 +-- .../roles/opensearch/tasks/configure-ops.yml | 18 ++++++------- .../roles/opensearch/tasks/install-ops.yml | 26 +++++++++---------- .../templates/opensearch.service.j2 | 8 +++--- .../opensearch/templates/opensearch.yml.j2 | 6 ++--- .../opensearch_dashboards/defaults/main.yml | 8 +++--- .../templates/opensearch_dashboards.yml.j2 | 2 +- .../tasks/logging_opensearch_conf.yml | 6 ++--- .../roles/upgrade/tasks/opensearch-01.yml | 26 +++++++++---------- .../tasks/opensearch/migrate-odfe-serial.yml | 14 +++++----- .../upgrade/tasks/opensearch/migrate-odfe.yml | 22 ++++++++-------- .../opensearch/upgrade-opensearch-01.yml | 6 ++--- .../common/defaults/configuration/logging.yml | 18 ++++++------- .../defaults/configuration/opensearch.yml | 16 ++++++------ .../configuration/opensearch-dashboards.yml | 2 +- 15 files changed, 91 insertions(+), 91 deletions(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml index ceca36f5e0..65dd5b88ad 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml @@ -7,7 +7,7 @@ - name: Include vars from opensearch role include_vars: file: roles/opensearch/vars/main.yml - name: ops_vars + name: opensearch_vars - name: Assert that the snapshot_name fact is defined and valid assert: @@ -24,7 +24,7 @@ vars: snapshot_prefix: "opensearch_conf" dirs_to_archive: - - "{{ ops_vars.specification.paths.ops_conf_dir }}" + - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" - name: Create snapshot checksum import_tasks: common/create_snapshot_checksum.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml index 971e540f30..1737801204 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml @@ -3,19 +3,19 @@ - name: Ensure snapshot folder exists file: - path: "{{ specification.paths.ops_repo }}/" + path: "{{ specification.paths.opensearch_repo }}/" state: directory - owner: "{{ specification.ops_user }}" - group: "{{ specification.ops_user }}" + owner: "{{ specification.opensearch_user }}" + group: "{{ specification.opensearch_user }}" mode: u=rwx,go= - name: Provide JVM configuration file template: backup: yes src: jvm.options.j2 - dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" - owner: "{{ specification.ops_user }}" - group: "{{ specification.ops_user }}" + dest: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" + owner: "{{ specification.opensearch_user }}" + group: "{{ specification.opensearch_user }}" mode: ug=rw,o= register: change_jvm_config vars: @@ -35,9 +35,9 @@ template: backup: yes src: opensearch.yml.j2 - dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" - owner: "{{ specification.ops_user }}" - group: "{{ specification.ops_user }}" + dest: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" + owner: "{{ specification.opensearch_user }}" + group: "{{ specification.opensearch_user }}" mode: ug=rw,o= register: change_config vars: diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 0eb55cff5c..b19a038c5a 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -11,7 +11,7 @@ - name: Ensure Opensearch service user exists user: - name: "{{ specification.ops_user }}" + name: "{{ specification.opensearch_user }}" state: present shell: /bin/bash @@ -19,22 +19,22 @@ file: path: "{{ item }}" state: directory - owner: "{{ specification.ops_user }}" - group: "{{ specification.ops_user }}" + owner: "{{ specification.opensearch_user }}" + group: "{{ specification.opensearch_user }}" with_items: - - "{{ specification.paths.ops_home }}" - - "{{ specification.paths.ops_perftop_home }}" - - "{{ specification.paths.ops_log_dir }}" - - "{{ specification.paths.ops_conf_dir }}" - - "{{ specification.paths.ops_data }}" - - "{{ specification.paths.ops_logs }}" + - "{{ specification.paths.opensearch_home }}" + - "{{ specification.paths.opensearch_perftop_home }}" + - "{{ specification.paths.opensearch_log_dir }}" + - "{{ specification.paths.opensearch_conf_dir }}" + - "{{ specification.paths.opensearch_data }}" + - "{{ specification.paths.opensearch_logs }}" - "{{ certificates.dirs.certs }}" - name: Extract the OPS tar file unarchive: src: "/tmp/opensearch-{{ versions[ansible_os_family].opensearch }}-linux-x64.tar.gz" - dest: "{{ specification.paths.ops_home }}" - owner: "{{ specification.ops_user }}" + dest: "{{ specification.paths.opensearch_home }}" + owner: "{{ specification.opensearch_user }}" remote_src: yes extra_opts: - --strip-components=1 @@ -42,8 +42,8 @@ - name: Extract OPS PerfTop the tar file unarchive: src: "/tmp/opensearch-perf-top-{{ versions[ansible_os_family].opensearch_perftop }}-linux-x64.zip" - dest: "{{ specification.paths.ops_perftop_home }}" - owner: "{{ specification.ops_user }}" + dest: "{{ specification.paths.opensearch_perftop_home }}" + owner: "{{ specification.opensearch_user }}" remote_src: yes - name: Create systemd service diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 index 5d4dba7016..f3446c61c7 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 @@ -7,12 +7,12 @@ After=network-online.target RuntimeDirectory=opensearch PrivateTmp=true -WorkingDirectory={{ specification.paths.ops_home }} +WorkingDirectory={{ specification.paths.opensearch_home }} -User={{ specification.ops_user }} -Group={{ specification.ops_user }} +User={{ specification.opensearch_user }} +Group={{ specification.opensearch_user }} -ExecStart={{ specification.paths.ops_home }}/bin/opensearch -p {{ specification.paths.ops_home }}/opensearch.pid -q +ExecStart={{ specification.paths.opensearch_home }}/bin/opensearch -p {{ specification.paths.opensearch_home }}/opensearch.pid -q StandardOutput=journal StandardError=inherit diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 index 6e23774b88..18c4268499 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 @@ -26,15 +26,15 @@ node.name: {{ ansible_hostname }} # # Path to directory where to store the data (separate multiple locations by comma): # -path.data: {{ specification.paths.ops_data }} +path.data: {{ specification.paths.opensearch_data }} # # Path to directory where the shared storage should be mounted: # -path.repo: {{ specification.paths.ops_repo }} +path.repo: {{ specification.paths.opensearch_repo }} # # Path to log files: # -path.logs: {{ specification.paths.ops_logs }} +path.logs: {{ specification.paths.opensearch_logs }} # # ----------------------------------- Memory ----------------------------------- # diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index aa8a521a3d..bc918d826c 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -4,17 +4,17 @@ versions: opsd_version: "1.2.0" Debian: opsd_version: "1.2.0" -ops_nodes: |- +opensearch_nodes: |- {% for item in groups['ops-cluster'] -%} {{ hostvars[item]['ip'] }}{% if not loop.last %}","{% endif %} {%- endfor %} populate_inventory_to_hosts_file: true -ops_api_port: 9200 -ops_nodes_dashboards: |- +opensearch_api_port: 9200 +opensearch_nodes_dashboards: |- {% for item in groups['opensearch_dashboards'] -%} - https://{{ hostvars[item]['ansible_host'] }}:{{ ops_api_port }}{% if not loop.last %}","{% endif %} + https://{{ hostvars[item]['ansible_host'] }}:{{ opensearch_api_port }}{% if not loop.last %}","{% endif %} {%- endfor %} systemctl_path: /etc/systemd/system diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index cce08f11d1..71cf9ca2fd 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -1,6 +1,6 @@ server.port: 5601 server.host: "{{ hostvars[inventory_hostname]['ansible_host'] }}" -opensearch.hosts: ["{{ ops_nodes_dashboards }}"] +opensearch.hosts: ["{{ opensearch_nodes_dashboards }}"] opensearch.ssl.verificationMode: none opensearch.username: "{{ specification.opsd_user }}" opensearch.password: "{{ specification.opsd_password }}" diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml index 65b855cb61..3b50d75ca1 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_conf.yml @@ -2,7 +2,7 @@ - name: Include vars from opensearch role include_vars: file: roles/opensearch/vars/main.yml - name: ops_vars + name: opensearch_vars - name: Find snapshot archive import_tasks: common/find_snapshot_archive.yml @@ -29,11 +29,11 @@ import_tasks: common/clear_directories.yml vars: dirs_to_clear: - - "{{ ops_vars.specification.paths.ops_conf_dir }}" + - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" - name: Extract the archive unarchive: - dest: "{{ ops_vars.specification.paths.ops_conf_dir }}" + dest: "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" src: "{{ recovery_dir }}/{{ snapshot_path | basename }}" remote_src: true diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index 6e6ef2d7d9..5396bd5cbc 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -13,16 +13,16 @@ - name: OPS | Include defaults from OpenSearch role include_vars: file: roles/opensearch/defaults/main.yml - name: ops_defaults + name: opensearch_defaults - name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml include_vars: file: roles/opensearch/vars/main.yml - name: ops_vars + name: opensearch_vars - name: Ensure Opensearch service user exists user: - name: "{{ ops_vars.specification.ops_user }}" + name: "{{ opensearch_vars.specification.opensearch_user }}" state: present shell: /bin/bash @@ -30,19 +30,19 @@ file: path: "{{ item }}" state: directory - owner: "{{ ops_vars.specification.ops_user }}" - group: "{{ ops_vars.specification.ops_user }}" + owner: "{{ opensearch_vars.specification.opensearch_user }}" + group: "{{ opensearch_vars.specification.opensearch_user }}" with_items: - - "{{ ops_vars.specification.paths.ops_home }}" - - "{{ ops_vars.specification.paths.ops_log_dir }}" - - "{{ ops_vars.specification.paths.ops_conf_dir }}" - - "{{ ops_defaults.certificates.dirs.certs }}" + - "{{ opensearch_vars.specification.paths.opensearch_home }}" + - "{{ opensearch_vars.specification.paths.opensearch_log_dir }}" + - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" + - "{{ opensearch_defaults.certificates.dirs.certs }}" - name: OPS | Print ElasticSearch ond OpenSearch versions debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Target version: {{ ops_defaults.versions[ansible_os_family].opensearch }}" + - "Target version: {{ opensearch_defaults.versions[ansible_os_family].opensearch }}" # If state file exists it means the previous run failed - name: OPS | Check if upgrade state file exists @@ -59,18 +59,18 @@ or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') and stat_upgrade_state_file.stat.exists) vars: - _target_version: "{{ ops_defaults.versions[ansible_os_family].opensearch }}" + _target_version: "{{ opensearch_defaults.versions[ansible_os_family].opensearch }}" - include_role: name: upgrade tasks_from: opensearch/migrate-odfe - when: ops_vars.specification.odfe_migration + when: opensearch_vars.specification.odfe_migration vars: current_group_name: logging - include_role: name: upgrade tasks_from: opensearch/migrate-kibana - when: ops_vars.specification.odfe_migration + when: opensearch_vars.specification.odfe_migration vars: current_group_name: logging diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index 62005acf03..50d7acb39d 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -28,8 +28,8 @@ mode: ug=rwx,o= directory_mode: yes with_items: - - { 1: "/var/lib/elasticsearch-snapshots/", 2: "{{ specification.paths.ops_repo }}/" } - - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.ops_data }}" } + - { 1: "/var/lib/elasticsearch-snapshots/", 2: "{{ specification.paths.opensearch_repo }}/" } + - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.opensearch_data }}" } - name: ODFE migr | Prepare a list of ESS certs and keys find: @@ -40,14 +40,14 @@ - name: ODFE migr | Copy a list of certs and keys to OPS directories copy: src: "{{ item.path }}" - dest: "{{ specification.paths.ops_conf_dir }}/" + dest: "{{ specification.paths.opensearch_conf_dir }}/" remote_src: yes with_items: "{{ pem_files.files }}" - name: ODFE migr | Clone JVM configuration file copy: src: /etc/elasticsearch/jvm.options - dest: "{{ specification.paths.ops_conf_dir }}/jvm.options" + dest: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" remote_src: yes owner: root group: opensearch @@ -56,7 +56,7 @@ - name: ODFE migr | Update JVM configuration file replace: - path: "{{ specification.paths.ops_conf_dir }}/jvm.options" + path: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" regexp: "{{ item.1 }}" replace: "{{ item.2 }}" with_items: @@ -66,7 +66,7 @@ - name: ODFE migr | Clone main configuration file copy: src: /etc/elasticsearch/elasticsearch.yml - dest: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" + dest: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" remote_src: yes owner: root group: opensearch @@ -75,7 +75,7 @@ - name: ODFE migr | Update main configuration file replace: - path: "{{ specification.paths.ops_conf_dir }}/opensearch.yml" + path: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" regexp: "{{ item.1 }}" replace: "{{ item.2 }}" with_items: diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index fc394b56ad..1a1efee29f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -8,14 +8,14 @@ debug: msg: - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Opensearch version to be installed: {{ ops_defaults.versions[ansible_os_family].opensearch }}" + - "Opensearch version to be installed: {{ opensearch_defaults.versions[ansible_os_family].opensearch }}" - name: ODFE migr | Ensure elasticsearch cluster is up and running block: - name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml include_vars: file: roles/opensearch/vars/main.yml - name: ops_vars + name: opensearch_vars - name: ODFE migr | Ensure elasticsearch cluster is up and running systemd: @@ -26,9 +26,9 @@ rescue: - name: ODFE migr | Suggest potential problem solution and fail fail: - msg: "Are you trying to migrate from ODFE ( ops_vars.specification.odfe_migration: true ) on an already migrated server?" + msg: "Are you trying to migrate from ODFE ( opensearch_vars.specification.odfe_migration: true ) on an already migrated server?" when: - - ops_vars.specification.odfe_migration == true + - opensearch_vars.specification.odfe_migration == true - name: ODFE migr | Set existing_config facts @@ -36,10 +36,10 @@ - name: ODFE migr | Set common facts set_fact: - certificates: "{{ ops_defaults.certificates }}" + certificates: "{{ opensearch_defaults.certificates }}" es_host: "{{ existing_config.main['network.host'] | default('_local_') }}" - es_http_port: "{{ existing_config.main['http.port'] | default(ops_defaults.ports.http) }}" - es_transport_port: "{{ existing_config.main['transport.port'] | default(ops_defaults.ports.transport) }}" + es_http_port: "{{ existing_config.main['http.port'] | default(opensearch_defaults.ports.http) }}" + es_transport_port: "{{ existing_config.main['transport.port'] | default(opensearch_defaults.ports.transport) }}" es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" es_node_name: "{{ existing_config.main['node.name'] }}" @@ -107,7 +107,7 @@ - op: "replace" path: "/admin" value: - password: "{{ specification.ops_password }}" + password: "{{ specification.opensearch_password }}" reserved: "true" backend_roles: - "admin" @@ -201,12 +201,12 @@ - name: ODFE migr | Check the OpenSearch status command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k - register: ops_status + register: opensearch_status - name: ODFE migr | Show the OpenSearch status debug: - msg: "{{ ops_status.stdout }}" - failed_when: "'number_of_nodes' not in ops_status.stdout" + msg: "{{ opensearch_status.stdout }}" + failed_when: "'number_of_nodes' not in opensearch_status.stdout" - name: ODFE migr | Reenable shard allocation for the cluster include_tasks: diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml index c5b951c2e2..e2166f0b59 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml @@ -20,10 +20,10 @@ - name: OPS | Set common facts set_fact: - certificates: "{{ ops_defaults.certificates }}" + certificates: "{{ opensearch_defaults.certificates }}" es_host: "{{ existing_config.main['network.host'] | default('_local_') }}" - es_http_port: "{{ existing_config.main['http.port'] | default(ops_defaults.ports.http) }}" - es_transport_port: "{{ existing_config.main['transport.port'] | default(ops_defaults.ports.transport) }}" + es_http_port: "{{ existing_config.main['http.port'] | default(opensearch_defaults.ports.http) }}" + es_transport_port: "{{ existing_config.main['transport.port'] | default(opensearch_defaults.ports.transport) }}" es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" es_node_name: "{{ existing_config.main['node.name'] }}" diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index fd217b7d3e..330553e1bc 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -3,7 +3,7 @@ title: Logging Config name: default specification: cluster_name: EpiphanyOpensearch - ops_user: opensearch + opensearch_user: opensearch opsd_user: opensearchboard opsd_password: PASSWORD_TO_CHANGE admin_password: PASSWORD_TO_CHANGE @@ -16,14 +16,14 @@ specification: - readall - snapshotrestore paths: - ops_home: /usr/share/opensearch - ops_conf_dir: /usr/share/opensearch/config - ops_log_dir: /var/log/opensearch - ops_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin - ops_repo: /var/lib/opensearch-snapshots - ops_data: /var/lib/opensearch - ops_logs: /var/log/opensearch - ops_perftop_home: /usr/share/opensearch/perftop + opensearch_home: /usr/share/opensearch + opensearch_conf_dir: /usr/share/opensearch/config + opensearch_log_dir: /var/log/opensearch + opensearch_plugin_bin_path: /usr/share/opensearch/bin/opensearch-plugin + opensearch_repo: /var/lib/opensearch-snapshots + opensearch_data: /var/lib/opensearch + opensearch_logs: /var/log/opensearch + opensearch_perftop_home: /usr/share/opensearch/perftop opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin opsd_home: /usr/share/opensearch-dashboards opsd_conf_dir: /usr/share/opensearch-dashboards/config diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index 29779825f3..b3ae0ed208 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -5,7 +5,7 @@ specification: cluster_name: EpiphanyOpensearch odfe_migration: false clustered: true - ops_user: opensearch + opensearch_user: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: false @@ -18,13 +18,13 @@ specification: - logstash - kibanaserver paths: - ops_home: /usr/share/opensearch - ops_conf_dir: /usr/share/opensearch/config - ops_log_dir: /var/log/opensearch - ops_repo: /var/lib/opensearch-snapshots - ops_data: /var/lib/opensearch - ops_logs: /var/log/opensearch - ops_perftop_home: /usr/share/opensearch/perftop + opensearch_home: /usr/share/opensearch + opensearch_conf_dir: /usr/share/opensearch/config + opensearch_log_dir: /var/log/opensearch + opensearch_repo: /var/lib/opensearch-snapshots + opensearch_data: /var/lib/opensearch + opensearch_logs: /var/log/opensearch + opensearch_perftop_home: /usr/share/opensearch/perftop opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin opsd_home: /usr/share/opensearch-dashboards opsd_conf_dir: /usr/share/opensearch-dashboards/config diff --git a/schema/common/validation/configuration/opensearch-dashboards.yml b/schema/common/validation/configuration/opensearch-dashboards.yml index ddba5a2283..527cae109d 100644 --- a/schema/common/validation/configuration/opensearch-dashboards.yml +++ b/schema/common/validation/configuration/opensearch-dashboards.yml @@ -2,4 +2,4 @@ kind: configuration/opensearch-dashboards title: "OpenSearch Dashboards specification schema" name: default specification: - ops_dashboards_log_dir: /var/log/opensearchdashboards + opensearch_dashboards_log_dir: /var/log/opensearchdashboards From 6a50545958caad7b6ba5fff71d760d675ef03833 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 11:50:04 +0100 Subject: [PATCH 094/157] Using the newer version of control keyword Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index b19a038c5a..b833f00327 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -5,7 +5,7 @@ tasks_from: download_file vars: file_name: "{{ item }}" - with_items: + loop: - "opensearch-{{ versions[ansible_os_family].opensearch }}-linux-x64.tar.gz" - "opensearch-perf-top-{{ versions[ansible_os_family].opensearch_perftop }}-linux-x64.zip" From fba7418096b92338c68dee632985f26cd32b2dcd Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Wed, 23 Feb 2022 11:50:22 +0100 Subject: [PATCH 095/157] Using the newer version of control keyword Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index b833f00327..68aa289b0c 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -21,7 +21,7 @@ state: directory owner: "{{ specification.opensearch_user }}" group: "{{ specification.opensearch_user }}" - with_items: + loop: - "{{ specification.paths.opensearch_home }}" - "{{ specification.paths.opensearch_perftop_home }}" - "{{ specification.paths.opensearch_log_dir }}" From 891ca3eab90e08dc797d6b78b4be78eb0448e688 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 12:12:24 +0100 Subject: [PATCH 096/157] Harcoding the path as requested during PR rev. --- ansible/playbooks/roles/opensearch/defaults/main.yml | 1 - ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 2 +- ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml | 2 -- .../playbooks/roles/opensearch_dashboards/tasks/dashboards.yml | 2 +- .../playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml | 2 +- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index e6f5d0060a..5424379da2 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -61,4 +61,3 @@ ports: http: 9200 # defaults to range but we want static port transport: 9300 # defaults to range but we want static port log4j_file_name: apache-log4j-2.17.1-bin.tar.gz -systemctl_path: /etc/systemd/system diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 68aa289b0c..9f3e4d51a1 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -49,4 +49,4 @@ - name: Create systemd service template: src: roles/opensearch/templates/opensearch.service.j2 - dest: "{{ systemctl_path }}/opensearch.service" + dest: "/etc/systemd/system/opensearch.service" diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index bc918d826c..9fb8c9bfa4 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -16,5 +16,3 @@ opensearch_nodes_dashboards: |- {% for item in groups['opensearch_dashboards'] -%} https://{{ hostvars[item]['ansible_host'] }}:{{ opensearch_api_port }}{% if not loop.last %}","{% endif %} {%- endfor %} - -systemctl_path: /etc/systemd/system diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index 01ecbe09e4..b7be84dc46 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -40,4 +40,4 @@ - name: Create systemd service template: src: dashboards.service - dest: "{{ systemctl_path }}/dashboards.service" + dest: /etc/systemd/system/dashboards.service diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index a6196b82ee..af0a37f1bc 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -74,7 +74,7 @@ - name: Kibana migr | Create OPSD systemd service template: src: roles/opensearch_dashboards/templates/dashboards.service - dest: "{{ systemctl_path }}/dashboards.service" + dest: /etc/systemd/system/dashboards.service - name: Kibana migr | Assure Opensearch Dashboards service is started service: From 034e787fe0f837a3fb3074d6ee5c121a8bf7cea5 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 13:23:39 +0100 Subject: [PATCH 097/157] ansible_facts.fqdn works ( instead of domain_name ) also with --no-infra machines --- .../playbooks/roles/opensearch_dashboards/tasks/etchosts.yml | 4 ++-- ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml index e7b614f05a..90055890f1 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml @@ -3,8 +3,8 @@ blockinfile: dest: /etc/hosts block: |- - {% for item in groups['dashboards'] %} - {{ hostvars[item]['ip'] }} {{ item }}.{{ domain_name }} {{ item }} + {% for item in groups['opensearch_dashboards'] %} + {{ hostvars[item]['ansible_host'] }} {{ hostvars[item]['ansible_facts']['fqdn'] }} {{ item }} {% endfor %} state: present create: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index b250592a3a..361ab863cb 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -9,8 +9,8 @@ state: disabled when: (ansible_distribution != "Ubuntu") and (ansible_distribution != "Amazon") -# - name: Populate the nodes to /etc/hosts -# import_tasks: etchosts.yml +- name: Populate the nodes to /etc/hosts + import_tasks: etchosts.yml - name: include dashboards installation include: dashboards.yml From 06026de7cc2766819c1fb59d4f8f3af0b1865fc3 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Wed, 23 Feb 2022 18:04:18 +0100 Subject: [PATCH 098/157] Differ between architectures the vars describing OPS and OPSD binary --- ansible/playbooks/roles/opensearch/defaults/main.yml | 8 -------- .../playbooks/roles/opensearch/tasks/install-ops.yml | 11 +++++++---- .../playbooks/roles/opensearch/tasks/set_facts.yml | 12 ++++++++++++ .../roles/opensearch_dashboards/defaults/main.yml | 5 ----- .../roles/opensearch_dashboards/tasks/dashboards.yml | 7 +++++-- .../roles/opensearch_dashboards/tasks/set_facts.yml | 10 ++++++++++ .../centos-7/requirements.aarch64.txt | 4 ++++ .../playbooks/roles/upgrade/tasks/opensearch-01.yml | 7 +++++-- .../upgrade/tasks/opensearch/migrate-kibana.yml | 7 +++++-- .../roles/upgrade/tasks/opensearch/migrate-odfe.yml | 2 +- 10 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 ansible/playbooks/roles/opensearch/tasks/set_facts.yml create mode 100644 ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index 5424379da2..c0df8833ac 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -1,13 +1,5 @@ --- # This file is meant to be also used by upgrade role - -versions: - RedHat: - opensearch: "1.2.4" - opensearch_perftop: "1.1.0.0" - Debian: - opensearch: "1.2.4" - opensearch_perftop: "1.1.0.0" certificates: dirs: certs: /usr/share/opensearch/config diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index 9f3e4d51a1..ecf62ed774 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -1,4 +1,7 @@ --- +- name: Define the filenames for the download task + include_tasks: roles/opensearch/tasks/set_facts.yml + - name: Download Opensearch include_role: name: download @@ -6,8 +9,8 @@ vars: file_name: "{{ item }}" loop: - - "opensearch-{{ versions[ansible_os_family].opensearch }}-linux-x64.tar.gz" - - "opensearch-perf-top-{{ versions[ansible_os_family].opensearch_perftop }}-linux-x64.zip" + - "opensearch-{{ opensearch_ver }}" + - "opensearch-perf-top-{{ opensearch_perftop_ver }}" - name: Ensure Opensearch service user exists user: @@ -32,7 +35,7 @@ - name: Extract the OPS tar file unarchive: - src: "/tmp/opensearch-{{ versions[ansible_os_family].opensearch }}-linux-x64.tar.gz" + src: "/tmp/opensearch-{{ opensearch_ver }}" dest: "{{ specification.paths.opensearch_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes @@ -41,7 +44,7 @@ - name: Extract OPS PerfTop the tar file unarchive: - src: "/tmp/opensearch-perf-top-{{ versions[ansible_os_family].opensearch_perftop }}-linux-x64.zip" + src: "/tmp/opensearch-perf-top-{{ opensearch_perftop_ver }}" dest: "{{ specification.paths.opensearch_perftop_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/opensearch/tasks/set_facts.yml b/ansible/playbooks/roles/opensearch/tasks/set_facts.yml new file mode 100644 index 0000000000..80738abb01 --- /dev/null +++ b/ansible/playbooks/roles/opensearch/tasks/set_facts.yml @@ -0,0 +1,12 @@ +--- +- name: Set the name of binary to download for x86_64 family + set_fact: + opensearch_ver: "1.2.4-linux-x64.tar.gz" + opensearch_perftop_ver: "1.1.0.0-linux-x64.zip" + when: ansible_architecture == "x86_64" + +- name: Set the name of binary to download for arm64 family + set_fact: + opensearch_ver: "1.2.4-linux-arm64.tar.gz" + opensearch_perftop_ver: "1.1.0.0-arm64.zip" + when: ansible_architecture == "aarch64" diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index 9fb8c9bfa4..bcfa9e943e 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -1,9 +1,4 @@ --- -versions: - RedHat: - opsd_version: "1.2.0" - Debian: - opsd_version: "1.2.0" opensearch_nodes: |- {% for item in groups['ops-cluster'] -%} {{ hostvars[item]['ip'] }}{% if not loop.last %}","{% endif %} diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index b7be84dc46..e59b9b9d20 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -1,10 +1,13 @@ --- +- name: Define the filename for the download task + include_tasks: set_facts.yml + - name: Download Opensearch dashbaords include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + file_name: "opensearch-dashboards-{{ opsd_version }}" - name: Create OpenSearch Dashboards user user: @@ -21,7 +24,7 @@ - name: Extract the OPSD tar file unarchive: - src: "/tmp/opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + src: "/tmp/opensearch-dashboards-{{ opsd_version }}" dest: "{{ specification.paths.opsd_home }}" owner: "{{ specification.opsd_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml new file mode 100644 index 0000000000..b16bbbff6c --- /dev/null +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml @@ -0,0 +1,10 @@ +--- +- name: Set the name of binary to download for x86_64 family + set_fact: + opsd_version: "1.2.0-linux-x64.tar.gz" + when: ansible_architecture == "x86_64" + +- name: Set the name of binary to download for arm64 family + set_fact: + opsd_version: "1.2.0-linux-arm64.tar.gz" + when: ansible_architecture == "aarch64" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index f31b6c4bf3..06b905be41 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -185,6 +185,10 @@ https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_17 https://grafana.com/api/dashboards/6663/revisions/1/download grafana_dashboard_6663.json # RabbitMQ cluster monitoring (via Prometheus) https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard_10991.json +# OpenSearch +https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-arm64.tar.gz +# OpenSearch Dashboards +https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-arm64.tar.gz [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml index 5396bd5cbc..26d4bbdb17 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml @@ -20,6 +20,9 @@ file: roles/opensearch/vars/main.yml name: opensearch_vars +- name: OPS | Set the versions of OpenSearch + include_tasks: roles/opensearch/tasks/set_facts.yml + - name: Ensure Opensearch service user exists user: name: "{{ opensearch_vars.specification.opensearch_user }}" @@ -42,7 +45,7 @@ debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Target version: {{ opensearch_defaults.versions[ansible_os_family].opensearch }}" + - "Target version: {{ opensearch_ver.split('-')[0] }}" # If state file exists it means the previous run failed - name: OPS | Check if upgrade state file exists @@ -59,7 +62,7 @@ or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') and stat_upgrade_state_file.stat.exists) vars: - _target_version: "{{ opensearch_defaults.versions[ansible_os_family].opensearch }}" + _target_version: "{{ opensearch_ver.split('-')[0] }}" - include_role: name: upgrade diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index af0a37f1bc..ef966b8720 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -13,12 +13,15 @@ enabled: no state: stopped +- name: Kibana migr | Define the filename for the download task + include_tasks: roles/opensearch_dashboards/tasks/set_facts.yml + - name: Kibana migr | Download Opensearch Dashboards binary include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + file_name: "opensearch-dashboards-{{ opsd_version }}" - name: Kibana migr | Create opensearch-dashboards user user: @@ -40,7 +43,7 @@ - name: Kibana migr | Extract the tar file unarchive: - src: "/tmp/opensearch-dashboards-{{ versions[ansible_os_family].opsd_version }}-linux-x64.tar.gz" + src: "/tmp/opensearch-dashboards-{{ opsd_version }}" dest: "{{ specification.paths.opsd_home }}" owner: "{{ specification.opsd_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 1a1efee29f..edc1c7a2ae 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -8,7 +8,7 @@ debug: msg: - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Opensearch version to be installed: {{ opensearch_defaults.versions[ansible_os_family].opensearch }}" + - "Opensearch version to be installed: {{opensearch_ver }}" - name: ODFE migr | Ensure elasticsearch cluster is up and running block: From c7228c513f0dacc67b763a068009e7f8be30f4e6 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 24 Feb 2022 07:49:46 +0100 Subject: [PATCH 099/157] Removed as target OPS ver includes log4j patch --- .../roles/opensearch/tasks/patch-log4j.yml | 68 ------------------- 1 file changed, 68 deletions(-) delete mode 100644 ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml b/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml deleted file mode 100644 index b500e2160c..0000000000 --- a/ansible/playbooks/roles/opensearch/tasks/patch-log4j.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -- name: Log4j patch - block: - - name: "elasticsearch : Log4j patch | Get archive" - include_role: - name: download - tasks_from: download_file - vars: - file_name: "{{ log4j_file_name }}" - - - name: Log4j patch | Extract archive - unarchive: - dest: /tmp/ - src: "{{ download_directory }}/{{ log4j_file_name }}" - remote_src: true - list_files: true - register: unarchive_list_files - - - name: Log4j patch | Copy new jars - register: log4j_patch - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: elasticsearch - group: root - mode: u=rw,g=r,o= - remote_src: true - loop: - - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/lib/ } - - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_api }}", dest: /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/lib/ } - - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_core }}", dest: /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/ } - - { src: "{{ download_directory }}/{{ log4j_slfj_impl }}", dest: /usr/share/elasticsearch/plugins/elasticsearch_security/ } - vars: - log4j_api: "{{ unarchive_list_files.files | select('contains', 'log4j-api-2.17.1.jar') | first }}" - log4j_core: "{{ unarchive_list_files.files | select('contains', 'log4j-core-2.17.1.jar') | first }}" - log4j_slfj_impl: "{{ unarchive_list_files.files | select('contains', 'log4j-slf4j-impl-2.17.1.jar') | first }}" - - - name: Log4j patch - cleanup - block: - - name: Log4j patch | Remove old jars - file: - state: absent - path: "{{ item }}" - loop: - - /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/log4j-api-2.13.0.jar - - /usr/share/elasticsearch/plugins/elasticsearch-performance-analyzer/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-api-2.13.0.jar - - /usr/share/elasticsearch/performance-analyzer-rca/lib/log4j-core-2.13.0.jar - - /usr/share/elasticsearch/lib/log4j-api-2.11.1.jar - - /usr/share/elasticsearch/lib/log4j-core-2.11.1.jar - - /usr/share/elasticsearch/plugins/elasticsearch_security/log4j-slf4j-impl-2.11.1.jar - - - name: Log4j patch | Delete temporary dir - file: - dest: "{{ download_directory }}/{{ _archive_root_dir }}" - state: absent - vars: - _archive_root_dir: >- - {{ unarchive_list_files.files | first | dirname }} - -- name: Restart elasticsearch-performance-analyzer service - systemd: - name: elasticsearch-performance-analyzer - state: restarted - when: log4j_patch.changed From 19a4c036e490dec7efc0bc5cd39c5ed07ac5eb84 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Thu, 24 Feb 2022 07:51:56 +0100 Subject: [PATCH 100/157] Renaming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index ecf62ed774..c48bea8b4e 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -33,7 +33,7 @@ - "{{ specification.paths.opensearch_logs }}" - "{{ certificates.dirs.certs }}" -- name: Extract the OPS tar file +- name: Extract OpenSearch tar file unarchive: src: "/tmp/opensearch-{{ opensearch_ver }}" dest: "{{ specification.paths.opensearch_home }}" From 86adfccd267b56f53a9f0718e57898dfa969caff Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Thu, 24 Feb 2022 07:52:17 +0100 Subject: [PATCH 101/157] Renaming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index c48bea8b4e..c0fab2e808 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -42,7 +42,7 @@ extra_opts: - --strip-components=1 -- name: Extract OPS PerfTop the tar file +- name: Extract OpenSearch PerfTop tar file unarchive: src: "/tmp/opensearch-perf-top-{{ opensearch_perftop_ver }}" dest: "{{ specification.paths.opensearch_perftop_home }}" From c232ac53f45af97829400ee9a68a473cf4382dda Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Thu, 24 Feb 2022 07:52:51 +0100 Subject: [PATCH 102/157] Renaming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/install-ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml index c0fab2e808..cbe5f3b9d1 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-ops.yml @@ -49,7 +49,7 @@ owner: "{{ specification.opensearch_user }}" remote_src: yes -- name: Create systemd service +- name: Create opensearch.service unit file template: src: roles/opensearch/templates/opensearch.service.j2 dest: "/etc/systemd/system/opensearch.service" From beeb7f0de804e59070084e1572694d541fbb27ee Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Thu, 24 Feb 2022 07:53:18 +0100 Subject: [PATCH 103/157] Renaming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index f4161ce63c..500abefcde 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -16,7 +16,7 @@ - name: Tune the system settings import_tasks: tune.yml -- include_tasks: install-ops.yml +- include_tasks: install-opensearch.yml - name: Include configuration tasks include_tasks: configure-ops.yml From 8b650fb718ff40127ad01997f22b1ba33efe541f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Thu, 24 Feb 2022 07:53:32 +0100 Subject: [PATCH 104/157] Renaming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index 500abefcde..388e9e4051 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -19,4 +19,4 @@ - include_tasks: install-opensearch.yml - name: Include configuration tasks - include_tasks: configure-ops.yml + include_tasks: configure-opensearch.yml From e3a429ecbf7616d0340418315e00a2b40caf2592 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 24 Feb 2022 08:36:40 +0100 Subject: [PATCH 105/157] Renamed as requested during PR review --- .../tasks/{configure-ops.yml => configure-opensearch.yml} | 0 .../tasks/{install-ops.yml => install-opensearch.yml} | 0 .../roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml | 2 +- .../roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename ansible/playbooks/roles/opensearch/tasks/{configure-ops.yml => configure-opensearch.yml} (100%) rename ansible/playbooks/roles/opensearch/tasks/{install-ops.yml => install-opensearch.yml} (100%) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-ops.yml b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml similarity index 100% rename from ansible/playbooks/roles/opensearch/tasks/configure-ops.yml rename to ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml diff --git a/ansible/playbooks/roles/opensearch/tasks/install-ops.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml similarity index 100% rename from ansible/playbooks/roles/opensearch/tasks/install-ops.yml rename to ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index 50d7acb39d..231add8717 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -16,7 +16,7 @@ file: roles/opensearch/vars/main.yml - name: ODFE migr | Inastall Opensearch binaries - include_tasks: roles/opensearch/tasks/install-ops.yml + include_tasks: roles/opensearch/tasks/install-opensearch.yml - name: ODFE migr | Copy ES directories to OPS directories copy: diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml index 3730200eab..a060a6dc0e 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml @@ -25,12 +25,12 @@ - name: OPS | Include Elasticsearch installation tasks include_role: name: opensearch - tasks_from: install-ops.yml + tasks_from: install-opensearch.yml - name: OPS | Include Elasticsearch configuration tasks include_role: name: opensearch - tasks_from: configure-ops.yml + tasks_from: configure-opensearch.yml vars: _old: "{{ existing_config.main }}" # Keep the same data structure as for apply mode From 16f043a9f2cff11788ae2e0b267c8fc2018360e4 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 24 Feb 2022 09:27:31 +0100 Subject: [PATCH 106/157] Perftop is not supported on ARM --- .../roles/opensearch/tasks/install-opensearch.yml | 14 ++++++++++---- .../playbooks/roles/opensearch/tasks/set_facts.yml | 4 ++-- .../centos-7/requirements.x86_64.txt | 2 +- .../redhat-7/requirements.x86_64.txt | 2 +- .../ubuntu-20.04/requirements.x86_64.txt | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index cbe5f3b9d1..5b62f6761e 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -7,10 +7,15 @@ name: download tasks_from: download_file vars: - file_name: "{{ item }}" - loop: - - "opensearch-{{ opensearch_ver }}" - - "opensearch-perf-top-{{ opensearch_perftop_ver }}" + file_name: "opensearch-{{ opensearch_ver }}" + +- name: Download PerfTop + include_role: + name: download + tasks_from: download_file + vars: + file_name: "opensearch-perf-top-{{ opensearch_perftop_ver }}" + when: ansible_architecture == "x86_64" # Perftop is not yet supported on ARM (https://github.com/opensearch-project/perftop/issues/26) - name: Ensure Opensearch service user exists user: @@ -48,6 +53,7 @@ dest: "{{ specification.paths.opensearch_perftop_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes + when: ansible_architecture == "x86_64" # Perftop is not yet supported on ARM (https://github.com/opensearch-project/perftop/issues/26) - name: Create opensearch.service unit file template: diff --git a/ansible/playbooks/roles/opensearch/tasks/set_facts.yml b/ansible/playbooks/roles/opensearch/tasks/set_facts.yml index 80738abb01..0883e971f7 100644 --- a/ansible/playbooks/roles/opensearch/tasks/set_facts.yml +++ b/ansible/playbooks/roles/opensearch/tasks/set_facts.yml @@ -2,11 +2,11 @@ - name: Set the name of binary to download for x86_64 family set_fact: opensearch_ver: "1.2.4-linux-x64.tar.gz" - opensearch_perftop_ver: "1.1.0.0-linux-x64.zip" + opensearch_perftop_ver: "1.2.0.0-linux-x64.zip" when: ansible_architecture == "x86_64" - name: Set the name of binary to download for arm64 family set_fact: opensearch_ver: "1.2.4-linux-arm64.tar.gz" - opensearch_perftop_ver: "1.1.0.0-arm64.zip" + # Perftop is not supported on ARM (https://github.com/opensearch-project/perftop/issues/26) when: ansible_architecture == "aarch64" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index d1022a181e..4f1fbcae3e 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -191,7 +191,7 @@ https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2 # OpenSearch Dashboards https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz # OpenSearch PerfTop -https://github.com/opensearch-project/perftop/releases/download/1.1.0.0/opensearch-perf-top-1.1.0.0-linux-x64.zip +https://github.com/opensearch-project/perftop/releases/download/1.2.0.0/opensearch-perf-top-1.2.0.0-linux-x64.zip [images] diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index 245242764a..2d958fc7a3 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -189,7 +189,7 @@ https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2 # OpenSearch Dashboards https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz # OpenSearch PerfTop -https://github.com/opensearch-project/perftop/releases/download/1.1.0.0/opensearch-perf-top-1.1.0.0-linux-x64.zip +https://github.com/opensearch-project/perftop/releases/download/1.2.0.0/opensearch-perf-top-1.2.0.0-linux-x64.zip [images] haproxy:2.2.2-alpine diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt index 39633ad5f4..565b130a94 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-20.04/requirements.x86_64.txt @@ -226,7 +226,7 @@ https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2 # OpenSearch Dashboards https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz # OpenSearch PerfTop -https://github.com/opensearch-project/perftop/releases/download/1.1.0.0/opensearch-perf-top-1.1.0.0-linux-x64.zip +https://github.com/opensearch-project/perftop/releases/download/1.2.0.0/opensearch-perf-top-1.2.0.0-linux-x64.zip [images] haproxy:2.2.2-alpine From 20b692171a450c840cde8d600f2c8ddd5861e302 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Fri, 25 Feb 2022 08:02:50 +0100 Subject: [PATCH 107/157] Shortening the path when within a single role Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index 5b62f6761e..7a56dc3783 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -57,5 +57,5 @@ - name: Create opensearch.service unit file template: - src: roles/opensearch/templates/opensearch.service.j2 + src: opensearch.service.j2 dest: "/etc/systemd/system/opensearch.service" From b79c134f0ae9df62d5a3893001a87eea16b8af4b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Fri, 25 Feb 2022 08:06:35 +0100 Subject: [PATCH 108/157] Renaming change as requested during PR rev. Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../playbooks/roles/opensearch_dashboards/tasks/dashboards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index e59b9b9d20..12218026b6 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -22,7 +22,7 @@ owner: "{{ specification.opsd_user }}" group: "{{ specification.opsd_user }}" -- name: Extract the OPSD tar file +- name: Extract OpenSearch Dashboards tar file unarchive: src: "/tmp/opensearch-dashboards-{{ opsd_version }}" dest: "{{ specification.paths.opsd_home }}" From c309c79cd4e1b52b44c2f17789ff74e78b26e98b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <62810569+romsok24@users.noreply.github.com> Date: Fri, 25 Feb 2022 08:39:24 +0100 Subject: [PATCH 109/157] Update Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 361ab863cb..032d97a163 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -12,8 +12,8 @@ - name: Populate the nodes to /etc/hosts import_tasks: etchosts.yml -- name: include dashboards installation - include: dashboards.yml +- name: Include dashboards installation + include_tasks: dashboards.yml - name: Make sure OpenSearch Ddashboards is started service: From 09313581e1d67a2ddb0e5fe58070c1e7db642155 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 10:21:32 +0100 Subject: [PATCH 110/157] Renaming change as requested during PR rev. --- .../roles/opensearch/tasks/{tune.yml => configure-sysctl.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ansible/playbooks/roles/opensearch/tasks/{tune.yml => configure-sysctl.yml} (100%) diff --git a/ansible/playbooks/roles/opensearch/tasks/tune.yml b/ansible/playbooks/roles/opensearch/tasks/configure-sysctl.yml similarity index 100% rename from ansible/playbooks/roles/opensearch/tasks/tune.yml rename to ansible/playbooks/roles/opensearch/tasks/configure-sysctl.yml From 52a218a59e532667cc35891e411ba7f958981482 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 10:23:12 +0100 Subject: [PATCH 111/157] This action was doubled here --- .../roles/opensearch_dashboards/tasks/etchosts.yml | 13 ------------- .../roles/opensearch_dashboards/tasks/main.yml | 3 --- 2 files changed, 16 deletions(-) delete mode 100644 ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml deleted file mode 100644 index 90055890f1..0000000000 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/etchosts.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Hosts | populate inventory into hosts file - blockinfile: - dest: /etc/hosts - block: |- - {% for item in groups['opensearch_dashboards'] %} - {{ hostvars[item]['ansible_host'] }} {{ hostvars[item]['ansible_facts']['fqdn'] }} {{ item }} - {% endfor %} - state: present - create: yes - backup: yes - marker: "# Ansible inventory hosts {mark}" - when: populate_inventory_to_hosts_file diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 032d97a163..12a160d9c0 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -9,9 +9,6 @@ state: disabled when: (ansible_distribution != "Ubuntu") and (ansible_distribution != "Amazon") -- name: Populate the nodes to /etc/hosts - import_tasks: etchosts.yml - - name: Include dashboards installation include_tasks: dashboards.yml From d355c5d15fb641271495864f6132f066b9ab6e3f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 10:26:20 +0100 Subject: [PATCH 112/157] Rewariting on canonical yaml format --- .../playbooks/roles/opensearch_dashboards/handlers/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml index 69d704bd1f..5dc4ae19e6 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml @@ -1,3 +1,6 @@ --- - name: restart dashboards - systemd: name=dashboards state=restarted enabled=yes + systemd: + name: dashboards + state: restarted + enabled: yes From fb2dc26fb61d5ff3a89b26d56e5c7ef3cb126003 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 10:26:51 +0100 Subject: [PATCH 113/157] Renaming --- ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml | 2 +- ansible/playbooks/roles/opensearch/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index 7a56dc3783..5b62f6761e 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -57,5 +57,5 @@ - name: Create opensearch.service unit file template: - src: opensearch.service.j2 + src: roles/opensearch/templates/opensearch.service.j2 dest: "/etc/systemd/system/opensearch.service" diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index 388e9e4051..adb984b38d 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -14,7 +14,7 @@ - { limit_type: 'hard', limit_item: 'memlock', value: unlimited } - name: Tune the system settings - import_tasks: tune.yml + import_tasks: configure-sysctl.yml - include_tasks: install-opensearch.yml From b7bf2433ee5a6d016b05261e954487d6c679b072 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 11:12:24 +0100 Subject: [PATCH 114/157] Using ansible sudo mechanism --- .../playbooks/roles/opensearch_dashboards/tasks/main.yml | 9 ++------- .../roles/upgrade/tasks/opensearch/migrate-kibana.yml | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 12a160d9c0..97020b7579 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -3,12 +3,6 @@ - hostname: name: "{{ inventory_hostname }}" -# Disabling for Amazon Linux 2 as selinux is disabled by default. -- name: Disable the selinux - selinux: - state: disabled - when: (ansible_distribution != "Ubuntu") and (ansible_distribution != "Amazon") - - name: Include dashboards installation include_tasks: dashboards.yml @@ -19,7 +13,8 @@ enabled: yes - name: Get all the installed dashboards plugins - command: "sudo -u {{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + command: "{{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + become: true register: list_plugins - name: Show all the installed dashboards plugins diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index ef966b8720..81b0440c66 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -86,7 +86,8 @@ enabled: yes - name: Kibana migr | Get all the installed dashboards plugins - command: "sudo -u {{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + command: "{{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + become: true register: list_plugins - name: Kibana migr | Show all the installed dashboards plugins From 9033886a9cb4686f766adffbb58d45d198c4203a Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 11:53:40 +0100 Subject: [PATCH 115/157] Var name optimization --- .../templates/opensearch_dashboards.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index 71cf9ca2fd..9520ce2c83 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -1,5 +1,5 @@ server.port: 5601 -server.host: "{{ hostvars[inventory_hostname]['ansible_host'] }}" +server.host: "{{ ansible_host }}" opensearch.hosts: ["{{ opensearch_nodes_dashboards }}"] opensearch.ssl.verificationMode: none opensearch.username: "{{ specification.opsd_user }}" From 7edeca1da13a685040eb48e33bd39ea86274c8c6 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 11:54:54 +0100 Subject: [PATCH 116/157] Using ansible sudo mechanism --- ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml | 3 ++- .../roles/upgrade/tasks/opensearch/migrate-kibana.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 97020b7579..8ecb8e126b 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -13,8 +13,9 @@ enabled: yes - name: Get all the installed dashboards plugins - command: "{{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + command: "{{ specification.paths.opsd_plugin_bin_path }} list" become: true + become_user: "{{ specification.opsd_user }}" register: list_plugins - name: Show all the installed dashboards plugins diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 81b0440c66..fb5610aff6 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -86,7 +86,7 @@ enabled: yes - name: Kibana migr | Get all the installed dashboards plugins - command: "{{ specification.opsd_user }} {{ specification.paths.opsd_plugin_bin_path }} list" + command: "{{ specification.paths.opsd_plugin_bin_path }} list" become: true register: list_plugins From c064496929ab53ec4a80bead3b2a8f0e6dc80137 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 12:03:26 +0100 Subject: [PATCH 117/157] Moved to defaults --- .../playbooks/roles/opensearch_dashboards/defaults/main.yml | 1 + ansible/playbooks/roles/opensearch_dashboards/vars/main.yml | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 ansible/playbooks/roles/opensearch_dashboards/vars/main.yml diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index bcfa9e943e..4026e0602f 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -11,3 +11,4 @@ opensearch_nodes_dashboards: |- {% for item in groups['opensearch_dashboards'] -%} https://{{ hostvars[item]['ansible_host'] }}:{{ opensearch_api_port }}{% if not loop.last %}","{% endif %} {%- endfor %} +java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" diff --git a/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml b/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml deleted file mode 100644 index 017318905e..0000000000 --- a/ansible/playbooks/roles/opensearch_dashboards/vars/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# vars file for openseOpenSearcharch -java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" From fe6f0382bb63ac7389f6132e4a3aa73ed4a5b405 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 12:07:34 +0100 Subject: [PATCH 118/157] The roles are supported for the give arch --- ansible/playbooks/roles/preflight/defaults/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index cc27378795..2cbb5f877d 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -38,7 +38,6 @@ unsupported_roles: - haproxy - logging - elasticsearch_curator - - opensearch - kibana - filebeat - prometheus @@ -74,7 +73,6 @@ unsupported_roles: - haproxy - logging - elasticsearch_curator - - opensearch - kibana - filebeat - prometheus From fbaa71443f8640ebb3fe0e45426876a76e64d0df Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 12:08:59 +0100 Subject: [PATCH 119/157] A typo correction --- ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 8ecb8e126b..ee42fdf1bd 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -6,7 +6,7 @@ - name: Include dashboards installation include_tasks: dashboards.yml -- name: Make sure OpenSearch Ddashboards is started +- name: Make sure OpenSearch Dashboards is started service: name: dashboards state: started From 95b46d722a2db012027a97e60d88f956e70e5c1c Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 12:10:53 +0100 Subject: [PATCH 120/157] A typo correction --- ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml | 2 +- .../playbooks/roles/opensearch_dashboards/tasks/dashboards.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml index 5dc4ae19e6..032ed5b73f 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml @@ -1,5 +1,5 @@ --- -- name: restart dashboards +- name: Restart dashboards systemd: name: dashboards state: restarted diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index 12218026b6..b5c836ff13 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -31,7 +31,7 @@ extra_opts: - --strip-components=1 -- name: Copy Configuration File +- name: Copy configuration file template: src: opensearch_dashboards.yml.j2 dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" From 44ef557fb0d27e93ea2ec9ba222173aa714d567a Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 12:40:42 +0100 Subject: [PATCH 121/157] Renaming change as requested during PR rev. --- .../playbooks/roles/opensearch_dashboards/handlers/main.yml | 2 +- .../roles/opensearch_dashboards/tasks/dashboards.yml | 6 +++--- .../playbooks/roles/opensearch_dashboards/tasks/main.yml | 2 +- ...{dashboards.service => opensearch-dashboards.service.j2} | 0 .../recovery/tasks/logging_opensearch_dashboards_conf.yml | 4 ++-- .../roles/recovery/tasks/logging_opensearch_snapshot.yml | 4 ++-- .../roles/upgrade/tasks/opensearch/migrate-kibana.yml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) rename ansible/playbooks/roles/opensearch_dashboards/templates/{dashboards.service => opensearch-dashboards.service.j2} (100%) diff --git a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml index 032ed5b73f..09474e767a 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/handlers/main.yml @@ -1,6 +1,6 @@ --- - name: Restart dashboards systemd: - name: dashboards + name: opensearch-dashboards state: restarted enabled: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index b5c836ff13..f110203cc8 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -40,7 +40,7 @@ mode: 0644 backup: yes -- name: Create systemd service +- name: Create opensearch-dashboards.service unit file template: - src: dashboards.service - dest: /etc/systemd/system/dashboards.service + src: opensearch-dashboards.service.j2 + dest: /etc/systemd/system/opensearch-dashboards.service diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index ee42fdf1bd..b682cf9b53 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -8,7 +8,7 @@ - name: Make sure OpenSearch Dashboards is started service: - name: dashboards + name: opensearch-dashboards state: started enabled: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 similarity index 100% rename from ansible/playbooks/roles/opensearch_dashboards/templates/dashboards.service rename to ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml index 05d69f6456..c547edacf7 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml @@ -22,7 +22,7 @@ - name: Stop OPSD service systemd: - name: dashboards + name: opensearch-dashboards state: stopped - name: Clear directories @@ -39,5 +39,5 @@ - name: Start OPSD service systemd: - name: dashboards + name: opensearch-dashboards state: started diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml index 15b360d45c..fa56a7a777 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml @@ -73,7 +73,7 @@ - name: Stop all OPS Dashboards instances delegate_to: "{{ item }}" systemd: - name: dashboards + name: opensearch-dashboards state: stopped enabled: false loop: "{{ groups.kibana | default([]) }}" @@ -108,7 +108,7 @@ - name: Start all OPS Dashboards instances delegate_to: "{{ item }}" systemd: - name: dashboards + name: opensearch-dashboards state: started enabled: true loop: "{{ groups.kibana | default([]) }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index fb5610aff6..e77af2ba22 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -81,7 +81,7 @@ - name: Kibana migr | Assure Opensearch Dashboards service is started service: - name: dashboards + name: opensearch-dashboards state: started enabled: yes From 64bc9a8f1071bce41628b10af79d87f82d1f7310 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 12:46:22 +0100 Subject: [PATCH 122/157] Renaming change as requested during PR rev. --- .../tasks/logging_opensearch_dashboards_conf.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml index c547edacf7..fcbfcd0f2e 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_dashboards_conf.yml @@ -2,7 +2,7 @@ - name: Include vars from opensearch role include_vars: file: roles/opensearch_dashboards/vars/main.yml - name: opsd_vars + name: opensearch_dashboards_vars - name: Find snapshot archive import_tasks: common/find_snapshot_archive.yml @@ -20,7 +20,7 @@ - name: Verify snapshot checksum import_tasks: common/verify_snapshot_checksum.yml -- name: Stop OPSD service +- name: Stop opensearch-dashboards service systemd: name: opensearch-dashboards state: stopped @@ -29,15 +29,15 @@ import_tasks: common/clear_directories.yml vars: dirs_to_clear: - - "{{ opsd_vars.specification.paths.opsd_conf_dir }}" + - "{{ opensearch_dashboards_vars.specification.paths.opsd_conf_dir }}" - name: Extract the archive unarchive: - dest: "{{ opsd_vars.specification.paths.opsd_conf_dir }}" + dest: "{{ opensearch_dashboards_vars.specification.paths.opsd_conf_dir }}" src: "{{ recovery_dir }}/{{ snapshot_path | basename }}" remote_src: true -- name: Start OPSD service +- name: Start opensearch-dashboards service systemd: name: opensearch-dashboards state: started From 3cf854e05f6bb5c7875b896461fb0d0305d61ff6 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 13:16:18 +0100 Subject: [PATCH 123/157] Renaming change as requested during PR rev. --- .../roles/recovery/tasks/logging_opensearch_snapshot.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml index fa56a7a777..dde65f50b9 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml @@ -70,13 +70,13 @@ - name: Ensure all OPSD and filebeat instances are stopped, then restore the snapshot block: - - name: Stop all OPS Dashboards instances + - name: Stop allOpenSearch Dashboards instances delegate_to: "{{ item }}" systemd: name: opensearch-dashboards state: stopped enabled: false - loop: "{{ groups.kibana | default([]) }}" + loop: "{{ groups.opensearch_dashboards | default([]) }}" - name: Stop all filebeat instances delegate_to: "{{ item }}" @@ -105,13 +105,13 @@ method: POST always: - - name: Start all OPS Dashboards instances + - name: Start allOpenSearch Dashboards instances delegate_to: "{{ item }}" systemd: name: opensearch-dashboards state: started enabled: true - loop: "{{ groups.kibana | default([]) }}" + loop: "{{ groups.opensearch_dashboards | default([]) }}" - name: Start all filebeat instances delegate_to: "{{ item }}" From 4421210701190b7048d841978220e1862cd936bb Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 14:54:23 +0100 Subject: [PATCH 124/157] Using a var instead of hardoded value --- .../roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index 231add8717..3722169728 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -61,7 +61,7 @@ replace: "{{ item.2 }}" with_items: - { 1: 'elasticsearch', 2: 'opensearch' } - - { 1: '\${ES_TMPDIR}', 2: '/tmp' } + - { 1: '\${ES_TMPDIR}', 2: '${OPENSEARCH_TMPDIR}' } - name: ODFE migr | Clone main configuration file copy: From f6a4d34bdef8b775d476abb96fb1ff2d55f72d5d Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 14:55:19 +0100 Subject: [PATCH 125/157] Renaming change as requested during PR rev. --- .../roles/upgrade/tasks/opensearch/migrate-kibana.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index e77af2ba22..9a8e1683b1 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -74,10 +74,10 @@ - { 1: 'telemetry.optIn', 2: '#telemetry.optIn' } - { 1: 'telemetry.enabled', 2: '#telemetry.enabled' } -- name: Kibana migr | Create OPSD systemd service +- name: Kibana migr | Create OpenSearch Dashboards service template: - src: roles/opensearch_dashboards/templates/dashboards.service - dest: /etc/systemd/system/dashboards.service + src: roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 + dest: /etc/systemd/system/opensearch-dashboards.service - name: Kibana migr | Assure Opensearch Dashboards service is started service: @@ -88,6 +88,7 @@ - name: Kibana migr | Get all the installed dashboards plugins command: "{{ specification.paths.opsd_plugin_bin_path }} list" become: true + become_user: "{{ specification.opsd_user }}" register: list_plugins - name: Kibana migr | Show all the installed dashboards plugins From f7db9bbf471fc917e1fcd7f639d6b43c9ebb6c59 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 15:29:10 +0100 Subject: [PATCH 126/157] Renaming change as requested during PR rev --- .../opensearch_dashboards/tasks/dashboards.yml | 12 ++++++------ .../roles/opensearch_dashboards/tasks/main.yml | 2 +- .../templates/opensearch-dashboards.service.j2 | 4 ++-- .../templates/opensearch_dashboards.yml.j2 | 4 ++-- .../upgrade/tasks/opensearch/migrate-kibana.yml | 14 +++++++------- .../configuration/opensearch-dashboards.yml | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index f110203cc8..dcd6898b4b 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -11,7 +11,7 @@ - name: Create OpenSearch Dashboards user user: - name: "{{ specification.opsd_user }}" + name: "{{ specification.dashboards_user }}" state: present shell: /bin/bash @@ -19,14 +19,14 @@ file: path: "{{ specification.paths.opsd_home }}" state: directory - owner: "{{ specification.opsd_user }}" - group: "{{ specification.opsd_user }}" + owner: "{{ specification.dashboards_user }}" + group: "{{ specification.dashboards_user }}" - name: Extract OpenSearch Dashboards tar file unarchive: src: "/tmp/opensearch-dashboards-{{ opsd_version }}" dest: "{{ specification.paths.opsd_home }}" - owner: "{{ specification.opsd_user }}" + owner: "{{ specification.dashboards_user }}" remote_src: yes extra_opts: - --strip-components=1 @@ -35,8 +35,8 @@ template: src: opensearch_dashboards.yml.j2 dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" - owner: "{{ specification.opsd_user }}" - group: "{{ specification.opsd_user }}" + owner: "{{ specification.dashboards_user }}" + group: "{{ specification.dashboards_user }}" mode: 0644 backup: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index b682cf9b53..7b1eb2dd65 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -15,7 +15,7 @@ - name: Get all the installed dashboards plugins command: "{{ specification.paths.opsd_plugin_bin_path }} list" become: true - become_user: "{{ specification.opsd_user }}" + become_user: "{{ specification.dashboards_user }}" register: list_plugins - name: Show all the installed dashboards plugins diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 index 47ef120956..10b84f2a61 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 @@ -9,8 +9,8 @@ PrivateTmp=true WorkingDirectory={{ specification.paths.opsd_home }} -User={{ specification.opsd_user }} -Group={{ specification.opsd_user }} +User={{ specification.dashboards_user }} +Group={{ specification.dashboards_user }} ExecStart={{ specification.paths.opsd_home }}/bin/opensearch-dashboards -q diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index 9520ce2c83..8f31fc3943 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -2,8 +2,8 @@ server.port: 5601 server.host: "{{ ansible_host }}" opensearch.hosts: ["{{ opensearch_nodes_dashboards }}"] opensearch.ssl.verificationMode: none -opensearch.username: "{{ specification.opsd_user }}" -opensearch.password: "{{ specification.opsd_password }}" +opensearch.username: "{{ specification.dashboards_user }}" +opensearch.password: "{{ specification.dashboards_password }}" opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] opensearch_security.multitenancy.enabled: true diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 9a8e1683b1..ec68a26216 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -25,8 +25,8 @@ - name: Kibana migr | Create opensearch-dashboards user user: - name: "{{ specification.opsd_user }}" - password: "{{ specification.opsd_password }}" + name: "{{ specification.dashboards_user }}" + password: "{{ specification.dashboards_password }}" state: present shell: /bin/bash @@ -34,8 +34,8 @@ file: path: "{{ item }}" state: directory - owner: "{{ specification.opsd_user }}" - group: "{{ specification.opsd_user }}" + owner: "{{ specification.dashboards_user }}" + group: "{{ specification.dashboards_user }}" mode: ug=rwx,o=rx with_items: - "{{ specification.paths.opsd_log_dir }}" @@ -45,7 +45,7 @@ unarchive: src: "/tmp/opensearch-dashboards-{{ opsd_version }}" dest: "{{ specification.paths.opsd_home }}" - owner: "{{ specification.opsd_user }}" + owner: "{{ specification.dashboards_user }}" remote_src: yes extra_opts: - --strip-components=1 @@ -55,7 +55,7 @@ src: /etc/kibana/kibana.yml dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" remote_src: yes - owner: "{{ specification.opsd_user }}" + owner: "{{ specification.dashboards_user }}" group: root mode: ug=rw,o= backup: yes @@ -88,7 +88,7 @@ - name: Kibana migr | Get all the installed dashboards plugins command: "{{ specification.paths.opsd_plugin_bin_path }} list" become: true - become_user: "{{ specification.opsd_user }}" + become_user: "{{ specification.dashboards_user }}" register: list_plugins - name: Kibana migr | Show all the installed dashboards plugins diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml index 07281b528d..968c9b2705 100644 --- a/schema/common/defaults/configuration/opensearch-dashboards.yml +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -2,8 +2,8 @@ kind: configuration/opensearch-dashboards title: "OpenSearch-Dashboards" name: default specification: - opsd_user: opensearchdboard - opsd_password: PASSWORD_TO_CHANGE + dashboards_user: opensearchdboard + dashboards_password: PASSWORD_TO_CHANGE paths: opsd_home: /usr/share/opensearch-dashboards opsd_conf_dir: /usr/share/opensearch-dashboards/config From c11f2261478373be43d9aba45cdb05b117782429 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 15:37:43 +0100 Subject: [PATCH 127/157] Reaplcing kibana --- schema/common/defaults/configuration/feature-mapping.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index 3b99b4a2a0..43de112f41 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -21,7 +21,7 @@ specification: enabled: true - name: elasticsearch-curator enabled: true - - name: kibana + - name: opensearch-dashboards enabled: true - name: filebeat enabled: true @@ -66,7 +66,7 @@ specification: - firewall logging: - logging - - kibana + - opensearch-dashboards - node-exporter - filebeat - firewall From efb2b05b1ad60aff4006def2fbce5565f8ed7d1b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 25 Feb 2022 16:13:02 +0100 Subject: [PATCH 128/157] Using a var instead of hardoded value --- ansible/playbooks/roles/opensearch/templates/jvm.options.j2 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 index 43ba467b7a..def6b9e830 100644 --- a/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 +++ b/ansible/playbooks/roles/opensearch/templates/jvm.options.j2 @@ -51,9 +51,7 @@ 14-:-XX:InitiatingHeapOccupancyPercent=30 ## JVM temporary directory -# Workaround for: opensearch[38222]: ERROR: Temporary file directory [${ES_TMPDIR}] does not exist or is not accessible -# -Djava.io.tmpdir=${ES_TMPDIR} --Djava.io.tmpdir=/tmp +-Djava.io.tmpdir=${OPENSEARCH_TMPDIR} ## heap dumps From 2392aace15ba68e47e69e60ec609840754af98ef Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 09:06:27 +0100 Subject: [PATCH 129/157] Changing the type of porting the tasks from stat to dyn --- ansible/playbooks/roles/opensearch/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index adb984b38d..e81ef02c63 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -14,7 +14,7 @@ - { limit_type: 'hard', limit_item: 'memlock', value: unlimited } - name: Tune the system settings - import_tasks: configure-sysctl.yml + include_tasks: configure-sysctl.yml - include_tasks: install-opensearch.yml From dc2b2768b91ced489bf71531ebebe99994acc854 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 09:07:18 +0100 Subject: [PATCH 130/157] Not used now --- .../roles/opensearch_dashboards/defaults/main.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index 4026e0602f..715efaca8f 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -1,11 +1,4 @@ --- -opensearch_nodes: |- - {% for item in groups['ops-cluster'] -%} - {{ hostvars[item]['ip'] }}{% if not loop.last %}","{% endif %} - {%- endfor %} - -populate_inventory_to_hosts_file: true - opensearch_api_port: 9200 opensearch_nodes_dashboards: |- {% for item in groups['opensearch_dashboards'] -%} From 3994b3aca34c1be1f48fa7e0191d9623d8882b89 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 09:08:11 +0100 Subject: [PATCH 131/157] This can not be run as root --- ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml | 3 +-- .../roles/upgrade/tasks/opensearch/migrate-kibana.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 7b1eb2dd65..352c417062 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -14,8 +14,7 @@ - name: Get all the installed dashboards plugins command: "{{ specification.paths.opsd_plugin_bin_path }} list" - become: true - become_user: "{{ specification.dashboards_user }}" + become: false # This command can not be run as root user register: list_plugins - name: Show all the installed dashboards plugins diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index ec68a26216..dfae66d36b 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -87,8 +87,7 @@ - name: Kibana migr | Get all the installed dashboards plugins command: "{{ specification.paths.opsd_plugin_bin_path }} list" - become: true - become_user: "{{ specification.dashboards_user }}" + become: false # This command can not be run as root user register: list_plugins - name: Kibana migr | Show all the installed dashboards plugins From f9e5c709347013eeaf6e024d8e7d9184bbc2a671 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 12:18:08 +0100 Subject: [PATCH 132/157] Needed for succesfull bck/recov because as workaround of err:Kibana API is not available --- .../roles/upgrade/tasks/opensearch/migrate-kibana.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index dfae66d36b..e69eb068bc 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -93,3 +93,9 @@ - name: Kibana migr | Show all the installed dashboards plugins debug: msg: "{{ list_plugins.stdout }}" + +- name: Kibana migr | Prevent Filebeat API access problem # Workaround for https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656 + replace: + path: /etc/filebeat/filebeat.yml + regexp: 'setup.dashboards.enabled: true' + replace: 'setup.dashboards.enabled: false' From 8603423fb92b90599ca328fa26b4de224ab50045 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 15:02:21 +0100 Subject: [PATCH 133/157] We do not support upgr of not EPI provided versions of ODFE --- .../roles/upgrade/tasks/opensearch-01.yml | 79 ------------- .../roles/upgrade/tasks/opensearch-02.yml | 13 --- .../opensearch/upgrade-opensearch-02.yml | 109 ------------------ 3 files changed, 201 deletions(-) delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml deleted file mode 100644 index 26d4bbdb17..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-01.yml +++ /dev/null @@ -1,79 +0,0 @@ ---- -- name: OPS | Get information about installed packages as facts - package_facts: - manager: auto - when: ansible_facts.packages is undefined - -- name: OPS | Assert that elasticsearch-oss package is installed - assert: - that: ansible_facts.packages['elasticsearch-oss'] is defined - fail_msg: elasticsearch-oss package not found, nothing to upgrade - quiet: true - -- name: OPS | Include defaults from OpenSearch role - include_vars: - file: roles/opensearch/defaults/main.yml - name: opensearch_defaults - -- name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml - include_vars: - file: roles/opensearch/vars/main.yml - name: opensearch_vars - -- name: OPS | Set the versions of OpenSearch - include_tasks: roles/opensearch/tasks/set_facts.yml - -- name: Ensure Opensearch service user exists - user: - name: "{{ opensearch_vars.specification.opensearch_user }}" - state: present - shell: /bin/bash - -- name: Ensure directory structure exists - file: - path: "{{ item }}" - state: directory - owner: "{{ opensearch_vars.specification.opensearch_user }}" - group: "{{ opensearch_vars.specification.opensearch_user }}" - with_items: - - "{{ opensearch_vars.specification.paths.opensearch_home }}" - - "{{ opensearch_vars.specification.paths.opensearch_log_dir }}" - - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" - - "{{ opensearch_defaults.certificates.dirs.certs }}" - -- name: OPS | Print ElasticSearch ond OpenSearch versions - debug: - msg: - - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Target version: {{ opensearch_ver.split('-')[0] }}" - -# If state file exists it means the previous run failed -- name: OPS | Check if upgrade state file exists - stat: - path: "{{ opensearch.upgrade_state_file_path }}" - get_attributes: false - get_checksum: false - get_mime: false - register: stat_upgrade_state_file - -- name: OPS | Upgrade Elasticsearch and ODFE plugins (part 1/2) - include_tasks: opensearch/upgrade-opensearch-01.yml - when: _target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') - or (_target_version is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') - and stat_upgrade_state_file.stat.exists) - vars: - _target_version: "{{ opensearch_ver.split('-')[0] }}" - -- include_role: - name: upgrade - tasks_from: opensearch/migrate-odfe - when: opensearch_vars.specification.odfe_migration - vars: - current_group_name: logging - -- include_role: - name: upgrade - tasks_from: opensearch/migrate-kibana - when: opensearch_vars.specification.odfe_migration - vars: - current_group_name: logging diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml deleted file mode 100644 index 06107ecde1..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch-02.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# If state file exists, it means upgrade has been started by the previous play and should be continued -- name: OPS | Check if upgrade state file exists - stat: - path: "{{ opensearch.upgrade_state_file_path }}" - get_attributes: false - get_checksum: false - get_mime: false - register: stat_upgrade_state_file - -- name: OPS | Upgrade Elasticsearch and ODFE plugins (part 2/2) - include_tasks: opensearch/upgrade-opensearch-02.yml - when: stat_upgrade_state_file.stat.exists diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml deleted file mode 100644 index a060a6dc0e..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-02.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -# This file contains flow that cannot be run in parallel on multiple hosts because of rolling upgrades. -# It's run after upgrade-opensearch-01.yml so some facts are already set. - -# Run migration procedure - the second (serial) part -- include_tasks: opensearch/migrate-from-demo-certs-02.yml - when: - - es_api.cert_type == 'demo' - - es_clustered # rolling upgrade only for clustered installation - -- name: OPS | Print API facts - debug: - var: es_api - tags: [ never, debug ] # only runs when debug or never tag requested - -- name: OPS | Prepare cluster for rolling upgrade - include_tasks: opensearch/utils/prepare-cluster-for-node-restart.yml - when: es_clustered - -- name: OPS | Stop elasticsearch service - systemd: - name: opensearch - state: stopped - -- name: OPS | Include Elasticsearch installation tasks - include_role: - name: opensearch - tasks_from: install-opensearch.yml - -- name: OPS | Include Elasticsearch configuration tasks - include_role: - name: opensearch - tasks_from: configure-opensearch.yml - vars: - _old: "{{ existing_config.main }}" - # Keep the same data structure as for apply mode - specification: - jvm_options: "{{ existing_config.jvm_options }}" - cluster_name: "{{ _old['cluster.name'] }}" - clustered: "{{ 'True' if _old['discovery.seed_hosts'] | length > 1 else 'False' }}" - paths: - data: "{{ _old['path.data'] }}" - repo: "{{ _old['path.repo'] | default('/var/lib/opensearch-snapshots') }}" # absent in Epiphany v0.6 thus we use default - logs: "{{ _old['path.logs'] }}" - opensearch_security: - ssl: - transport: - enforce_hostname_verification: "{{ _old['opensearch_security.ssl.transport.enforce_hostname_verification'] }}" - - _demo_DNs: - admin: "{{ opensearch.certs_migration.demo_DNs.admin }}" - node: "{{ opensearch.certs_migration.demo_DNs.node }}" - _dual_root_ca_filename: "{{ opensearch.certs_migration.dual_root_ca.filename }}" - _epiphany_root_ca_filename: "{{ certificates.files.root_ca.cert.filename }}" - _updated_existing_config: - opensearch_security.authcz.admin_dn: "{{ _old['opensearch_security.authcz.admin_dn'] | reject('search', _demo_DNs.admin) }}" - opensearch_security.nodes_dn: "{{ _old['opensearch_security.nodes_dn'] | default([]) | reject('search', _demo_DNs.node) }}" - opensearch_security.ssl.http.pemtrustedcas_filepath: >- - {{ _old['opensearch_security.ssl.http.pemtrustedcas_filepath'] | replace(_dual_root_ca_filename, _epiphany_root_ca_filename) }} - opensearch_security.ssl.transport.pemtrustedcas_filepath: >- - {{ _old['opensearch_security.ssl.transport.pemtrustedcas_filepath'] | replace(_dual_root_ca_filename, _epiphany_root_ca_filename) }} - - http.port: "{{ _old['http.port'] | default(odfe_defaults.ports.http) }}" - transport.port: "{{ _old['transport.port'] | default(odfe_defaults.ports.transport) }}" - - existing_es_config: "{{ _old | combine(_updated_existing_config) }}" - -- name: OPS | Include upgrade plugins tasks - include_tasks: opensearch/upgrade-plugins.yml - -# Restart elasticsearch service (unconditionally to ensure this task is not skipped in case of rerunning after interruption) -- include_tasks: opensearch/utils/restart-node.yml - vars: - daemon_reload: true # opensearch-performance-analyzer provides opensearch-performance-analyzer.service - target_inventory_hostname: "{{ inventory_hostname }}" - -# Post-upgrade tasks - -- name: Re-enable shard allocation - when: es_clustered - block: - - include_tasks: opensearch/utils/enable-shard-allocation.yml - - - include_tasks: opensearch/utils/wait-for-shard-allocation.yml - -# Read cluster health status from before the upgrade -- name: Load upgrade state file - slurp: - src: "{{ opensearch.upgrade_state_file_path }}" - register: slurp_upgrade_state_file - -# Verify cluster status -- include_tasks: opensearch/utils/wait-for-cluster-status.yml - when: not es_clustered or - (es_clustered and inventory_hostname == ansible_play_hosts_all[-1]) # for 'green' status at least 2 nodes must be already upgraded - vars: - initial_status: "{{ (slurp_upgrade_state_file.content | b64decode | from_json)['status'] }}" - expected_status: "{{ [ initial_status, 'green'] | unique }}" - -- name: OPS | Remove dual root CA temporary file - file: - path: "{{ (certificates.dirs.certs, opensearch.certs_migration.dual_root_ca.filename) | path_join }}" - state: absent - when: es_api.cert_type == 'Epiphany' - -- name: OPS | Remove upgrade state file - file: - path: "{{ opensearch.upgrade_state_file_path }}" - state: absent From 735db7ee44956a6b73f68ce46b0d49e8e7c8fac7 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 15:03:46 +0100 Subject: [PATCH 134/157] simplifying the structure due to lack of not EPI provided versions of ODFE --- .../roles/upgrade/tasks/opensearch.yml | 71 +++++++++++++++++++ ansible/playbooks/upgrade.yml | 39 +--------- 2 files changed, 73 insertions(+), 37 deletions(-) create mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml new file mode 100644 index 0000000000..be48b7d8a1 --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml @@ -0,0 +1,71 @@ +--- +- name: OPS | Get information about installed packages as facts + package_facts: + manager: auto + when: ansible_facts.packages is undefined + +- name: OPS | Assert that elasticsearch-oss package is installed + assert: + that: ansible_facts.packages['elasticsearch-oss'] is defined + fail_msg: elasticsearch-oss package not found, nothing to upgrade + quiet: true + +- name: OPS | Include defaults from OpenSearch role + include_vars: + file: roles/opensearch/defaults/main.yml + name: opensearch_defaults + +- name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml + include_vars: + file: roles/opensearch/vars/main.yml + name: opensearch_vars + +- name: OPS | Set the versions of OpenSearch + include_tasks: roles/opensearch/tasks/set_facts.yml + +- name: OPS | Ensure Opensearch service user exists + user: + name: "{{ opensearch_vars.specification.opensearch_user }}" + state: present + shell: /bin/bash + +- name: OPS | Ensure directory structure exists + file: + path: "{{ item }}" + state: directory + owner: "{{ opensearch_vars.specification.opensearch_user }}" + group: "{{ opensearch_vars.specification.opensearch_user }}" + with_items: + - "{{ opensearch_vars.specification.paths.opensearch_home }}" + - "{{ opensearch_vars.specification.paths.opensearch_log_dir }}" + - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" + - "{{ opensearch_defaults.certificates.dirs.certs }}" + +- name: OPS | Print ElasticSearch ond OpenSearch versions + debug: + msg: + - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" + - "Target version: {{ opensearch_ver.split('-')[0] }}" + +# If state file exists it means the previous run failed +- name: OPS | Check if upgrade state file exists + stat: + path: "{{ opensearch.upgrade_state_file_path }}" + get_attributes: false + get_checksum: false + get_mime: false + register: stat_upgrade_state_file + +- include_role: + name: upgrade + tasks_from: opensearch/migrate-odfe + when: opensearch_vars.specification.odfe_migration + vars: + current_group_name: logging + +- include_role: + name: upgrade + tasks_from: opensearch/migrate-kibana + when: opensearch_vars.specification.odfe_migration + vars: + current_group_name: logging diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index 3125792199..e9bcb9dd9a 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -138,65 +138,30 @@ # === logging === -# Some pre-upgrade tasks can be run in parallel (what saves time) while others must be run in serial (to support rolling upgrades). -# Such a separation in Ansible can be applied only at play level thus we have two plays below. - -# play 1/2: pre-upgrade parallel tasks -- hosts: logging - become: true - become_method: sudo - tasks: - - include_role: - name: upgrade - tasks_from: opensearch-01 - when: "'logging' in upgrade_components or upgrade_components|length == 0" - vars: - current_group_name: logging - -# play 2/2: serial tasks - hosts: logging become: true become_method: sudo - gather_facts: false # gathered by previous play - serial: 1 tasks: - include_role: name: upgrade - tasks_from: opensearch-02 + tasks_from: opensearch when: "'logging' in upgrade_components or upgrade_components|length == 0" vars: current_group_name: logging # === opensearch === -# Some pre-upgrade tasks can be run in parallel (what saves time) while others must be run in serial (to support rolling upgrades). -# Such a separation in Ansible can be applied only at play level thus we have two plays below. - -# play 1/2: parallel tasks - hosts: opensearch become: true become_method: sudo tasks: - include_role: name: upgrade - tasks_from: opensearch-01 + tasks_from: opensearc when: "'opensearch' in upgrade_components or upgrade_components|length == 0" vars: current_group_name: opensearch -# play 2/2: serial tasks -- hosts: opensearch - become: true - become_method: sudo - gather_facts: false # gathered by previous play - serial: 1 - tasks: - - include_role: - name: upgrade - tasks_from: opensearch-02 - when: "'opensearch' in upgrade_components or upgrade_components|length == 0" - vars: - current_group_name: opensearch - hosts: grafana become: true From ead07fc1a0a35a9486d5a3d523a91aeb0ab26adc Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 28 Feb 2022 17:23:16 +0100 Subject: [PATCH 135/157] simplifying the structure due to lack of not EPI provided versions of ODFE --- .../opensearch/migrate-from-demo-certs-01.yml | 71 -------- .../opensearch/migrate-from-demo-certs-02.yml | 115 ------------- .../migrate-from-demo-certs-non-clustered.yml | 77 --------- .../opensearch/upgrade-opensearch-01.yml | 157 ------------------ 4 files changed, 420 deletions(-) delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml delete mode 100644 ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml deleted file mode 100644 index 4ea4e8d24f..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-01.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -# ================================================================================================= -# Migration from demo certs to generated by Epiphany -# ------------------------------------------------------------------------------------------------- -# A) Parallel part (all nodes at the same time) - THIS FILE -# 1. Assert API access using demo cert (done in pre-migration part) -# 2. Generate Epiphany certs (done in pre-migration part) -# 3. Save cluster status to file (done in pre-migration part) -# 4. Create dual root CA file for the migration (demo + Epiphany root CAs concatenated), needed temporarily -# 5. Patch the following properties in existing elasticsearch.yml: -# a) opensearch_security.authcz.admin_dn - add Epiphany admin cert -# b) opensearch_security.nodes_dn - by default not present, add all Epiphany node certs -# c) opensearch_security.ssl.http.pemtrustedcas_filepath - replace demo root CA with the dual root CA file -# d) opensearch_security.ssl.transport.pemtrustedcas_filepath - replace demo root CA with the dual root CA file -# B) Serial part (node by node) - tasks from migrate-from-demo-certs-02.yml - -# Create dual root CA transitional file -- include_tasks: utils/create-dual-cert-file.yml - vars: - certs_to_concatenate: - - "{{ (certificates.dirs.certs, certificates.files.demo.root_ca.cert) | path_join }}" - - "{{ (certificates.dirs.certs, certificates.files.root_ca.cert.filename) | path_join }}" - target_path: "{{ (certificates.dirs.certs, opensearch.certs_migration.dual_root_ca.filename) | path_join }}" - -- name: OPS | Load /etc/elasticsearch/elasticsearch.yml - slurp: - src: /etc/elasticsearch/elasticsearch.yml - register: _elasticsearch_yml - -- name: OFDE | Patch /etc/elasticsearch/elasticsearch.yml (switch to dual root CA) - copy: - dest: /etc/elasticsearch/elasticsearch.yml - content: "{{ _patched_content | to_nice_yaml }}" - mode: u=rw,g=rw,o= - owner: root - group: opensearch - backup: true - vars: - _epiphany_subjects: - admin: "{{ certificates.files.admin.cert.subject }}" - node: "{{ certificates.files.node.cert.subject }}" - _epiphany_dn_attributes: - admin: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.admin.keys()) }}" - node: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.node.keys()) }}" - _epiphany_DNs: - admin: >- - {{ _epiphany_dn_attributes.admin | zip(_epiphany_dn_attributes.admin | map('extract', _epiphany_subjects.admin)) - | map('join','=') | join(',') }} - node: >- - {{ _epiphany_dn_attributes.node | zip(_epiphany_dn_attributes.node | map('extract', _epiphany_subjects.node)) - | map('join','=') | join(',') }} - _epiphany_nodes_dn: >- - {%- for node in ansible_play_hosts_all -%} - {%- if loop.first -%}[{%- endif -%} - '{{ _epiphany_DNs.node.split(',') | map('regex_replace', '^CN=.+$', 'CN=' + hostvars[node].ansible_nodename) | join(',') }}' - {%- if not loop.last -%},{%- else -%}]{%- endif -%} - {%- endfor -%} - _old_content: >- - {{ _elasticsearch_yml.content | b64decode | from_yaml }} - _updated_settings: - opensearch_security.authcz.admin_dn: >- - {{ _old_content['opensearch_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') - | union([opensearch.certs_migration.demo_DNs.admin] + [_epiphany_DNs.admin]) }} - opensearch_security.nodes_dn: >- - {{ _old_content['opensearch_security.nodes_dn'] | default([]) - | union([opensearch.certs_migration.demo_DNs.node] + _epiphany_nodes_dn) }} - - opensearch_security.ssl.http.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" - opensearch_security.ssl.transport.pemtrustedcas_filepath: "{{ opensearch.certs_migration.dual_root_ca.filename }}" - _patched_content: >- - {{ _old_content | combine(_updated_settings) }} diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml deleted file mode 100644 index b6cccd4a6e..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-02.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- -# ================================================================================================= -# Migration from demo certs to generated by Epiphany -# ------------------------------------------------------------------------------------------------- -# A) Parallel part (all nodes at the same time) - tasks from migrate-from-demo-certs-01.yml -# B) Serial part (node by node) - THIS FILE -# 1. Prepare cluster for a node restart (disable shard allocation) -# 2. Restart all nodes one by one waiting for yellow cluster status after each restart -# 3. Patch elasticsearch.yml to use Epiphany node cert instead of demo (all nodes) -# 4. Restart all nodes one by one waiting for yellow cluster status after each restart -# 5. Re-enable shard allocation -# 6. Wait for green/yellow cluster status -# 7. Test API access using Epiphany admin cert (all nodes) -# 8. Update API related facts to use Epiphany admin cert instead of demo -# 9. Reload config file - -- when: inventory_hostname == ansible_play_hosts_all[0] # run once - block: - # Prepare cluster for a node restart - - include_tasks: utils/prepare-cluster-for-node-restart.yml - - # Restart all nodes (special flow: run once but in loop for each host) - - include_tasks: - file: utils/restart-node.yml - apply: - delegate_to: "{{ target_inventory_hostname }}" - delegate_facts: true - loop: "{{ ansible_play_hosts_all }}" - loop_control: - loop_var: target_inventory_hostname - - # Patch elasticsearch.yml to use Epiphany node cert (all hosts) - - - name: OPS | Load /etc/elasticsearch/elasticsearch.yml - slurp: - src: /etc/elasticsearch/elasticsearch.yml - register: _elasticsearch_yml - delegate_to: "{{ target_inventory_hostname }}" - loop: "{{ ansible_play_hosts_all }}" - loop_control: - loop_var: target_inventory_hostname - - - name: OFDE | Patch /etc/elasticsearch/elasticsearch.yml (switch to Epiphany node certificates) - copy: - dest: /etc/elasticsearch/elasticsearch.yml - content: "{{ _patched_content | to_nice_yaml }}" - mode: u=rw,g=rw,o= - owner: root - group: opensearch - backup: true - delegate_to: "{{ target_inventory_hostname }}" - delegate_facts: true - loop: "{{ ansible_play_hosts_all }}" - loop_control: - index_var: loop_index0 - loop_var: target_inventory_hostname - vars: - _node_hostname: "{{ hostvars[target_inventory_hostname].ansible_nodename }}" - _epiphany_node_cert: - cert_filename: "{{ certificates.files.node.cert.filename | replace(ansible_nodename, _node_hostname) }}" - key_filename: "{{ certificates.files.node.key.filename | replace(ansible_nodename, _node_hostname) }}" - _old_content: >- - {{ _elasticsearch_yml.results[loop_index0].content | b64decode | from_yaml }} - _updated_settings: - opensearch_security.ssl.http.pemcert_filepath: "{{ _epiphany_node_cert.cert_filename }}" - opensearch_security.ssl.http.pemkey_filepath: "{{ _epiphany_node_cert.key_filename }}" - opensearch_security.ssl.transport.pemcert_filepath: "{{ _epiphany_node_cert.cert_filename }}" - opensearch_security.ssl.transport.pemkey_filepath: "{{ _epiphany_node_cert.key_filename }}" - _patched_content: >- - {{ _old_content | combine(_updated_settings) }} - - # Restart all nodes (special flow: run once but in loop for each host) - - include_tasks: - file: utils/restart-node.yml - apply: - delegate_to: "{{ target_inventory_hostname }}" - delegate_facts: true - loop: "{{ ansible_play_hosts_all }}" - loop_control: - loop_var: target_inventory_hostname - - # Re-enable shard allocation - - include_tasks: utils/enable-shard-allocation.yml - - # Wait for shard allocation (for 'green' status at least 2 nodes must be already upgraded) - - include_tasks: utils/wait-for-shard-allocation.yml - - # Test API access using Epiphany admin cert (all nodes) - - include_tasks: - file: utils/assert-api-access.yml - apply: - delegate_to: "{{ target_inventory_hostname }}" - delegate_facts: true - loop: "{{ ansible_play_hosts_all }}" - loop_control: - loop_var: target_inventory_hostname - vars: - es_api: - cert_type: Epiphany - cert_path: &epi_cert_path "{{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join }}" - key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join }}" - url: "{{ hostvars[target_inventory_hostname].es_api.url }}" - fail_msg: API access test failed. - -- name: Update API related facts to use Epiphany admin certificate instead of demo - set_fact: - es_api: "{{ es_api | combine(_es_api) }}" - vars: - _es_api: - cert_type: Epiphany - cert_path: *epi_cert_path - key_path: *epi_key_path - -# Reload config file to preserve patched settings (sets 'existing_config' fact) -- include_tasks: utils/get-config-from-files.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml deleted file mode 100644 index b3d376838d..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-from-demo-certs-non-clustered.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- -- name: OPS | Load /etc/elasticsearch/elasticsearch.yml - slurp: - src: /etc/elasticsearch/elasticsearch.yml - register: _elasticsearch_yml - -- name: OFDE | Patch /etc/elasticsearch/elasticsearch.yml (switch to generated certificates) - copy: - dest: /etc/elasticsearch/elasticsearch.yml - content: "{{ _patched_content | to_nice_yaml }}" - mode: u=rw,g=rw,o= - owner: root - group: opensearch - backup: true - vars: - _epiphany_subjects: - admin: "{{ certificates.files.admin.cert.subject }}" - node: "{{ certificates.files.node.cert.subject }}" - _epiphany_dn_attributes: - admin: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.admin.keys()) }}" - node: "{{ certificates.dn_attributes_order | intersect(_epiphany_subjects.node.keys()) }}" - _epiphany_DNs: - admin: >- - {{ _epiphany_dn_attributes.admin | zip(_epiphany_dn_attributes.admin | map('extract', _epiphany_subjects.admin)) - | map('join','=') | join(',') }} - node: >- - {{ _epiphany_dn_attributes.node | zip(_epiphany_dn_attributes.node | map('extract', _epiphany_subjects.node)) - | map('join','=') | join(',') }} - _old_content: >- - {{ _elasticsearch_yml.content | b64decode | from_yaml }} - _updated_settings: - opensearch_security.authcz.admin_dn: >- - {{ _old_content['opensearch_security.authcz.admin_dn'] | default([]) | map('replace', ', ', ',') - | union([_epiphany_DNs.admin]) }} - opensearch_security.nodes_dn: >- - {{ _old_content['opensearch_security.nodes_dn'] | default([]) - | union([_epiphany_DNs.node]) }} - - opensearch_security.ssl.http.pemcert_filepath: "{{ certificates.files.node.cert.filename }}" - opensearch_security.ssl.http.pemkey_filepath: "{{ certificates.files.node.key.filename }}" - opensearch_security.ssl.transport.pemcert_filepath: "{{ certificates.files.node.cert.filename }}" - opensearch_security.ssl.transport.pemkey_filepath: "{{ certificates.files.node.key.filename }}" - - opensearch_security.ssl.http.pemtrustedcas_filepath: "{{ certificates.files.root_ca.cert.filename }}" - opensearch_security.ssl.transport.pemtrustedcas_filepath: "{{ certificates.files.root_ca.cert.filename }}" - - _patched_content: >- - {{ _old_content | combine(_updated_settings) }} - -- include_tasks: - file: utils/restart-node.yml - vars: - target_inventory_hostname: "{{ inventory_hostname }}" - skip_waiting_for_node: true # because after restart demo certificate stops working - -# Test API access using Epiphany admin cert -- include_tasks: - file: utils/assert-api-access.yml - vars: - es_api: - cert_type: Epiphany - cert_path: &epi_cert_path "{{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join }}" - key_path: &epi_key_path "{{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join }}" - url: "{{ hostvars[inventory_hostname].es_api.url }}" - fail_msg: API access test failed. - -- name: Update API related facts to use Epiphany admin certificate instead of demo - set_fact: - es_api: "{{ es_api | combine(_es_api) }}" - vars: - _es_api: - cert_type: Epiphany - cert_path: *epi_cert_path - key_path: *epi_key_path - -# Reload config file to preserve patched settings (sets 'existing_config' fact) -- include_tasks: utils/get-config-from-files.yml diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml deleted file mode 100644 index e2166f0b59..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/upgrade-opensearch-01.yml +++ /dev/null @@ -1,157 +0,0 @@ ---- -# This file contains only pre-upgrade tasks that can be run in parallel on all hosts - -- name: OPS | Create upgrade state file - become: true - file: - path: "{{ opensearch.upgrade_state_file_path }}" - state: touch - mode: u=rw,g=r,o= - -- name: OPS | Ensure elasticsearch service is running - systemd: - name: elasticsearch - enabled: yes - state: started - register: elasticsearch_state - -# Sets 'existing_config' fact -- include_tasks: utils/get-config-from-files.yml - -- name: OPS | Set common facts - set_fact: - certificates: "{{ opensearch_defaults.certificates }}" - es_host: "{{ existing_config.main['network.host'] | default('_local_') }}" - es_http_port: "{{ existing_config.main['http.port'] | default(opensearch_defaults.ports.http) }}" - es_transport_port: "{{ existing_config.main['transport.port'] | default(opensearch_defaults.ports.transport) }}" - es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" - es_node_name: "{{ existing_config.main['node.name'] }}" - -- name: OPS | Wait for elasticsearch service to start up - wait_for: - port: "{{ es_transport_port }}" - host: "{{ es_host if (es_host is not regex('^_.+_$')) else '0.0.0.0' }}" # 0.0.0.0 means any IP - when: elasticsearch_state.changed - -# This block requires elasticsearch service to be running -- name: Get host address when special value is used # e.g. '_site_' - when: es_host is regex('^_.+_$') - block: - - name: Gather facts on listening ports - community.general.listen_ports_facts: - - - name: Get host address based on transport port - set_fact: - es_host: "{{ ansible_facts.tcp_listen | selectattr('port', '==', es_transport_port|int) - | map(attribute='address') | reject('match', '::') | first }}" - -# NOTE: We need admin certificate for passwordless administrative access to REST API (since we don't know admin's password) - -- include_role: - name: certificate - tasks_from: install-packages # requirements for Ansible certificate modules - -- name: OPS | Get information on root CA certificate - community.crypto.x509_certificate_info: - # 'pemtrustedcas_filepath' is a relative path - path: "{{ ('/etc/elasticsearch', existing_config.main['opendistro_security.ssl.transport.pemtrustedcas_filepath']) | path_join }}" - register: _root_ca_info - -- name: OPS | Check if demo or Epiphany certificates are in use # self-signed - set_fact: - _is_demo_cert_in_use: "{{ 'True' if _root_ca_info.subject.commonName == 'Example Com Inc. Root CA' else 'False' }}" - _is_epiphany_cert_in_use: "{{ 'True' if _root_ca_info.subject.commonName == 'Epiphany Managed ODFE Root CA' else 'False' }}" - -# For custom admin cert (non-demo and non-Epiphany), we use workaround (upgrade_config.custom_admin_certificate). -# The workaround should be replaced after implementing task #2127. -- name: OPS | Set API access facts - set_fact: - es_api: - cert_path: "{{ _cert_path[_cert_type] }}" - cert_type: "{{ _cert_type }}" - key_path: "{{ _key_path[_cert_type] }}" - url: https://{{ es_host }}:{{ es_http_port }} - vars: - _cert_type: >- - {{ 'demo' if (_is_demo_cert_in_use) else - 'Epiphany' if (_is_epiphany_cert_in_use) else - 'custom' }} - _cert_path: - custom: "{{ lookup('vars', current_group_name).upgrade_config.custom_admin_certificate.cert_path }}" # defaults are not available via hostvars - demo: "{{ (certificates.dirs.certs, certificates.files.demo.admin.cert) | path_join }}" - Epiphany: "{{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join }}" - _key_path: - custom: "{{ lookup('vars', current_group_name).upgrade_config.custom_admin_certificate.key_path }}" - demo: "{{ (certificates.dirs.certs, certificates.files.demo.admin.key) | path_join }}" - Epiphany: "{{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join }}" - -- include_tasks: utils/assert-cert-files-exist.yml - -# ================================================================================================= -# FLOW -# ------------------------------------------------------------------------------------------------- -# NOTE: For clustered nodes it's recommended to disable shard allocation for the cluster before restarting a node (https://www.elastic.co/guide/en/elasticsearch/reference/current/restart-cluster.html#restart-cluster-rolling) -# -# if cert_type == 'demo': -# Test API access -# Genereate Epiphany self-signed certs -# Save cluster status to file -# Run certificates migration procedure for all nodes when 'es_clustered is true' -# // Subtasks of the migration procedure: -# Test API access -# Update API related facts to use Epiphany admin certificate instead of demo -# if cert_type == 'Epiphany': -# Genereate Epiphany self-signed certs - to re-new certs if expiration date differs -# Test API access -# Save cluster status to file -# if cert_type == 'custom': -# Test API access -# Save cluster status to file -# Run upgrade (removes known demo certificate files) -# if cert_type == 'Epiphany': -# Remove dual root CA file (created as part of the migration, needed until all nodes are upgraded) -# ================================================================================================= - -# Test API access (demo or custom certs) -- include_tasks: utils/assert-api-access.yml - when: es_api.cert_type in ['demo', 'custom'] - vars: - _fail_msg: - common: Test of accessing API with TLS authentication failed. - custom: >- - It looks like you use custom certificates. - Please refer to 'Open Distro for Elasticsearch upgrade' section of How-To docs. - demo: >- - It looks like you use demo certificates but your configuration might be incorrect or unsupported. - fail_msg: "{{ _fail_msg.common }} {{ _fail_msg[es_api.cert_type] }}" - -- name: Generate self-signed certificates - include_role: - name: opensearch - tasks_from: generate-certs - when: es_api.cert_type != 'custom' - -# Test API access (Epiphany certs) -- include_tasks: utils/assert-api-access.yml - when: es_api.cert_type == 'Epiphany' - vars: - fail_msg: >- - Test of accessing API with TLS authentication failed. - It looks like you use certificates generated by Epiphany but your configuration might be incorrect or an unexpected error occurred. - -# Save cluster health status before upgrade to file -- include_tasks: utils/save-initial-cluster-status.yml - -# Run migration procedure - the first (parallel) part for clustered installation -- include_tasks: migrate-from-demo-certs-01.yml - when: - - es_api.cert_type == 'demo' - - es_clustered # rolling upgrade only for clustered installation - -# Run migration procedure for non-clustered installation -- include_tasks: migrate-from-demo-certs-non-clustered.yml - when: - - es_api.cert_type == 'demo' - - not es_clustered - -# Next tasks are run in serial mode in the next play From 79b3c87962af3de4ff99058ead083056daa6a14d Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 1 Mar 2022 18:54:25 +0100 Subject: [PATCH 136/157] Renaming change as requested during PR rev. --- .../roles/upgrade/tasks/opensearch.yml | 18 +++++++++--------- .../upgrade/tasks/opensearch/migrate-odfe.yml | 6 +++--- .../opensearch/utils/assert-api-access.yml | 4 ++-- .../utils/assert-cert-files-exist.yml | 6 +++--- .../opensearch/utils/create-dual-cert-file.yml | 4 ++-- .../utils/enable-shard-allocation.yml | 2 +- .../opensearch/utils/get-cluster-health.yml | 2 +- .../opensearch/utils/get-config-from-files.yml | 6 +++--- .../utils/prepare-cluster-for-node-restart.yml | 6 +++--- .../tasks/opensearch/utils/restart-node.yml | 6 +++--- .../utils/save-initial-cluster-status.yml | 4 ++-- .../tasks/opensearch/utils/test-api-access.yml | 2 +- .../utils/wait-for-cluster-status.yml | 2 +- .../opensearch/utils/wait-for-node-to-join.yml | 2 +- .../utils/wait-for-shard-allocation.yml | 2 +- 15 files changed, 36 insertions(+), 36 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml index be48b7d8a1..64b206da94 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml @@ -1,35 +1,35 @@ --- -- name: OPS | Get information about installed packages as facts +- name: OpenSearch | Get information about installed packages as facts package_facts: manager: auto when: ansible_facts.packages is undefined -- name: OPS | Assert that elasticsearch-oss package is installed +- name: OpenSearch | Assert that elasticsearch-oss package is installed assert: that: ansible_facts.packages['elasticsearch-oss'] is defined fail_msg: elasticsearch-oss package not found, nothing to upgrade quiet: true -- name: OPS | Include defaults from OpenSearch role +- name: OpenSearch | Include defaults from OpenSearch role include_vars: file: roles/opensearch/defaults/main.yml name: opensearch_defaults -- name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml +- name: OpenSearch | Include vars from opensearch role # requires epicli upgrade -f .yml include_vars: file: roles/opensearch/vars/main.yml name: opensearch_vars -- name: OPS | Set the versions of OpenSearch +- name: OpenSearch | Set the versions of OpenSearch include_tasks: roles/opensearch/tasks/set_facts.yml -- name: OPS | Ensure Opensearch service user exists +- name: OpenSearch | Ensure Opensearch service user exists user: name: "{{ opensearch_vars.specification.opensearch_user }}" state: present shell: /bin/bash -- name: OPS | Ensure directory structure exists +- name: OpenSearch | Ensure directory structure exists file: path: "{{ item }}" state: directory @@ -41,14 +41,14 @@ - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}" - "{{ opensearch_defaults.certificates.dirs.certs }}" -- name: OPS | Print ElasticSearch ond OpenSearch versions +- name: OpenSearch | Print ElasticSearch ond OpenSearch versions debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - "Target version: {{ opensearch_ver.split('-')[0] }}" # If state file exists it means the previous run failed -- name: OPS | Check if upgrade state file exists +- name: OpenSearch | Check if upgrade state file exists stat: path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index edc1c7a2ae..0ecdd21e70 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -1,10 +1,10 @@ --- -- name: OPS | Get information about installed packages as facts +- name: OpenSearch | Get information about installed packages as facts package_facts: manager: auto when: ansible_facts.packages is undefined -- name: OPS | Print ElasticSearch ond OpenSearch versions +- name: OpenSearch | Print ElasticSearch ond OpenSearch versions debug: msg: - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" @@ -12,7 +12,7 @@ - name: ODFE migr | Ensure elasticsearch cluster is up and running block: - - name: OPS | Include vars from opensearch role # requires epicli upgrade -f .yml + - name: OpenSearch | Include vars from opensearch role # requires epicli upgrade -f .yml include_vars: file: roles/opensearch/vars/main.yml name: opensearch_vars diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml index 9ae45b7ae8..c99c75ad72 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-api-access.yml @@ -1,5 +1,5 @@ --- -- name: OPS | Assert input parameters +- name: OpenSearch | Assert input parameters assert: that: - es_api.cert_path is defined @@ -13,7 +13,7 @@ # Sets 'test_api_access' - include_tasks: test-api-access.yml -- name: OPS | Assert API access +- name: OpenSearch | Assert API access assert: that: test_api_access.status == 200 fail_msg: diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml index b8dd104935..8166ad52af 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/assert-cert-files-exist.yml @@ -1,5 +1,5 @@ --- -- name: OPS | Assert input parameters +- name: OpenSearch | Assert input parameters assert: that: - es_api.cert_path is defined @@ -8,7 +8,7 @@ - es_api.key_path is defined quiet: true -- name: OPS | Get info on files +- name: OpenSearch | Get info on files stat: path: "{{ item }}" get_attributes: false @@ -20,7 +20,7 @@ - "{{ es_api.key_path }}" # Specific case for custom certificates (we don't know the paths so they have to be specified manually) -- name: OPS | Assert files exist +- name: OpenSearch | Assert files exist assert: that: stat_result.stat.exists fail_msg: "{{ _custom_cert_fail_msg if (es_api.cert_type == 'custom') else _common_fail_msg }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml index 40877c305c..316078d694 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/create-dual-cert-file.yml @@ -3,13 +3,13 @@ # - certs_to_concatenate # - target_path -- name: OPS | Read certificates to concatenate +- name: OpenSearch | Read certificates to concatenate slurp: src: "{{ item }}" register: _files loop: "{{ certs_to_concatenate }}" -- name: OPS | Create dual root CA transitional file for migration +- name: OpenSearch | Create dual root CA transitional file for migration copy: dest: "{{ target_path }}" content: "{{ _files.results | map(attribute='content') | map('b64decode') | join('') }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml index 52ec5744c2..4978f10a5a 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/enable-shard-allocation.yml @@ -4,7 +4,7 @@ # - es_api.cert_path # - es_api.key_path -- name: OPS | Enable shard allocation for the cluster +- name: OpenSearch | Enable shard allocation for the cluster uri: url: "{{ es_api.url }}/_cluster/settings" method: PUT diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml index 12ad125baf..fae3164ded 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-cluster-health.yml @@ -4,7 +4,7 @@ # - es_api.cert_path # - es_api.key_path -- name: OPS | Get cluster health +- name: OpenSearch | Get cluster health uri: url: "{{ es_api.url }}/_cluster/health" method: GET diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml index 0aae6a6f97..8678908038 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/get-config-from-files.yml @@ -1,17 +1,17 @@ --- # Sets facts on existing configuration -- name: OPS | Load /etc/elasticsearch/elasticsearch.yml +- name: OpenSearch | Load /etc/elasticsearch/elasticsearch.yml slurp: src: /etc/elasticsearch/elasticsearch.yml register: _elasticsearch_yml -- name: OPS | Get Xmx value from /etc/elasticsearch/jvm.options +- name: OpenSearch | Get Xmx value from /etc/elasticsearch/jvm.options command: grep -oP '(?<=^-Xmx)\d+[kKmMgG]?' /etc/elasticsearch/jvm.options register: _grep_xmx changed_when: false -- name: OPS | Set existing configuration facts +- name: OpenSearch | Set existing configuration facts set_fact: existing_config: main: "{{ _elasticsearch_yml.content | b64decode | from_yaml }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml index d28bf90192..9a0c6ff977 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/prepare-cluster-for-node-restart.yml @@ -16,7 +16,7 @@ body_format: json block: # It's safe to run this task many times regardless of the state - - name: OPS | Disable shard allocation for the cluster + - name: OpenSearch | Disable shard allocation for the cluster uri: url: "{{ es_api.url }}/_cluster/settings" method: PUT @@ -35,7 +35,7 @@ # In epicli 0.7.x there is ES 7.3.2 but this step is optional. - name: Handle flush failure block: - - name: OPS | Perform a synced flush (optional step) + - name: OpenSearch | Perform a synced flush (optional step) uri: url: "{{ es_api.url }}/_flush" method: POST @@ -46,7 +46,7 @@ retries: 120 delay: 1 rescue: - - name: OPS | Print warning + - name: OpenSearch | Print warning debug: msg: - "WARNING: flush command failed" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml index dd88fe87bb..ee5c496756 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/restart-node.yml @@ -10,18 +10,18 @@ # - daemon_reload # - skip_waiting_for_status -- name: OPS | Restart elasticsearch service +- name: OpenSearch | Restart elasticsearch service systemd: name: opensearch state: restarted daemon_reload: "{{ daemon_reload | default(omit) }}" -- name: OPS | Wait for Elasticsearch transport port to become available +- name: OpenSearch | Wait for Elasticsearch transport port to become available wait_for: port: "{{ es_transport_port }}" host: "{{ hostvars[target_inventory_hostname].es_host }}" -- name: OPS | Wait for Elasticsearch http port to become available +- name: OpenSearch | Wait for Elasticsearch http port to become available wait_for: port: "{{ es_http_port }}" host: "{{ hostvars[target_inventory_hostname].es_host }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml index 87d01c58a6..cd6253396c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/save-initial-cluster-status.yml @@ -1,5 +1,5 @@ --- -- name: OPS | Get size of upgrade state file +- name: OpenSearch | Get size of upgrade state file stat: path: "{{ opensearch.upgrade_state_file_path }}" get_attributes: false @@ -12,7 +12,7 @@ block: - include_tasks: get-cluster-health.yml - - name: OPS | Save cluster health to upgrade state file + - name: OpenSearch | Save cluster health to upgrade state file copy: content: "{{ cluster_health.json }}" dest: "{{ opensearch.upgrade_state_file_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml index 0a620baa1e..cb8e49d961 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/test-api-access.yml @@ -5,7 +5,7 @@ # - es_api.key_path # - es_api.url -- name: OPS | Test API access using {{ es_api.cert_type }} certificate +- name: OpenSearch | Test API access using {{ es_api.cert_type }} certificate uri: client_cert: "{{ es_api.cert_path }}" client_key: "{{ es_api.key_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml index ef31a44613..78615ea41c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-cluster-status.yml @@ -5,7 +5,7 @@ # - es_api.key_path # - expected_status (type: list, e.g. [ 'green', 'yellow' ]) -- name: OPS | Wait for '{{ expected_status | join("' or '") }}' cluster health status +- name: OpenSearch | Wait for '{{ expected_status | join("' or '") }}' cluster health status uri: url: "{{ es_api.url }}/_cluster/health" client_cert: "{{ es_api.cert_path }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml index 1c3f1ae4f6..82bf3ef35c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-node-to-join.yml @@ -6,7 +6,7 @@ # - target_inventory_hostname # - hostvars[target_inventory_hostname].es_node_name -- name: OPS | Wait for Elasticsearch node to join the cluster +- name: OpenSearch | Wait for Elasticsearch node to join the cluster uri: url: "{{ es_api.url }}/_cat/nodes?h=name" method: GET diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml index bb294c0b1d..2517d57286 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/utils/wait-for-shard-allocation.yml @@ -4,7 +4,7 @@ # - es_api.cert_path # - es_api.key_path -- name: OPS | Wait for the cluster to finish shard allocation +- name: OpenSearch | Wait for the cluster to finish shard allocation uri: url: "{{ es_api.url }}/_cluster/health" method: GET From 17195038512b0cdfb9a045d8aa3f89ea76277ac4 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 1 Mar 2022 19:53:03 +0100 Subject: [PATCH 137/157] Higher failure level allowed --- .../roles/backup/tasks/logging_opensearch_snapshot.yml | 4 ++-- .../roles/recovery/tasks/logging_opensearch_snapshot.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml index 9665caacf5..f15c850f24 100644 --- a/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml @@ -25,8 +25,8 @@ return_content: yes register: cluster_status until: cluster_status.json.status - retries: 12 - delay: 5 + retries: 60 + delay: 1 - name: No backup warning when: not cluster_status.json.number_of_nodes == 1 diff --git a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml index dde65f50b9..af1ba56789 100644 --- a/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml +++ b/ansible/playbooks/roles/recovery/tasks/logging_opensearch_snapshot.yml @@ -105,7 +105,7 @@ method: POST always: - - name: Start allOpenSearch Dashboards instances + - name: Start all OpenSearch Dashboards instances delegate_to: "{{ item }}" systemd: name: opensearch-dashboards From 52560b81987cc0c1e2795695c9f33b71d6fe53bf Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Mar 2022 21:17:40 +0100 Subject: [PATCH 138/157] Fixing a typo --- ansible/playbooks/upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index e9bcb9dd9a..8b14dd745d 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -157,7 +157,7 @@ tasks: - include_role: name: upgrade - tasks_from: opensearc + tasks_from: opensearch when: "'opensearch' in upgrade_components or upgrade_components|length == 0" vars: current_group_name: opensearch From 9b2eb1f7ed6b5086b15ff49eeeeb0ed1370551c7 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Mar 2022 21:18:03 +0100 Subject: [PATCH 139/157] Correct var assigned --- .../playbooks/roles/opensearch/templates/opensearch.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 index 18c4268499..b8e627d5cb 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.yml.j2 @@ -106,12 +106,12 @@ cluster.initial_master_nodes: ["{{ ansible_hostname }}"] ######## OpenSearch Security Configuration ######## # WARNING: revise all the lines below before you go into production plugins.security.ssl.transport.pemcert_filepath: "{{ certificates.dirs.certs }}/{{ node_cert_filename.transport }}" -plugins.security.ssl.transport.pemkey_filepath: "{{ certificates.dirs.certs }}/{{ node_key_filename.transport }}" +plugins.security.ssl.transport.pemkey_filepath: "{{ certificates.dirs.ca_key }}/{{ node_key_filename.transport }}" plugins.security.ssl.transport.pemtrustedcas_filepath: "{{ certificates.dirs.certs }}/{{ root_ca_cert_filename.transport }}" plugins.security.ssl.transport.enforce_hostname_verification: {{ specification.opensearch_security.ssl.transport.enforce_hostname_verification | lower }} plugins.security.ssl.http.enabled: true plugins.security.ssl.http.pemcert_filepath: "{{ certificates.dirs.certs }}/{{ node_cert_filename.http }}" -plugins.security.ssl.http.pemkey_filepath: "{{ certificates.dirs.certs }}/{{ node_key_filename.http }}" +plugins.security.ssl.http.pemkey_filepath: "{{ certificates.dirs.ca_key }}/{{ node_key_filename.http }}" plugins.security.ssl.http.pemtrustedcas_filepath: "{{ certificates.dirs.certs }}/{{ root_ca_cert_filename.http }}" plugins.security.allow_unsafe_democertificates: {{ opensearch_security_allow_unsafe_democertificates | lower }} plugins.security.allow_default_init_securityindex: true From 206cd2db224a52fe0ca737b0ccea6c65a0b93d01 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 3 Mar 2022 21:19:43 +0100 Subject: [PATCH 140/157] Ensure ccorrect folder rights --- ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml | 2 ++ ansible/playbooks/roles/upgrade/tasks/opensearch.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index 5b62f6761e..ff83781037 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -29,6 +29,8 @@ state: directory owner: "{{ specification.opensearch_user }}" group: "{{ specification.opensearch_user }}" + mode: '0740' + recurse: yes loop: - "{{ specification.paths.opensearch_home }}" - "{{ specification.paths.opensearch_perftop_home }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml index 64b206da94..b8f86bd1a0 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml @@ -35,6 +35,8 @@ state: directory owner: "{{ opensearch_vars.specification.opensearch_user }}" group: "{{ opensearch_vars.specification.opensearch_user }}" + mode: '0740' + recurse: yes with_items: - "{{ opensearch_vars.specification.paths.opensearch_home }}" - "{{ opensearch_vars.specification.paths.opensearch_log_dir }}" From 1b2fa24dd0c260a3fc9b7723755ef2e72f40c49f Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 4 Mar 2022 16:25:14 +0100 Subject: [PATCH 141/157] Changing the way we describe the perms --- ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml | 2 +- ansible/playbooks/roles/upgrade/tasks/opensearch.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index ff83781037..714eaca502 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -29,7 +29,7 @@ state: directory owner: "{{ specification.opensearch_user }}" group: "{{ specification.opensearch_user }}" - mode: '0740' + mode: u=rwx,go=r recurse: yes loop: - "{{ specification.paths.opensearch_home }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml index b8f86bd1a0..31b69facfe 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml @@ -35,7 +35,7 @@ state: directory owner: "{{ opensearch_vars.specification.opensearch_user }}" group: "{{ opensearch_vars.specification.opensearch_user }}" - mode: '0740' + mode: u=rw,go=r recurse: yes with_items: - "{{ opensearch_vars.specification.paths.opensearch_home }}" From c09e5ae93820573c7467997f5eed2b3953212d33 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 4 Mar 2022 16:26:57 +0100 Subject: [PATCH 142/157] Replacing the given uname with var --- .../roles/opensearch/tasks/generate-certs.yml | 16 ++++++++-------- .../tasks/opensearch/migrate-odfe-serial.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml index 0c0dca8d55..194cd58865 100644 --- a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml +++ b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml @@ -5,7 +5,7 @@ file: state: directory path: "{{ certificates.dirs.ca_key }}" - owner: opensearch + owner: "{{ specification.opensearch_user }}" group: opensearch mode: u=rwx,g=rwx,o= @@ -14,7 +14,7 @@ file: state: directory path: "{{ certificates.dirs.csr }}" - owner: opensearch + owner: "{{ specification.opensearch_user }}" group: opensearch mode: u=rwx,g=rwx,o= # CSR file doesn't contain private key @@ -25,17 +25,17 @@ size: 2048 # based on ODFE docs type: RSA mode: u=rw,go= - owner: opensearch + owner: "{{ specification.opensearch_user }}" format: pkcs8 community.crypto.openssl_csr: mode: u=rw,g=r,o= - owner: opensearch + owner: "{{ specification.opensearch_user }}" use_common_name_for_san: false community.crypto.x509_certificate: selfsigned_digest: sha256 ownca_digest: sha256 mode: u=rw,g=r,o= - owner: opensearch + owner: "{{ specification.opensearch_user }}" block: # --- Generate CA root certificate --- @@ -169,7 +169,7 @@ size: 2048 type: RSA mode: u=rw,g=r,o= - owner: opensearch + owner: "{{ specification.opensearch_user }}" group: opensearch return_content: false register: node_key @@ -196,7 +196,7 @@ subjectAltName: "{{ _dns_list + [ 'IP:' + ansible_default_ipv4.address ] }}" use_common_name_for_san: false mode: u=rw,g=r,o= - owner: opensearch + owner: "{{ specification.opensearch_user }}" group: opensearch register: node_csr vars: @@ -214,5 +214,5 @@ ownca_not_after: "{{ certificates.files.node.cert.ownca_not_after }}" ownca_digest: sha256 mode: u=rw,go=r - owner: opensearch + owner: "{{ specification.opensearch_user }}" group: opensearch diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index 3722169728..c8f44d584e 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -23,7 +23,7 @@ src: "{{ item.1 }}" dest: "{{ item.2 }}" remote_src: yes - owner: opensearch + owner: "{{ opensearch_vars.specification.opensearch_user }}" group: root mode: ug=rwx,o= directory_mode: yes From 21f1aef19466bd223c686d3a1c8a39956a1cb63c Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 4 Mar 2022 16:27:35 +0100 Subject: [PATCH 143/157] Docs corrections --- docs/architecture/logical-view.md | 4 ++-- docs/changelogs/CHANGELOG-2.0.md | 1 - docs/home/howto/MAINTENANCE.md | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/architecture/logical-view.md b/docs/architecture/logical-view.md index 4ec650dd18..ab3a65c922 100644 --- a/docs/architecture/logical-view.md +++ b/docs/architecture/logical-view.md @@ -68,9 +68,9 @@ Containers | Kubernetes components that run in a container ### OpenSearch Dashboards -`OpenSearch Dashboards` like `Grafana` is used in Epiphany for visualization,It uses `OpenSearch` as datasource for logs, it allows to create full text queries, dashboards and analytics that are performed on logs. +`OpenSearch Dashboards` like `Grafana` is used in Epiphany for visualization. It uses `OpenSearch` as datasource for logs, it allows to create full text queries, dashboards and analytics that are performed on logs. -[Read more](https://www.elastic.co/products/kibana) +[Read more](https://opensearch.org/docs/latest/dashboards/index/) ## Computing diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index de560de4d6..ffe2fc8e5d 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -4,7 +4,6 @@ ### Added - [#2870](https://github.com/epiphany-platform/epiphany/issues/2870) - OpenDistro for ElasticSearch project migrated to OpenSearch - - [#959](https://github.com/epiphany-platform/epiphany/issues/959) - Add usage of use_network_security_groups to disable NSG on AWS - [#2701](https://github.com/epiphany-platform/epiphany/issues/2701) - Epicli prepare - generate files in separate directory - [#2812](https://github.com/epiphany-platform/epiphany/issues/2812) - Extend K8s config validation diff --git a/docs/home/howto/MAINTENANCE.md b/docs/home/howto/MAINTENANCE.md index fe5ba0e7f9..f42ead43c6 100644 --- a/docs/home/howto/MAINTENANCE.md +++ b/docs/home/howto/MAINTENANCE.md @@ -123,7 +123,7 @@ status prometheus-node-exporter #### - OpenSearch -To check status of OpenSearch we can use the command: +To check the status of OpenSearch we can use the command: ```shell systemct status opensearch @@ -141,7 +141,7 @@ Check if service is listening on 9300 (nodes communication port): netstat -antup | grep 9300 ``` -We can also check status of OpenSearch cluster: +We can also check the status of OpenSearch cluster: ```shell :9200/_cluster/health From f1145a344bef01d1f7c3dc8d02dfde4813a4687b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Fri, 4 Mar 2022 16:53:04 +0100 Subject: [PATCH 144/157] Making this var more laconic --- .../opensearch_dashboards/tasks/dashboards.yml | 10 +++++----- .../roles/opensearch_dashboards/tasks/main.yml | 2 +- .../opensearch_dashboards/tasks/set_facts.yml | 4 ++-- .../templates/opensearch-dashboards.service.j2 | 4 ++-- .../upgrade/tasks/opensearch/migrate-kibana.yml | 16 ++++++++-------- .../configuration/opensearch-dashboards.yml | 8 ++++---- .../common/defaults/configuration/opensearch.yml | 6 +++--- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index dcd6898b4b..fa43837ad2 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -7,7 +7,7 @@ name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ opsd_version }}" + file_name: "opensearch-dashboards-{{ opensearchdash_version }}" - name: Create OpenSearch Dashboards user user: @@ -17,15 +17,15 @@ - name: Create home directory file: - path: "{{ specification.paths.opsd_home }}" + path: "{{ specification.paths.opensearchdash_home }}" state: directory owner: "{{ specification.dashboards_user }}" group: "{{ specification.dashboards_user }}" - name: Extract OpenSearch Dashboards tar file unarchive: - src: "/tmp/opensearch-dashboards-{{ opsd_version }}" - dest: "{{ specification.paths.opsd_home }}" + src: "/tmp/opensearch-dashboards-{{ opensearchdash_version }}" + dest: "{{ specification.paths.opensearchdash_home }}" owner: "{{ specification.dashboards_user }}" remote_src: yes extra_opts: @@ -34,7 +34,7 @@ - name: Copy configuration file template: src: opensearch_dashboards.yml.j2 - dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" + dest: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" owner: "{{ specification.dashboards_user }}" group: "{{ specification.dashboards_user }}" mode: 0644 diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml index 352c417062..6c1d137b13 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/main.yml @@ -13,7 +13,7 @@ enabled: yes - name: Get all the installed dashboards plugins - command: "{{ specification.paths.opsd_plugin_bin_path }} list" + command: "{{ specification.paths.opensearchdash_plugin_bin_path }} list" become: false # This command can not be run as root user register: list_plugins diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml index b16bbbff6c..50cf565ef0 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml @@ -1,10 +1,10 @@ --- - name: Set the name of binary to download for x86_64 family set_fact: - opsd_version: "1.2.0-linux-x64.tar.gz" + opensearchdash_version: "1.2.0-linux-x64.tar.gz" when: ansible_architecture == "x86_64" - name: Set the name of binary to download for arm64 family set_fact: - opsd_version: "1.2.0-linux-arm64.tar.gz" + opensearchdash_version: "1.2.0-linux-arm64.tar.gz" when: ansible_architecture == "aarch64" diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 index 10b84f2a61..41085a243d 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 @@ -7,12 +7,12 @@ After=network-online.target RuntimeDirectory=opensearch-dashboards PrivateTmp=true -WorkingDirectory={{ specification.paths.opsd_home }} +WorkingDirectory={{ specification.paths.opensearchdash_home }} User={{ specification.dashboards_user }} Group={{ specification.dashboards_user }} -ExecStart={{ specification.paths.opsd_home }}/bin/opensearch-dashboards -q +ExecStart={{ specification.paths.opensearchdash_home }}/bin/opensearch-dashboards -q StandardOutput=journal StandardError=inherit diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index e69eb068bc..67261cc289 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -21,7 +21,7 @@ name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ opsd_version }}" + file_name: "opensearch-dashboards-{{ opensearchdash_version }}" - name: Kibana migr | Create opensearch-dashboards user user: @@ -38,13 +38,13 @@ group: "{{ specification.dashboards_user }}" mode: ug=rwx,o=rx with_items: - - "{{ specification.paths.opsd_log_dir }}" - - "{{ specification.paths.opsd_home }}" + - "{{ specification.paths.opensearchdash_log_dir }}" + - "{{ specification.paths.opensearchdash_home }}" - name: Kibana migr | Extract the tar file unarchive: - src: "/tmp/opensearch-dashboards-{{ opsd_version }}" - dest: "{{ specification.paths.opsd_home }}" + src: "/tmp/opensearch-dashboards-{{ opensearchdash_version }}" + dest: "{{ specification.paths.opensearchdash_home }}" owner: "{{ specification.dashboards_user }}" remote_src: yes extra_opts: @@ -53,7 +53,7 @@ - name: Kibana migr | Clone kibana settings copy: src: /etc/kibana/kibana.yml - dest: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" + dest: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" remote_src: yes owner: "{{ specification.dashboards_user }}" group: root @@ -62,7 +62,7 @@ - name: Kibana migr | Porting kibana settings to OpenSearch Dashboards replace: - path: "{{ specification.paths.opsd_conf_dir }}/opensearch_dashboards.yml" + path: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" regexp: "{{ item.1 }}" replace: "{{ item.2 }}" with_items: @@ -86,7 +86,7 @@ enabled: yes - name: Kibana migr | Get all the installed dashboards plugins - command: "{{ specification.paths.opsd_plugin_bin_path }} list" + command: "{{ specification.paths.opensearchdash_plugin_bin_path }} list" become: false # This command can not be run as root user register: list_plugins diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml index 968c9b2705..71c46c15e3 100644 --- a/schema/common/defaults/configuration/opensearch-dashboards.yml +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -5,7 +5,7 @@ specification: dashboards_user: opensearchdboard dashboards_password: PASSWORD_TO_CHANGE paths: - opsd_home: /usr/share/opensearch-dashboards - opsd_conf_dir: /usr/share/opensearch-dashboards/config - opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin - opsd_log_dir: /var/log/opensearchdashboards + opensearchdash_home: /usr/share/opensearch-dashboards + opensearchdash_conf_dir: /usr/share/opensearch-dashboards/config + opensearchdash_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin + opensearchdash_log_dir: /var/log/opensearchdashboards diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index b3ae0ed208..e324e6ff7b 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -25,9 +25,9 @@ specification: opensearch_data: /var/lib/opensearch opensearch_logs: /var/log/opensearch opensearch_perftop_home: /usr/share/opensearch/perftop - opsd_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin - opsd_home: /usr/share/opensearch-dashboards - opsd_conf_dir: /usr/share/opensearch-dashboards/config + opensearchdash_plugin_bin_path: /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin + opensearchdash_home: /usr/share/opensearch-dashboards + opensearchdash_conf_dir: /usr/share/opensearch-dashboards/config jvm_options: Xmx: 1g opensearch_security: From f95cd7082efad8855ec3d5adf75ddb5dee17a714 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 14:34:37 +0100 Subject: [PATCH 145/157] Type correction --- docs/home/howto/DATABASES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 09c864f957..cc6177a3a0 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -455,9 +455,9 @@ Properly configured application (kubernetes service) to use fully HA configurati PostgreSQL native replication is now deprecated and removed. Use [PostgreSQL HA replication with repmgr](#how-to-set-up-postgresql-ha-replication-with-repmgr-cluster) instead. -## How to start working with Opensearch +## How to start working with OpenSearch -OpenSearch is the [successor](https://opendistro.github.io/for-elasticsearch-docs/) of OpenDistro for ElasticSearch project. Epipahny is providing an [automated solution](./UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) for migrating your existing ODFE installation to OpenSearch. +OpenSearch is the [successor](https://opendistro.github.io/for-elasticsearch-docs/) of OpenDistro for ElasticSearch project. Epiphany is providing an [automated solution](./UPGRADE.md#migration-from-open-distro-for-elasticsearch--kibana-to-opensearch-and-opensearch-dashboards) for migrating your existing ODFE installation to OpenSearch. On the other hand, if you plan to just start working with OpenSearch - change machines count to value greater than 0 in your cluster configuration: ```yaml @@ -485,7 +485,7 @@ kind: configuration/opensearch title: OpenSearch Config name: default specification: - cluster_name: EpiphanyOpensearch + cluster_name: EpiphanyOpenSearch ``` By default, OpenSearch Dashboards ( previously Kibana component ) is deployed only for `logging` component. If you want to deploy it From 71a693f23edf73f1e66584cfc44ef0242697d9dc Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 14:36:07 +0100 Subject: [PATCH 146/157] Typo correction --- .../playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml | 2 +- schema/common/defaults/configuration/logging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index 0ecdd21e70..a8f243097f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -8,7 +8,7 @@ debug: msg: - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Opensearch version to be installed: {{opensearch_ver }}" + - "Opensearch version to be installed: {{ opensearch_defaults.file_name_version.opensearch[ansible_architecture].split('-')[1] }}" - name: ODFE migr | Ensure elasticsearch cluster is up and running block: diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 330553e1bc..1d15337beb 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -2,7 +2,7 @@ kind: configuration/logging title: Logging Config name: default specification: - cluster_name: EpiphanyOpensearch + cluster_name: EpiphanyOpenSearch opensearch_user: opensearch opsd_user: opensearchboard opsd_password: PASSWORD_TO_CHANGE From e657a234c96916bb285bfcdd7ffa2ca1ad958a99 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 14:38:24 +0100 Subject: [PATCH 147/157] Moving the vars definitions to ops\defaults --- ansible/playbooks/roles/opensearch/defaults/main.yml | 7 +++++++ .../roles/opensearch/tasks/install-opensearch.yml | 11 ++++------- .../playbooks/roles/opensearch/tasks/set_facts.yml | 12 ------------ .../roles/opensearch_dashboards/defaults/main.yml | 4 ++++ .../roles/opensearch_dashboards/tasks/dashboards.yml | 7 ++----- .../roles/opensearch_dashboards/tasks/set_facts.yml | 10 ---------- ansible/playbooks/roles/upgrade/tasks/opensearch.yml | 5 +---- .../upgrade/tasks/opensearch/migrate-kibana.yml | 7 ++----- 8 files changed, 20 insertions(+), 43 deletions(-) delete mode 100644 ansible/playbooks/roles/opensearch/tasks/set_facts.yml delete mode 100644 ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index c0df8833ac..f8ed73e3ec 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -1,5 +1,12 @@ --- # This file is meant to be also used by upgrade role +file_name_version: + opensearch: + x86_64: opensearch-1.2.4-linux-x64.tar.gz + aarch64: opensearch-1.2.4-linux-arm64.tar.gz + opensearch_perftop: + x86_64: opensearch-perf-top-1.2.0.0-linux-x64.zip + # Perftop is not supported on ARM (https://github.com/opensearch-project/perftop/issues/26) certificates: dirs: certs: /usr/share/opensearch/config diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index 714eaca502..b398bc1ac6 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -1,20 +1,17 @@ --- -- name: Define the filenames for the download task - include_tasks: roles/opensearch/tasks/set_facts.yml - - name: Download Opensearch include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-{{ opensearch_ver }}" + file_name: "{{ file_name_version.opensearch[ansible_architecture] }}" - name: Download PerfTop include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-perf-top-{{ opensearch_perftop_ver }}" + file_name: "{{ file_name_version.opensearch_perftop[ansible_architecture] }}" when: ansible_architecture == "x86_64" # Perftop is not yet supported on ARM (https://github.com/opensearch-project/perftop/issues/26) - name: Ensure Opensearch service user exists @@ -42,7 +39,7 @@ - name: Extract OpenSearch tar file unarchive: - src: "/tmp/opensearch-{{ opensearch_ver }}" + src: "/tmp/{{ file_name_version.opensearch[ansible_architecture] }}" dest: "{{ specification.paths.opensearch_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes @@ -51,7 +48,7 @@ - name: Extract OpenSearch PerfTop tar file unarchive: - src: "/tmp/opensearch-perf-top-{{ opensearch_perftop_ver }}" + src: "/tmp/{{ file_name_version.opensearch_perftop[ansible_architecture] }}" dest: "{{ specification.paths.opensearch_perftop_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/opensearch/tasks/set_facts.yml b/ansible/playbooks/roles/opensearch/tasks/set_facts.yml deleted file mode 100644 index 0883e971f7..0000000000 --- a/ansible/playbooks/roles/opensearch/tasks/set_facts.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Set the name of binary to download for x86_64 family - set_fact: - opensearch_ver: "1.2.4-linux-x64.tar.gz" - opensearch_perftop_ver: "1.2.0.0-linux-x64.zip" - when: ansible_architecture == "x86_64" - -- name: Set the name of binary to download for arm64 family - set_fact: - opensearch_ver: "1.2.4-linux-arm64.tar.gz" - # Perftop is not supported on ARM (https://github.com/opensearch-project/perftop/issues/26) - when: ansible_architecture == "aarch64" diff --git a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml index 715efaca8f..e282dc07ae 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/defaults/main.yml @@ -1,4 +1,8 @@ --- +file_name_version: + opensearch_dashboards: + x86_64: opensearch-dashboards-1.2.0-linux-x64.tar.gz + aarch64: opensearch-dashboards-1.2.0-linux-arm64.tar.gz opensearch_api_port: 9200 opensearch_nodes_dashboards: |- {% for item in groups['opensearch_dashboards'] -%} diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index fa43837ad2..801eb4c64c 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -1,13 +1,10 @@ --- -- name: Define the filename for the download task - include_tasks: set_facts.yml - - name: Download Opensearch dashbaords include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ opensearchdash_version }}" + file_name: "{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" - name: Create OpenSearch Dashboards user user: @@ -24,7 +21,7 @@ - name: Extract OpenSearch Dashboards tar file unarchive: - src: "/tmp/opensearch-dashboards-{{ opensearchdash_version }}" + src: "/tmp/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" dest: "{{ specification.paths.opensearchdash_home }}" owner: "{{ specification.dashboards_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml deleted file mode 100644 index 50cf565ef0..0000000000 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/set_facts.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Set the name of binary to download for x86_64 family - set_fact: - opensearchdash_version: "1.2.0-linux-x64.tar.gz" - when: ansible_architecture == "x86_64" - -- name: Set the name of binary to download for arm64 family - set_fact: - opensearchdash_version: "1.2.0-linux-arm64.tar.gz" - when: ansible_architecture == "aarch64" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml index 31b69facfe..945cd41c2c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml @@ -20,9 +20,6 @@ file: roles/opensearch/vars/main.yml name: opensearch_vars -- name: OpenSearch | Set the versions of OpenSearch - include_tasks: roles/opensearch/tasks/set_facts.yml - - name: OpenSearch | Ensure Opensearch service user exists user: name: "{{ opensearch_vars.specification.opensearch_user }}" @@ -47,7 +44,7 @@ debug: msg: - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - - "Target version: {{ opensearch_ver.split('-')[0] }}" + - "Target version: {{ opensearch_defaults.file_name_version.opensearch[ansible_architecture].split('-')[1] }}" # If state file exists it means the previous run failed - name: OpenSearch | Check if upgrade state file exists diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 67261cc289..54a4a8b11a 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -13,15 +13,12 @@ enabled: no state: stopped -- name: Kibana migr | Define the filename for the download task - include_tasks: roles/opensearch_dashboards/tasks/set_facts.yml - - name: Kibana migr | Download Opensearch Dashboards binary include_role: name: download tasks_from: download_file vars: - file_name: "opensearch-dashboards-{{ opensearchdash_version }}" + file_name: "{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" - name: Kibana migr | Create opensearch-dashboards user user: @@ -43,7 +40,7 @@ - name: Kibana migr | Extract the tar file unarchive: - src: "/tmp/opensearch-dashboards-{{ opensearchdash_version }}" + src: "/tmp/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" dest: "{{ specification.paths.opensearchdash_home }}" owner: "{{ specification.dashboards_user }}" remote_src: yes From e503eb981af38a2f4e6fb4ee80bdd0ba539823ac Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 14:58:38 +0100 Subject: [PATCH 148/157] Fixing a typo --- docs/home/howto/LOGGING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/home/howto/LOGGING.md b/docs/home/howto/LOGGING.md index 73e6e432d8..93921b769e 100644 --- a/docs/home/howto/LOGGING.md +++ b/docs/home/howto/LOGGING.md @@ -1,6 +1,6 @@ # Centralized logging setup -For centralized logging Epiphany uses [OpenSearch](https://opensearch.org/) stack - an opensource successor[1] of Elasticsearch & Kibana projects. +For centralized logging Epiphany uses [Open Search](https://opensearch.org/) stack - an opensource successor[1] of Elasticsearch & Kibana projects. In order to enable centralized logging, be sure to set `count` property for `logging` feature to the value greater than 0 in your configuration manifest. From bfb1ec39f21b14eaa2b4bf0e9e2bd209106c34aa Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 14:59:17 +0100 Subject: [PATCH 149/157] Var renaming as requested in https://github.com/epiphany-platform/epiphany/pull/2983#discussion_r820593112 --- .../roles/opensearch_dashboards/tasks/dashboards.yml | 12 ++++++------ .../templates/opensearch-dashboards.service.j2 | 4 ++-- .../templates/opensearch_dashboards.yml.j2 | 2 +- .../upgrade/tasks/opensearch/migrate-kibana.yml | 10 +++++----- .../roles/upgrade/tasks/opensearch/migrate-odfe.yml | 2 +- .../defaults/configuration/opensearch-dashboards.yml | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index 801eb4c64c..2998304c74 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -8,7 +8,7 @@ - name: Create OpenSearch Dashboards user user: - name: "{{ specification.dashboards_user }}" + name: "{{ specification.dashboards_os_user }}" state: present shell: /bin/bash @@ -16,14 +16,14 @@ file: path: "{{ specification.paths.opensearchdash_home }}" state: directory - owner: "{{ specification.dashboards_user }}" - group: "{{ specification.dashboards_user }}" + owner: "{{ specification.dashboards_os_user }}" + group: "{{ specification.dashboards_os_user }}" - name: Extract OpenSearch Dashboards tar file unarchive: src: "/tmp/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" dest: "{{ specification.paths.opensearchdash_home }}" - owner: "{{ specification.dashboards_user }}" + owner: "{{ specification.dashboards_os_user }}" remote_src: yes extra_opts: - --strip-components=1 @@ -32,8 +32,8 @@ template: src: opensearch_dashboards.yml.j2 dest: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" - owner: "{{ specification.dashboards_user }}" - group: "{{ specification.dashboards_user }}" + owner: "{{ specification.dashboards_os_user }}" + group: "{{ specification.dashboards_os_user }}" mode: 0644 backup: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 index 41085a243d..4613cb3ace 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 @@ -9,8 +9,8 @@ PrivateTmp=true WorkingDirectory={{ specification.paths.opensearchdash_home }} -User={{ specification.dashboards_user }} -Group={{ specification.dashboards_user }} +User={{ specification.dashboards_os_user }} +Group={{ specification.dashboards_os_user }} ExecStart={{ specification.paths.opensearchdash_home }}/bin/opensearch-dashboards -q diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index 8f31fc3943..d295b2a52c 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -2,7 +2,7 @@ server.port: 5601 server.host: "{{ ansible_host }}" opensearch.hosts: ["{{ opensearch_nodes_dashboards }}"] opensearch.ssl.verificationMode: none -opensearch.username: "{{ specification.dashboards_user }}" +opensearch.username: "{{ specification.dashboards_os_user }}" opensearch.password: "{{ specification.dashboards_password }}" opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 54a4a8b11a..963ea855b3 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -22,7 +22,7 @@ - name: Kibana migr | Create opensearch-dashboards user user: - name: "{{ specification.dashboards_user }}" + name: "{{ specification.dashboards_os_user }}" password: "{{ specification.dashboards_password }}" state: present shell: /bin/bash @@ -31,8 +31,8 @@ file: path: "{{ item }}" state: directory - owner: "{{ specification.dashboards_user }}" - group: "{{ specification.dashboards_user }}" + owner: "{{ specification.dashboards_os_user }}" + group: "{{ specification.dashboards_os_user }}" mode: ug=rwx,o=rx with_items: - "{{ specification.paths.opensearchdash_log_dir }}" @@ -42,7 +42,7 @@ unarchive: src: "/tmp/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" dest: "{{ specification.paths.opensearchdash_home }}" - owner: "{{ specification.dashboards_user }}" + owner: "{{ specification.dashboards_os_user }}" remote_src: yes extra_opts: - --strip-components=1 @@ -52,7 +52,7 @@ src: /etc/kibana/kibana.yml dest: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" remote_src: yes - owner: "{{ specification.dashboards_user }}" + owner: "{{ specification.dashboards_os_user }}" group: root mode: ug=rw,o= backup: yes diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index a8f243097f..b98b7a3bbd 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -28,7 +28,7 @@ fail: msg: "Are you trying to migrate from ODFE ( opensearch_vars.specification.odfe_migration: true ) on an already migrated server?" when: - - opensearch_vars.specification.odfe_migration == true + - opensearch_vars.specification.odfe_migration - name: ODFE migr | Set existing_config facts diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml index 71c46c15e3..6c3f6d17a7 100644 --- a/schema/common/defaults/configuration/opensearch-dashboards.yml +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -2,7 +2,7 @@ kind: configuration/opensearch-dashboards title: "OpenSearch-Dashboards" name: default specification: - dashboards_user: opensearchdboard + dashboards_os_user: opensearchdboard dashboards_password: PASSWORD_TO_CHANGE paths: opensearchdash_home: /usr/share/opensearch-dashboards From 7f9e2fc30bb202e93e88a99bef4fc81df9980f71 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 15:23:26 +0100 Subject: [PATCH 150/157] Task renaming as requested in PR#2983 --- .../tasks/opensearch/migrate-kibana.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 963ea855b3..09ca2fc991 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -1,33 +1,33 @@ --- -- name: Kibana migr | Load deafults from Opensearch Dashboards role +- name: Kibana migration | Load deafults from Opensearch Dashboards role include_vars: file: roles/opensearch_dashboards/defaults/main.yml -- name: Kibana migr | Load vars from Opensearch Dashboards role # requires epicli upgrade -f .yml +- name: Kibana migration | Load vars from Opensearch Dashboards role # requires epicli upgrade -f .yml include_vars: file: roles/opensearch_dashboards/vars/main.yml -- name: Kibana migr | Stop Kibana service +- name: Kibana migration | Stop Kibana service systemd: name: kibana enabled: no state: stopped -- name: Kibana migr | Download Opensearch Dashboards binary +- name: Kibana migration | Download Opensearch Dashboards binary include_role: name: download tasks_from: download_file vars: file_name: "{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" -- name: Kibana migr | Create opensearch-dashboards user +- name: Kibana migration | Create opensearch-dashboards user user: name: "{{ specification.dashboards_os_user }}" password: "{{ specification.dashboards_password }}" state: present shell: /bin/bash -- name: Kibana migr | Create OPSD directories +- name: Kibana migration | Create OPSD directories file: path: "{{ item }}" state: directory @@ -38,16 +38,16 @@ - "{{ specification.paths.opensearchdash_log_dir }}" - "{{ specification.paths.opensearchdash_home }}" -- name: Kibana migr | Extract the tar file +- name: Kibana migration | Extract the tar file unarchive: - src: "/tmp/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" + src: "{{ download_directory }}/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" dest: "{{ specification.paths.opensearchdash_home }}" owner: "{{ specification.dashboards_os_user }}" remote_src: yes extra_opts: - --strip-components=1 -- name: Kibana migr | Clone kibana settings +- name: Kibana migration | Clone kibana settings copy: src: /etc/kibana/kibana.yml dest: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" @@ -57,7 +57,7 @@ mode: ug=rw,o= backup: yes -- name: Kibana migr | Porting kibana settings to OpenSearch Dashboards +- name: Kibana migration | Porting kibana settings to OpenSearch Dashboards replace: path: "{{ specification.paths.opensearchdash_conf_dir }}/opensearch_dashboards.yml" regexp: "{{ item.1 }}" @@ -71,27 +71,27 @@ - { 1: 'telemetry.optIn', 2: '#telemetry.optIn' } - { 1: 'telemetry.enabled', 2: '#telemetry.enabled' } -- name: Kibana migr | Create OpenSearch Dashboards service +- name: Kibana migration | Create OpenSearch Dashboards service template: src: roles/opensearch_dashboards/templates/opensearch-dashboards.service.j2 dest: /etc/systemd/system/opensearch-dashboards.service -- name: Kibana migr | Assure Opensearch Dashboards service is started +- name: Kibana migration | Assure Opensearch Dashboards service is started service: name: opensearch-dashboards state: started enabled: yes -- name: Kibana migr | Get all the installed dashboards plugins +- name: Kibana migration | Get all the installed dashboards plugins command: "{{ specification.paths.opensearchdash_plugin_bin_path }} list" become: false # This command can not be run as root user register: list_plugins -- name: Kibana migr | Show all the installed dashboards plugins +- name: Kibana migration | Show all the installed dashboards plugins debug: msg: "{{ list_plugins.stdout }}" -- name: Kibana migr | Prevent Filebeat API access problem # Workaround for https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656 +- name: Kibana migration | Prevent Filebeat API access problem # Workaround for https://github.com/opensearch-project/OpenSearch-Dashboards/issues/656 replace: path: /etc/filebeat/filebeat.yml regexp: 'setup.dashboards.enabled: true' From d59acfcaf134fd50bb0ca81da11fe7594ca7ce6b Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 15:24:12 +0100 Subject: [PATCH 151/157] Moving the static path to variable --- .../playbooks/roles/opensearch/tasks/install-opensearch.yml | 4 ++-- .../roles/opensearch_dashboards/tasks/dashboards.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index b398bc1ac6..363346883f 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -39,7 +39,7 @@ - name: Extract OpenSearch tar file unarchive: - src: "/tmp/{{ file_name_version.opensearch[ansible_architecture] }}" + src: "{{ download_directory }}/{{ file_name_version.opensearch[ansible_architecture] }}" dest: "{{ specification.paths.opensearch_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes @@ -48,7 +48,7 @@ - name: Extract OpenSearch PerfTop tar file unarchive: - src: "/tmp/{{ file_name_version.opensearch_perftop[ansible_architecture] }}" + src: "{{ download_directory }}/{{ file_name_version.opensearch_perftop[ansible_architecture] }}" dest: "{{ specification.paths.opensearch_perftop_home }}" owner: "{{ specification.opensearch_user }}" remote_src: yes diff --git a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml index 2998304c74..1dc5d97b43 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml +++ b/ansible/playbooks/roles/opensearch_dashboards/tasks/dashboards.yml @@ -21,7 +21,7 @@ - name: Extract OpenSearch Dashboards tar file unarchive: - src: "/tmp/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" + src: "{{ download_directory }}/{{ file_name_version.opensearch_dashboards[ansible_architecture] }}" dest: "{{ specification.paths.opensearchdash_home }}" owner: "{{ specification.dashboards_os_user }}" remote_src: yes From 2ebf81dae8ed2d04410bcb784026e3757cce098c Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Mon, 7 Mar 2022 21:04:29 +0100 Subject: [PATCH 152/157] Rephrasing as requested in PR#2983 --- docs/home/howto/MONITORING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/home/howto/MONITORING.md b/docs/home/howto/MONITORING.md index b2ba767f4b..7456c26133 100644 --- a/docs/home/howto/MONITORING.md +++ b/docs/home/howto/MONITORING.md @@ -233,7 +233,7 @@ When dashboard creation or import succeeds you will see it on your dashboard lis # OpenSearch Dashboards -OpenSearch Dashboards ( a Kibana successor ) is an open source search and analytics visualization layer. It also serves as a user interface for many OpenSearch project plugins. For more information please refer to [the official website](https://opensearch.org/docs/latest/dashboards/index/). +OpenSearch Dashboards ( a Kibana counterpart ) is an open source search and analytics visualization layer. It also serves as a user interface for many OpenSearch project plugins. For more information please refer to [the official website](https://opensearch.org/docs/latest/dashboards/index/). ## How to configure OpenSearch Dashboards From 185a05284835cd53ed1fdeca82366b6c83d3d8fc Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 8 Mar 2022 20:48:19 +0100 Subject: [PATCH 153/157] Adding the step name --- ansible/playbooks/roles/opensearch/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/main.yml b/ansible/playbooks/roles/opensearch/tasks/main.yml index e81ef02c63..1f1c8e3843 100644 --- a/ansible/playbooks/roles/opensearch/tasks/main.yml +++ b/ansible/playbooks/roles/opensearch/tasks/main.yml @@ -16,7 +16,8 @@ - name: Tune the system settings include_tasks: configure-sysctl.yml -- include_tasks: install-opensearch.yml +- name: Include installation tasks + include_tasks: install-opensearch.yml - name: Include configuration tasks include_tasks: configure-opensearch.yml From 726d7576f41ce06b3f164e4a9353439503e63409 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Tue, 8 Mar 2022 20:48:44 +0100 Subject: [PATCH 154/157] Renaming as requested in PR#2983 --- .../opensearch/tasks/configure-opensearch.yml | 12 +++--- .../roles/opensearch/tasks/generate-certs.yml | 16 ++++---- .../opensearch/tasks/install-opensearch.yml | 10 ++--- .../templates/opensearch.service.j2 | 4 +- .../templates/opensearch_dashboards.yml.j2 | 4 +- .../roles/upgrade/tasks/opensearch.yml | 6 +-- .../tasks/opensearch/migrate-kibana.yml | 2 +- .../tasks/opensearch/migrate-odfe-serial.yml | 28 ++++++------- .../upgrade/tasks/opensearch/migrate-odfe.yml | 40 +++++++++---------- .../common/defaults/configuration/logging.yml | 6 +-- .../configuration/opensearch-dashboards.yml | 3 +- .../defaults/configuration/opensearch.yml | 2 +- 12 files changed, 66 insertions(+), 67 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml index 1737801204..bb3c6b11cd 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml @@ -5,8 +5,8 @@ file: path: "{{ specification.paths.opensearch_repo }}/" state: directory - owner: "{{ specification.opensearch_user }}" - group: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_user }}" mode: u=rwx,go= - name: Provide JVM configuration file @@ -14,8 +14,8 @@ backup: yes src: jvm.options.j2 dest: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" - owner: "{{ specification.opensearch_user }}" - group: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_user }}" mode: ug=rw,o= register: change_jvm_config vars: @@ -36,8 +36,8 @@ backup: yes src: opensearch.yml.j2 dest: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" - owner: "{{ specification.opensearch_user }}" - group: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_user }}" mode: ug=rw,o= register: change_config vars: diff --git a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml index 194cd58865..476604a2f8 100644 --- a/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml +++ b/ansible/playbooks/roles/opensearch/tasks/generate-certs.yml @@ -5,7 +5,7 @@ file: state: directory path: "{{ certificates.dirs.ca_key }}" - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" group: opensearch mode: u=rwx,g=rwx,o= @@ -14,7 +14,7 @@ file: state: directory path: "{{ certificates.dirs.csr }}" - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" group: opensearch mode: u=rwx,g=rwx,o= # CSR file doesn't contain private key @@ -25,17 +25,17 @@ size: 2048 # based on ODFE docs type: RSA mode: u=rw,go= - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" format: pkcs8 community.crypto.openssl_csr: mode: u=rw,g=r,o= - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" use_common_name_for_san: false community.crypto.x509_certificate: selfsigned_digest: sha256 ownca_digest: sha256 mode: u=rw,g=r,o= - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" block: # --- Generate CA root certificate --- @@ -169,7 +169,7 @@ size: 2048 type: RSA mode: u=rw,g=r,o= - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" group: opensearch return_content: false register: node_key @@ -196,7 +196,7 @@ subjectAltName: "{{ _dns_list + [ 'IP:' + ansible_default_ipv4.address ] }}" use_common_name_for_san: false mode: u=rw,g=r,o= - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" group: opensearch register: node_csr vars: @@ -214,5 +214,5 @@ ownca_not_after: "{{ certificates.files.node.cert.ownca_not_after }}" ownca_digest: sha256 mode: u=rw,go=r - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" group: opensearch diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index 363346883f..9ffb57a21d 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -16,7 +16,7 @@ - name: Ensure Opensearch service user exists user: - name: "{{ specification.opensearch_user }}" + name: "{{ specification.opensearch_os_user }}" state: present shell: /bin/bash @@ -24,8 +24,8 @@ file: path: "{{ item }}" state: directory - owner: "{{ specification.opensearch_user }}" - group: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_user }}" mode: u=rwx,go=r recurse: yes loop: @@ -41,7 +41,7 @@ unarchive: src: "{{ download_directory }}/{{ file_name_version.opensearch[ansible_architecture] }}" dest: "{{ specification.paths.opensearch_home }}" - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" remote_src: yes extra_opts: - --strip-components=1 @@ -50,7 +50,7 @@ unarchive: src: "{{ download_directory }}/{{ file_name_version.opensearch_perftop[ansible_architecture] }}" dest: "{{ specification.paths.opensearch_perftop_home }}" - owner: "{{ specification.opensearch_user }}" + owner: "{{ specification.opensearch_os_user }}" remote_src: yes when: ansible_architecture == "x86_64" # Perftop is not yet supported on ARM (https://github.com/opensearch-project/perftop/issues/26) diff --git a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 index f3446c61c7..a886e79dd1 100644 --- a/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 +++ b/ansible/playbooks/roles/opensearch/templates/opensearch.service.j2 @@ -9,8 +9,8 @@ PrivateTmp=true WorkingDirectory={{ specification.paths.opensearch_home }} -User={{ specification.opensearch_user }} -Group={{ specification.opensearch_user }} +User={{ specification.opensearch_os_user }} +Group={{ specification.opensearch_os_user }} ExecStart={{ specification.paths.opensearch_home }}/bin/opensearch -p {{ specification.paths.opensearch_home }}/opensearch.pid -q diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index d295b2a52c..16211adb94 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -2,8 +2,8 @@ server.port: 5601 server.host: "{{ ansible_host }}" opensearch.hosts: ["{{ opensearch_nodes_dashboards }}"] opensearch.ssl.verificationMode: none -opensearch.username: "{{ specification.dashboards_os_user }}" -opensearch.password: "{{ specification.dashboards_password }}" +opensearch.username: "{{ specification.dashboards_user }}" +opensearch.password: "{{ specification.dashboards_os_user_password }}" opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] opensearch_security.multitenancy.enabled: true diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml index 945cd41c2c..6ef6d2e430 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch.yml @@ -22,7 +22,7 @@ - name: OpenSearch | Ensure Opensearch service user exists user: - name: "{{ opensearch_vars.specification.opensearch_user }}" + name: "{{ opensearch_vars.specification.opensearch_os_user }}" state: present shell: /bin/bash @@ -30,8 +30,8 @@ file: path: "{{ item }}" state: directory - owner: "{{ opensearch_vars.specification.opensearch_user }}" - group: "{{ opensearch_vars.specification.opensearch_user }}" + owner: "{{ opensearch_vars.specification.opensearch_os_user }}" + group: "{{ opensearch_vars.specification.opensearch_os_user }}" mode: u=rw,go=r recurse: yes with_items: diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml index 09ca2fc991..6cd7d0e16f 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-kibana.yml @@ -23,7 +23,7 @@ - name: Kibana migration | Create opensearch-dashboards user user: name: "{{ specification.dashboards_os_user }}" - password: "{{ specification.dashboards_password }}" + password: "{{ specification.dashboards_os_user_password }}" state: present shell: /bin/bash diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml index c8f44d584e..a50234e4bf 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe-serial.yml @@ -1,29 +1,29 @@ --- # Below tasks need to be run in serial -- name: ODFE migr | Stop elasticsearch service +- name: ODFE migration | Stop elasticsearch service systemd: name: elasticsearch enabled: no state: stopped register: elasticsearch_state -- name: ODFE migr | Include defaults Opensearch binaries installation +- name: ODFE migration | Include defaults Opensearch binaries installation include_vars: file: roles/opensearch/defaults/main.yml -- name: ODFE migr | Include vars for Opensearch binaries installation +- name: ODFE migration | Include vars for Opensearch binaries installation include_vars: file: roles/opensearch/vars/main.yml -- name: ODFE migr | Inastall Opensearch binaries +- name: ODFE migration | Inastall Opensearch binaries include_tasks: roles/opensearch/tasks/install-opensearch.yml -- name: ODFE migr | Copy ES directories to OPS directories +- name: ODFE migration | Copy ES directories to OPS directories copy: src: "{{ item.1 }}" dest: "{{ item.2 }}" remote_src: yes - owner: "{{ opensearch_vars.specification.opensearch_user }}" + owner: "{{ opensearch_vars.specification.opensearch_os_user }}" group: root mode: ug=rwx,o= directory_mode: yes @@ -31,20 +31,20 @@ - { 1: "/var/lib/elasticsearch-snapshots/", 2: "{{ specification.paths.opensearch_repo }}/" } - { 1: "/var/lib/elasticsearch", 2: "{{ specification.paths.opensearch_data }}" } -- name: ODFE migr | Prepare a list of ESS certs and keys +- name: ODFE migration | Prepare a list of ESS certs and keys find: paths: "/etc/elasticsearch/" patterns: "*pem" register: pem_files -- name: ODFE migr | Copy a list of certs and keys to OPS directories +- name: ODFE migration | Copy a list of certs and keys to OPS directories copy: src: "{{ item.path }}" dest: "{{ specification.paths.opensearch_conf_dir }}/" remote_src: yes with_items: "{{ pem_files.files }}" -- name: ODFE migr | Clone JVM configuration file +- name: ODFE migration | Clone JVM configuration file copy: src: /etc/elasticsearch/jvm.options dest: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" @@ -54,7 +54,7 @@ mode: ug=rw,o= backup: yes -- name: ODFE migr | Update JVM configuration file +- name: ODFE migration | Update JVM configuration file replace: path: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" regexp: "{{ item.1 }}" @@ -63,7 +63,7 @@ - { 1: 'elasticsearch', 2: 'opensearch' } - { 1: '\${ES_TMPDIR}', 2: '${OPENSEARCH_TMPDIR}' } -- name: ODFE migr | Clone main configuration file +- name: ODFE migration | Clone main configuration file copy: src: /etc/elasticsearch/elasticsearch.yml dest: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" @@ -73,7 +73,7 @@ mode: ug=rw,o= backup: yes -- name: ODFE migr | Update main configuration file +- name: ODFE migration | Update main configuration file replace: path: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" regexp: "{{ item.1 }}" @@ -83,14 +83,14 @@ - { 1: 'EpiphanyElastic', 2: 'EpiphanyOpensearch' } - { 1: 'opendistro_security.', 2: 'plugins.security.' } -- name: ODFE migr | Start OpenSearch service +- name: ODFE migration | Start OpenSearch service systemd: name: opensearch state: started enabled: yes register: restart_opensearch -- name: ODFE migr | Wait for opensearch to startup +- name: ODFE migration | Wait for opensearch to startup wait_for: port: 9200 host: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml index b98b7a3bbd..144747b32c 100644 --- a/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml +++ b/ansible/playbooks/roles/upgrade/tasks/opensearch/migrate-odfe.yml @@ -10,31 +10,29 @@ - "Elasticsearch version currently installed: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - "Opensearch version to be installed: {{ opensearch_defaults.file_name_version.opensearch[ansible_architecture].split('-')[1] }}" -- name: ODFE migr | Ensure elasticsearch cluster is up and running +- name: ODFE migration | Ensure elasticsearch cluster is up and running block: - name: OpenSearch | Include vars from opensearch role # requires epicli upgrade -f .yml include_vars: file: roles/opensearch/vars/main.yml name: opensearch_vars - - name: ODFE migr | Ensure elasticsearch cluster is up and running + - name: ODFE migration | Ensure elasticsearch cluster is up and running systemd: name: elasticsearch enabled: yes state: restarted register: elasticsearch_state rescue: - - name: ODFE migr | Suggest potential problem solution and fail + - name: ODFE migration | Suggest potential problem solution and fail fail: msg: "Are you trying to migrate from ODFE ( opensearch_vars.specification.odfe_migration: true ) on an already migrated server?" - when: - - opensearch_vars.specification.odfe_migration + when: opensearch_vars.specification.odfe_migration - -- name: ODFE migr | Set existing_config facts +- name: ODFE migration | Set existing_config facts include_tasks: opensearch/utils/get-config-from-files.yml -- name: ODFE migr | Set common facts +- name: ODFE migration | Set common facts set_fact: certificates: "{{ opensearch_defaults.certificates }}" es_host: "{{ existing_config.main['network.host'] | default('_local_') }}" @@ -43,7 +41,7 @@ es_clustered: "{{ (existing_config.main['discovery.seed_hosts'] | length > 1) | ternary(True, False) }}" es_node_name: "{{ existing_config.main['node.name'] }}" -- name: ODFE migr | Assure Elasticsearch files location will be used in following tasks +- name: ODFE migration | Assure Elasticsearch files location will be used in following tasks set_fact: certificates: dirs: @@ -55,7 +53,7 @@ cert: "epiphany-admin.pem" key: "epiphany-admin-key.pem" -- name: ODFE migr | Prepare for ODFE to OPS migration +- name: ODFE migration | Prepare for ODFE to OPS migration include_tasks: file: opensearch/utils/prepare-cluster-for-node-restart.yml apply: @@ -72,7 +70,7 @@ url: https://{{ es_host }}:{{ es_http_port }} fail_msg: API access test failed. -- name: ODFE migr | Run core migration tasks individually on each node +- name: ODFE migration | Run core migration tasks individually on each node include_tasks: file: opensearch/migrate-odfe-serial.yml apply: @@ -83,7 +81,7 @@ loop_var: target_hostname run_once: true -- name: ODFE migr | Check if default admin user exists +- name: ODFE migration | Check if default admin user exists uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/admin" method: GET @@ -98,7 +96,7 @@ delay: 1 run_once: true -- name: ODFE migr | Set Opensearch admin password +- name: ODFE migration | Set Opensearch admin password uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" method: PATCH @@ -107,7 +105,7 @@ - op: "replace" path: "/admin" value: - password: "{{ specification.opensearch_password }}" + password: "{{ specification.admin_password }}" reserved: "true" backend_roles: - "admin" @@ -123,7 +121,7 @@ run_once: true when: admin_check_response.status == 200 -- name: ODFE migr | Check if kibanaserver user exists +- name: ODFE migration | Check if kibanaserver user exists uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/kibanaserver" method: GET @@ -138,7 +136,7 @@ delay: 1 run_once: true -- name: ODFE migr | Set kibanaserver user password +- name: ODFE migration | Set kibanaserver user password uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" method: PATCH @@ -161,7 +159,7 @@ run_once: true when: kibanaserver_check_response.status == 200 -- name: ODFE migr | Check if logstash user exists +- name: ODFE migration | Check if logstash user exists uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers/logstash" method: GET @@ -176,7 +174,7 @@ delay: 1 run_once: true -- name: ODFE migr | Set logstash user password +- name: ODFE migration | Set logstash user password uri: url: "https://{{ inventory_hostname }}:{{ ports.http }}/_opendistro/_security/api/internalusers" method: PATCH @@ -199,16 +197,16 @@ run_once: true when: logstash_check_response.status == 200 -- name: ODFE migr | Check the OpenSearch status +- name: ODFE migration | Check the OpenSearch status command: curl https://{{ inventory_hostname }}:{{ ports.http }}/_cluster/health?pretty -u 'admin:{{ specification.admin_password }}' -k register: opensearch_status -- name: ODFE migr | Show the OpenSearch status +- name: ODFE migration | Show the OpenSearch status debug: msg: "{{ opensearch_status.stdout }}" failed_when: "'number_of_nodes' not in opensearch_status.stdout" -- name: ODFE migr | Reenable shard allocation for the cluster +- name: ODFE migration | Reenable shard allocation for the cluster include_tasks: file: opensearch/utils/enable-shard-allocation.yml apply: diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 1d15337beb..13ddf426cb 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -3,9 +3,9 @@ title: Logging Config name: default specification: cluster_name: EpiphanyOpenSearch - opensearch_user: opensearch - opsd_user: opensearchboard - opsd_password: PASSWORD_TO_CHANGE + opensearch_os_user: opensearch + dashboards_os_user: opensearchboard + dashboards_os_user_password: PASSWORD_TO_CHANGE admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: true diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml index 6c3f6d17a7..d772367be5 100644 --- a/schema/common/defaults/configuration/opensearch-dashboards.yml +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -3,7 +3,8 @@ title: "OpenSearch-Dashboards" name: default specification: dashboards_os_user: opensearchdboard - dashboards_password: PASSWORD_TO_CHANGE + dashboards_os_user_password: PASSWORD_TO_CHANGE + dashboards_user: kibanaserver paths: opensearchdash_home: /usr/share/opensearch-dashboards opensearchdash_conf_dir: /usr/share/opensearch-dashboards/config diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index e324e6ff7b..bdea84cf13 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -5,7 +5,7 @@ specification: cluster_name: EpiphanyOpensearch odfe_migration: false clustered: true - opensearch_user: opensearch + opensearch_os_user: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: false From 66411dbc2720ec0913fbf8da73ee1add41746b55 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 10 Mar 2022 17:03:37 +0100 Subject: [PATCH 155/157] Corrected names --- .../templates/opensearch_dashboards.yml.j2 | 2 +- schema/common/defaults/configuration/opensearch-dashboards.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 index 16211adb94..c3b62436fa 100644 --- a/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 +++ b/ansible/playbooks/roles/opensearch_dashboards/templates/opensearch_dashboards.yml.j2 @@ -3,7 +3,7 @@ server.host: "{{ ansible_host }}" opensearch.hosts: ["{{ opensearch_nodes_dashboards }}"] opensearch.ssl.verificationMode: none opensearch.username: "{{ specification.dashboards_user }}" -opensearch.password: "{{ specification.dashboards_os_user_password }}" +opensearch.password: "{{ specification.dashboards_user_password }}" opensearch.requestHeadersWhitelist: [ authorization,securitytenant ] opensearch_security.multitenancy.enabled: true diff --git a/schema/common/defaults/configuration/opensearch-dashboards.yml b/schema/common/defaults/configuration/opensearch-dashboards.yml index d772367be5..836b091c1f 100644 --- a/schema/common/defaults/configuration/opensearch-dashboards.yml +++ b/schema/common/defaults/configuration/opensearch-dashboards.yml @@ -5,6 +5,7 @@ specification: dashboards_os_user: opensearchdboard dashboards_os_user_password: PASSWORD_TO_CHANGE dashboards_user: kibanaserver + dashboards_user_password: PASSWORD_TO_CHANGE paths: opensearchdash_home: /usr/share/opensearch-dashboards opensearchdash_conf_dir: /usr/share/opensearch-dashboards/config From 2699b76a4783a3e2e5866e12039e3d369d7795ae Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 10 Mar 2022 17:51:38 +0100 Subject: [PATCH 156/157] Moving the group name from harcoded into var --- .../roles/opensearch/tasks/configure-opensearch.yml | 6 +++--- .../roles/opensearch/tasks/install-opensearch.yml | 9 +++++++-- schema/common/defaults/configuration/logging.yml | 1 + schema/common/defaults/configuration/opensearch.yml | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml index bb3c6b11cd..43d8026775 100644 --- a/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/configure-opensearch.yml @@ -6,7 +6,7 @@ path: "{{ specification.paths.opensearch_repo }}/" state: directory owner: "{{ specification.opensearch_os_user }}" - group: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_group }}" mode: u=rwx,go= - name: Provide JVM configuration file @@ -15,7 +15,7 @@ src: jvm.options.j2 dest: "{{ specification.paths.opensearch_conf_dir }}/jvm.options" owner: "{{ specification.opensearch_os_user }}" - group: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_group }}" mode: ug=rw,o= register: change_jvm_config vars: @@ -37,7 +37,7 @@ src: opensearch.yml.j2 dest: "{{ specification.paths.opensearch_conf_dir }}/opensearch.yml" owner: "{{ specification.opensearch_os_user }}" - group: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_group }}" mode: ug=rw,o= register: change_config vars: diff --git a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml index 9ffb57a21d..8cef9b5b70 100644 --- a/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml +++ b/ansible/playbooks/roles/opensearch/tasks/install-opensearch.yml @@ -14,7 +14,12 @@ file_name: "{{ file_name_version.opensearch_perftop[ansible_architecture] }}" when: ansible_architecture == "x86_64" # Perftop is not yet supported on ARM (https://github.com/opensearch-project/perftop/issues/26) -- name: Ensure Opensearch service user exists +- name: Ensure Opensearch service OS group exists + group: + name: "{{ specification.opensearch_os_group }}" + state: present + +- name: Ensure Opensearch service OS user exists user: name: "{{ specification.opensearch_os_user }}" state: present @@ -25,7 +30,7 @@ path: "{{ item }}" state: directory owner: "{{ specification.opensearch_os_user }}" - group: "{{ specification.opensearch_os_user }}" + group: "{{ specification.opensearch_os_group }}" mode: u=rwx,go=r recurse: yes loop: diff --git a/schema/common/defaults/configuration/logging.yml b/schema/common/defaults/configuration/logging.yml index 13ddf426cb..cb971cce68 100644 --- a/schema/common/defaults/configuration/logging.yml +++ b/schema/common/defaults/configuration/logging.yml @@ -4,6 +4,7 @@ name: default specification: cluster_name: EpiphanyOpenSearch opensearch_os_user: opensearch + opensearch_os_group: opensearch dashboards_os_user: opensearchboard dashboards_os_user_password: PASSWORD_TO_CHANGE admin_password: PASSWORD_TO_CHANGE diff --git a/schema/common/defaults/configuration/opensearch.yml b/schema/common/defaults/configuration/opensearch.yml index bdea84cf13..287d29dca9 100644 --- a/schema/common/defaults/configuration/opensearch.yml +++ b/schema/common/defaults/configuration/opensearch.yml @@ -6,6 +6,7 @@ specification: odfe_migration: false clustered: true opensearch_os_user: opensearch + opensearch_os_group: opensearch admin_password: PASSWORD_TO_CHANGE kibanaserver_password: PASSWORD_TO_CHANGE kibanaserver_user_active: false From c2e864189e98a8490343d3b51d135608d05bc1a1 Mon Sep 17 00:00:00 2001 From: Roman Sokalski <24roman12@gmail.com> Date: Thu, 10 Mar 2022 18:11:22 +0100 Subject: [PATCH 157/157] TO align w/ PR #3005 --- .../health-monitor/health-monitor.md | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 docs/design-docs/health-monitor/health-monitor.md diff --git a/docs/design-docs/health-monitor/health-monitor.md b/docs/design-docs/health-monitor/health-monitor.md deleted file mode 100644 index 1c51f91072..0000000000 --- a/docs/design-docs/health-monitor/health-monitor.md +++ /dev/null @@ -1,33 +0,0 @@ -# Epiphany Health Monitor service design proposal - -Affected version: 0.6.x/0.7.x - -## Goals - -Provide service that will be monitoring components (Kubernetes, Docker, Kafka, EFK, Prometheus, etc.) deployed using Epiphany. - -## Use cases - -Service will be installed and used on Virtual Machines/Bare Metal on Ubuntu and RedHat (systemd service). -Health Monitor will check status of components that were installed on the cluster. Combinations of those components can be different and will be provided to the service through configuration file. - -Components that Health Monitor should check: -- Kubernetes (kubelet)* -- Query Kubernetes health endpoint (/healthz)* -- Docker* -- Query Docker stats* -- PostgreSQL -- HAProxy -- Prometheus -- Kafka -- ZooKeeper -- OpenSearch -- RabbitMQ - -`*` means MVP version. - -Health Monitor exposes endpoint that is compliant with [Prometheus metrics format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-format-example) and serves data about health checks. This endpoint should listen on the configurable port (default 98XX). - -## Design proposal - -TODO \ No newline at end of file