[2.0.1] Migration to OpenSearch (#3093)

* All in one commit - from PR #2983

* Ansible-lint adjustments

* Remove leftovers

* Tests fix

* Adjust download-requirements

* HA fix, improvements

* Fix defaults, schema

* Fixes in migration to opensearch and opensearch dashboards, add cleanup

* Improvements

* Changes after review

* Update doc, changelog and schema after review

* Spec tests update, rebase + changes after review

* Fix defaults

* Fix unittest

* Fix backup/restore

* Replace kibana with opensearch_dashboards

* Fix apply mode, cleanup, add opensearch spec test

* Disable upgrade of logging/opensearch, cleanup and rename vars

README.md

@@ -8,9 +8,9 @@ Epiphany at its core is a full automation of Kubernetes and Docker plus addition
 
 - Kafka or RabbitMQ for high speed messaging/events
 - Prometheus and Alertmanager for monitoring with Graphana for visualization
-- Elasticsearch and Kibana for centralized logging (OpenDistro)
+- OpenSearch for centralized logging
 - HAProxy for loadbalancing
-- Postgres and Elasticsearch for data storage
+- Postgres and OpenSearch for data storage
 - KeyCloak for authentication
 - Helm as package manager for Kubernetes
 

ansible/playbooks/backup_logging.yml

@@ -16,27 +16,19 @@
         - name: Run elasticsearch snapshot tasks
           import_role:
             name: backup
-            tasks_from: logging_elasticsearch_snapshot
+            tasks_from: logging_opensearch_snapshot
         - name: Run elasticsearch archive tasks
           import_role:
             name: backup
-            tasks_from: logging_elasticsearch_etc
-
-- hosts: kibana[0]
-  gather_facts: true
-  become: true
-  become_method: sudo
-  serial: 1
-  tasks:
-    - when: specification.components.logging.enabled | default(false)
-      block:
-        - name: Include kibana vars
+            tasks_from: logging_opensearch_conf
+        # OpenSearch Dashboards
+        - name: Include opensearch_dashboards vars
           include_vars:
-            file: roles/kibana/vars/main.yml
+            file: roles/opensearch_dashboards/vars/main.yml
             name: component_vars
-        - name: Run kibana backup tasks
+        - name: Run opensearch_dashboards backup tasks
           import_role:
             name: backup
-            tasks_from: logging_kibana_etc
+            tasks_from: logging_opensearch_dashboards_conf
           vars:
             snapshot_name: "{{ hostvars[groups.logging.0].snapshot_name }}"

ansible/playbooks/filebeat.yml

@@ -1,7 +1,7 @@
 ---
 # Ansible playbook that installs and configures Filebeat
 
-- hosts: opendistro_for_elasticsearch:logging:kibana # to gather facts
+- hosts: opensearch:logging:opensearch_dashboards # to gather facts
   tasks: []
 
 - hosts: filebeat

ansible/playbooks/opensearch.yml

@@ -0,0 +1,10 @@
+---
+# Ansible playbook for installing OpenSearch
+
+- hosts: opensearch
+  become: true
+  become_method: sudo
+  roles:
+    - opensearch
+  vars:
+    current_group_name: "opensearch"

ansible/playbooks/opensearch_dashboards.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for installing OpenSearch Dashboards
+
+- hosts: repository # to gather facts
+  tasks: []
+
+- hosts: opensearch_dashboards
+  become: true
+  become_method: sudo
+  roles:
+    - opensearch_dashboards

ansible/playbooks/recovery_logging.yml

@@ -13,22 +13,15 @@
             name: component_vars
         - import_role:
             name: recovery
-            tasks_from: logging_elasticsearch_etc
+            tasks_from: logging_opensearch_conf
         - import_role:
             name: recovery
-            tasks_from: logging_elasticsearch_snapshot
+            tasks_from: logging_opensearch_snapshot
 
-- hosts: kibana[0]
-  gather_facts: true
-  become: true
-  become_method: sudo
-  serial: 1
-  tasks:
-    - when: specification.components.logging.enabled | default(false)
-      block:
+        # OpenSearch Dashboards
         - include_vars:
-            file: roles/kibana/vars/main.yml
+            file: roles/opensearch_dashboards/vars/main.yml
             name: component_vars
         - import_role:
             name: recovery
-            tasks_from: logging_kibana_etc
+            tasks_from: logging_opensearch_dashboards_conf

ansible/playbooks/roles/backup/defaults/main.yml

@@ -2,6 +2,6 @@
 backup_dir: /epibackup
 backup_destination_dir: "{{ backup_dir }}/mounted"
 backup_destination_host: >-
-  {{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}
-elasticsearch_snapshot_repository_name: epiphany
-elasticsearch_snapshot_repository_location: /var/lib/elasticsearch-snapshots
+  "{{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}"
+opensearch_snapshot_repository_name: epiphany
+opensearch_snapshot_repository_location: /var/lib/opensearch-snapshots

ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml → ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml

@@ -1,4 +1,14 @@
 ---
+- name: Include default vars from opensearch role
+  include_vars:
+    file: roles/opensearch/defaults/main.yml
+    name: opensearch_defaults
+
+- name: Include vars from opensearch role
+  include_vars:
+    file: roles/opensearch/vars/main.yml
+    name: opensearch_vars
+
 - name: Assert that the snapshot_name fact is defined and valid
   assert:
     that:
@@ -13,9 +23,9 @@
 - name: Create snapshot archive
   import_tasks: common/create_snapshot_archive.yml
   vars:
-    snapshot_prefix: "elasticsearch_etc"
+    snapshot_prefix: "opensearch_conf"
     dirs_to_archive:
-      - /etc/elasticsearch/
+      - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}"
 
 - name: Create snapshot checksum
   import_tasks: common/create_snapshot_checksum.yml

ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml → ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml

@@ -10,12 +10,17 @@
 - name: Display snapshot name
   debug: var=snapshot_name
 
+- name: Include vars from opensearch_dashboards role
+  include_vars:
+    file: roles/opensearch_dashboards/vars/main.yml
+    name: opensearch_dashboards_vars
+
 - name: Create snapshot archive
   import_tasks: common/create_snapshot_archive.yml
   vars:
-    snapshot_prefix: "kibana_etc"
+    snapshot_prefix: "opensearch_dashboards_conf_dir"
     dirs_to_archive:
-      - /etc/kibana/
+      - "{{ opensearch_dashboards_vars.specification.paths.opensearch_dashboards_conf_dir }}"
 
 - name: Create snapshot checksum
   import_tasks: common/create_snapshot_checksum.yml

ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml

@@ -0,0 +1,96 @@
+---
+- name: Include default vars from opensearch role
+  include_vars:
+    file: roles/opensearch/defaults/main.yml
+    name: opensearch_defaults
+
+- name: Set helper facts
+  set_fact:
+    opensearch_endpoint: >-
+      https://{{ ansible_default_ipv4.address }}:9200
+    snapshot_name: >-
+      {{ ansible_date_time.iso8601_basic_short | replace('T','-') }}
+  vars:
+    uri_template: &uri
+      client_cert: "{{ opensearch_defaults.certificates.dirs.certs }}/{{ opensearch_defaults.certificates.files.admin.cert.filename }}"
+      client_key:  "{{ opensearch_defaults.certificates.dirs.certs }}/{{ opensearch_defaults.certificates.files.admin.key.filename }}"
+      validate_certs: false
+      body_format: json
+
+- name: Check cluster health
+  uri:
+    <<: *uri
+    url: "{{ opensearch_endpoint }}/_cluster/health"
+    method: GET
+    return_content: yes
+  register: cluster_status
+  until: cluster_status.json.status
+  retries: 60
+  delay: 1
+
+- name: Show warning when backup is not supported
+  when: not cluster_status.json.number_of_nodes == 1
+  debug:
+    msg: "[WARNING] No snapshot backup created as only single-node cluster backup is supported."
+
+- name: Snapshot backup
+  when: cluster_status.json.number_of_nodes == 1  # https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/BACKUP.md#logging
+  block:
+    - name: Ensure snapshot repository is defined
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}"
+        method: PUT
+        body:
+          type: fs
+          settings:
+            location: "{{ opensearch_snapshot_repository_location }}"
+            compress: true
+
+    - name: Trigger snapshot creation
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}"
+        method: PUT
+
+    - name: Wait (up to 12h) for snapshot completion
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}"
+        method: GET
+      register: uri_response
+      until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS"
+      retries: "{{ (12 * 3600 // 10) | int }}"  # 12h
+      delay: 10
+
+    - name: Find all snapshots
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/_all"
+        method: GET
+      register: uri_response
+
+    - name: Delete old snapshots
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ item }}"
+        method: DELETE
+      loop: >-
+        {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }}
+
+    - name: Create snapshot archive
+      import_tasks: common/create_snapshot_archive.yml
+      vars:
+        snapshot_prefix: "opensearch_snapshot"
+        dirs_to_archive:
+          - "{{ opensearch_snapshot_repository_location }}/"
+
+    - name: Create snapshot checksum
+      import_tasks: common/create_snapshot_checksum.yml
+
+    - name: Transfer artifacts via rsync
+      import_tasks: common/download_via_rsync.yml
+      vars:
+        artifacts:
+          - "{{ snapshot_path }}"
+          - "{{ snapshot_path }}.sha1"

ansible/playbooks/roles/certificate/tasks/install-packages.yml

@@ -10,4 +10,4 @@
       RedHat:
         - python3-cryptography
   module_defaults:
-    yum: { lock_timeout: "{{ yum_lock_timeout }}" }
+    yum: {lock_timeout: "{{ yum_lock_timeout }}"}

ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 - name: Include installation task
-  include_tasks: install-es-curator-{{ ansible_os_family }}.yml
+  include_tasks: install-ops-curator-{{ ansible_os_family }}.yml
 
 - name: Include configuration tasks
   include_tasks: configure-cron-jobs.yml