ODFE/ElasticSearch to OpenSearch migration (#2870)

README.md

@@ -8,9 +8,9 @@ Epiphany at its core is a full automation of Kubernetes and Docker plus addition
 
 - Kafka or RabbitMQ for high speed messaging/events
 - Prometheus and Alertmanager for monitoring with Graphana for visualization
-- Elasticsearch and Kibana for centralized logging (OpenDistro)
+- OpenSearch for centralized logging
 - HAProxy for loadbalancing
-- Postgres and Elasticsearch for data storage
+- Postgres and OpenSearch for data storage
 - KeyCloak for authentication
 - Helm as package manager for Kubernetes
 

ansible/playbooks/backup_logging.yml

@@ -14,10 +14,10 @@
             name: component_vars
         - import_role:
             name: backup
-            tasks_from: logging_elasticsearch_snapshot
+            tasks_from: logging_opensearch_snapshot
         - import_role:
             name: backup
-            tasks_from: logging_elasticsearch_etc
+            tasks_from: logging_opensearch_conf
 
 - hosts: kibana[0]
   gather_facts: true
@@ -28,10 +28,10 @@
     - when: specification.components.logging.enabled | default(false)
       block:
         - include_vars:
-            file: roles/kibana/vars/main.yml
+            file: roles/opensearch_dashboards/vars/main.yml
             name: component_vars
         - import_role:
             name: backup
-            tasks_from: logging_kibana_etc
+            tasks_from: logging_opensearch_dashboards_conf
           vars:
             snapshot_name: "{{ hostvars[groups.logging.0].snapshot_name }}"

ansible/playbooks/filebeat.yml

@@ -1,7 +1,7 @@
 ---
 # Ansible playbook that installs and configures Filebeat
 
-- hosts: opendistro_for_elasticsearch:logging:kibana # to gather facts
+- hosts: opensearch:logging:opensearch_dashboards # to gather facts
   tasks: []
 
 - hosts: filebeat

ansible/playbooks/opensearch.yml

@@ -0,0 +1,10 @@
+---
+# Ansible playbook for installing OpenSearch
+
+- hosts: opensearch
+  become: true
+  become_method: sudo
+  roles:
+    - opensearch
+  vars:
+    current_group_name: "opensearch"

ansible/playbooks/opensearch_dashboards.yml

@@ -0,0 +1,10 @@
+---
+# Ansible playbook for installing OpenSearch Dashboards
+- hosts: repository # to gather facts
+  tasks: []
+
+- hosts: opensearch_dashboards
+  become: true
+  become_method: sudo
+  roles:
+    - opensearch_dashboards

ansible/playbooks/recovery_logging.yml

@@ -13,10 +13,10 @@
             name: component_vars
         - import_role:
             name: recovery
-            tasks_from: logging_elasticsearch_etc
+            tasks_from: logging_opensearch_conf
         - import_role:
             name: recovery
-            tasks_from: logging_elasticsearch_snapshot
+            tasks_from: logging_opensearch_snapshot
 
 - hosts: kibana[0]
   gather_facts: true
@@ -27,8 +27,8 @@
     - when: specification.components.logging.enabled | default(false)
       block:
         - include_vars:
-            file: roles/kibana/vars/main.yml
+            file: roles/opensearch_dashboards/vars/main.yml
             name: component_vars
         - import_role:
             name: recovery
-            tasks_from: logging_kibana_etc
+            tasks_from: logging_opensearch_dashboards_conf

ansible/playbooks/roles/backup/defaults/main.yml

@@ -2,5 +2,5 @@
 backup_dir: /epibackup
 backup_destination_dir: "{{ backup_dir }}/mounted"
 backup_destination_host: "{{ groups.repository[0] if (custom_repository_url | default(false)) else (resolved_repository_hostname | default(groups.repository[0])) }}"
-elasticsearch_snapshot_repository_name: epiphany
-elasticsearch_snapshot_repository_location: /var/lib/elasticsearch-snapshots
+opensearch_snapshot_repository_name: epiphany
+opensearch_snapshot_repository_location: /var/lib/opensearch-snapshots

ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml → ansible/playbooks/roles/backup/tasks/logging_opensearch_conf.yml

@@ -1,4 +1,14 @@
 ---
+- name: Include default vars from opensearch role
+  include_vars:
+    file: roles/opensearch/defaults/main.yml
+    name: opensearch_defaults
+
+- name: Include vars from opensearch role
+  include_vars:
+    file: roles/opensearch/vars/main.yml
+    name: opensearch_vars
+
 - name: Assert that the snapshot_name fact is defined and valid
   assert:
     that:
@@ -12,9 +22,9 @@
 - name: Create snapshot archive
   import_tasks: common/create_snapshot_archive.yml
   vars:
-    snapshot_prefix: "elasticsearch_etc"
+    snapshot_prefix: "opensearch_conf"
     dirs_to_archive:
-      - /etc/elasticsearch/
+      - "{{ opensearch_vars.specification.paths.opensearch_conf_dir }}"
 
 - name: Create snapshot checksum
   import_tasks: common/create_snapshot_checksum.yml

ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml → ansible/playbooks/roles/backup/tasks/logging_opensearch_dashboards_conf.yml

@@ -9,12 +9,17 @@
 
 - debug: var=snapshot_name
 
+- name: Include vars from opensearch_dashboards role
+  include_vars:
+    file: roles/opensearch_dashboards/vars/main.yml
+    name: opensearch_dashboards_vars
+
 - name: Create snapshot archive
   import_tasks: common/create_snapshot_archive.yml
   vars:
-    snapshot_prefix: "kibana_etc"
+    snapshot_prefix: "opensearch_dashboards_conf_dir"
     dirs_to_archive:
-      - /etc/kibana/
+      - "{{ opensearch_dashboards_vars.specification.paths.opensearch_dashboards_conf_dir }}"
 
 - name: Create snapshot checksum
   import_tasks: common/create_snapshot_checksum.yml

ansible/playbooks/roles/backup/tasks/logging_opensearch_snapshot.yml

@@ -0,0 +1,96 @@
+---
+- name: Include default vars from opensearch role
+  include_vars:
+    file: roles/opensearch/defaults/main.yml
+    name: opensearch_defaults
+
+- name: Set helper facts
+  set_fact:
+    opensearch_endpoint: >-
+      https://{{ ansible_default_ipv4.address }}:9200
+    snapshot_name: >-
+      {{ ansible_date_time.iso8601_basic_short | replace('T','-') }}
+  vars:
+    uri_template: &uri
+      client_cert: "{{ opensearch_defaults.certificates.dirs.certs }}/{{ opensearch_defaults.certificates.files.admin.cert.filename }}"
+      client_key:  "{{ opensearch_defaults.certificates.dirs.certs }}/{{ opensearch_defaults.certificates.files.admin.key.filename }}"
+      validate_certs: false
+      body_format: json
+
+- name: Check cluster health
+  uri:
+    <<: *uri
+    url: "{{ opensearch_endpoint }}/_cluster/health"
+    method: GET
+    return_content: yes
+  register: cluster_status
+  until: cluster_status.json.status
+  retries: 60
+  delay: 1
+
+- name: No backup warning
+  when: not cluster_status.json.number_of_nodes == 1
+  debug:
+    msg: "[WARNING] No snapshot backup created as only single-node cluster backup is supported."
+
+- name: Snapshot backup
+  when: cluster_status.json.number_of_nodes == 1  # https://github.com/epiphany-platform/epiphany/blob/develop/docs/home/howto/BACKUP.md#logging
+  block:
+    - name: Ensure snapshot repository is defined
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}"
+        method: PUT
+        body:
+          type: fs
+          settings:
+            location: "{{ opensearch_snapshot_repository_location }}"
+            compress: true
+
+    - name: Trigger snapshot creation
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}"
+        method: PUT
+
+    - name: Wait (up to 12h) for snapshot completion
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ snapshot_name }}"
+        method: GET
+      register: uri_response
+      until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS"
+      retries: "{{ (12 * 3600 // 10) | int }}"  # 12h
+      delay: 10
+
+    - name: Find all snapshots
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/_all"
+        method: GET
+      register: uri_response
+
+    - name: Delete old snapshots
+      uri:
+        <<: *uri
+        url: "{{ opensearch_endpoint }}/_snapshot/{{ opensearch_snapshot_repository_name }}/{{ item }}"
+        method: DELETE
+      loop: >-
+        {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }}
+
+    - name: Create snapshot archive
+      import_tasks: common/create_snapshot_archive.yml
+      vars:
+        snapshot_prefix: "opensearch_snapshot"
+        dirs_to_archive:
+          - "{{ opensearch_snapshot_repository_location }}/"
+
+    - name: Create snapshot checksum
+      import_tasks: common/create_snapshot_checksum.yml
+
+    - name: Transfer artifacts via rsync
+      import_tasks: common/download_via_rsync.yml
+      vars:
+        artifacts:
+          - "{{ snapshot_path }}"
+          - "{{ snapshot_path }}.sha1"

ansible/playbooks/roles/elasticsearch_curator/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 - name: Include installation task
-  include_tasks: install-es-curator-{{ ansible_os_family }}.yml
+  include_tasks: install-ops-curator-{{ ansible_os_family }}.yml
 
 - name: Include configuration tasks
-  include_tasks: configure-cron-jobs.yml
+  include_tasks: configure-cron-jobs.yml

ansible/playbooks/roles/filebeat/tasks/configure-filebeat.yml

@@ -7,7 +7,7 @@
     name: postgresql_defaults
   when: "'postgresql' in group_names"
 
-# Do not select Kibana configured to use ES deployed by 'opendistro_for_elasticsearch' role
+# Do not select OpenSearch Dashboards configured host to use OpenSearch deployed by 'opensearch' role
 - name: Set value for setup.kibana.host
   set_fact:
     setup_kibana_host: >-