Remove cert generation during upgrade process (#2174)

* Remove cert generation during upgrade process * Added new line * Added including certificate role * Revert jmx var * Remove including vars * Add default cert and key for post upgrade tasks Co-authored-by: Robert Pudlowski <[email protected]> Co-authored-by: Robert Pudłowski <[email protected]>
hitachienergy · Apr 1, 2021 · f9a623a · f9a623a
1 parent a48e3ae
commit f9a623a
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 30 deletions.
diff --git a/CHANGELOG-0.10.md b/CHANGELOG-0.10.md
@@ -29,6 +29,7 @@
 - [#2136](https://github.com/epiphany-platform/epiphany/issues/2136) - Replace Bintray repository
 - [#2161](https://github.com/epiphany-platform/epiphany/issues/2161) - Running out of disk space during upgrade from v0.6 and v0.7 where the default disks are 32GB
 - [#1844](https://github.com/epiphany-platform/epiphany/issues/1844) - Default password in Open Distro for Elasticsearch, Filebeat, Kibana and Grafana
+- [#2128](https://github.com/epiphany-platform/epiphany/issues/2128) - [upgrade] Configuration in elasticsearch.yml is overwritten
 
 ### Updated
 
@@ -43,6 +44,6 @@
 - [#2142](https://github.com/epiphany-platform/epiphany/issues/2142) - Update Ubuntu 18.04-LTS images to the latest version
 
 ### Breaking changes
-- Feature name: `elasticsearch` removed in favor of feature name: `opendistro-for-elasticsearch`.
+- Feature `elasticsearch` removed in favor of `opendistro-for-elasticsearch`.
 
 ### Known issues
diff --git a/...i/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml b/...i/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml
@@ -41,18 +41,18 @@
   register: change_config
   vars:
     root_ca_cert_filename: >-
-      {{ existing_es_config.certs.root_ca.cert.filename if (is_upgrade_run and not existing_es_config.certs.root_ca.is_demo_cert) else
+      {{ existing_es_config.certs.root_ca.cert.filename if is_upgrade_run else
          certificates.files.root_ca.cert.filename }}
     node_cert_filename: >-
-      {{ existing_es_config.certs.node.cert.filename if (is_upgrade_run and not existing_es_config.certs.root_ca.is_demo_cert) else
+      {{ existing_es_config.certs.node.cert.filename if is_upgrade_run else
          certificates.files.node.cert.filename }}
     node_key_filename: >-
-      {{ existing_es_config.certs.node.key.filename if (is_upgrade_run and not existing_es_config.certs.root_ca.is_demo_cert) else
+      {{ existing_es_config.certs.node.key.filename if is_upgrade_run else
          certificates.files.node.key.filename }}
     _epiphany_admin_dn: "CN={{ certificates.files.admin.cert.subject.CN }},OU={{ certificates.files.admin.cert.subject.OU }},\
                          O={{ certificates.files.admin.cert.subject.O }}"  # order matters
     admin_dn: >-
-      {{ existing_es_config.certs.admin_dn_without_demo_user if (is_upgrade_run and existing_es_config.certs.admin_dn_without_demo_user|count > 0) else
+      {{ existing_es_config.certs.admin_dn if is_upgrade_run else
          [ _epiphany_admin_dn ] }}
     _epiphany_nodes_dn: >-
       {%- if specification.clustered -%}
@@ -66,7 +66,7 @@
         {%- if not loop.last -%},{%- else -%}]{%- endif -%}
       {%- endfor -%}
     nodes_dn: >-
-      {{ existing_es_config.certs.nodes_dn if (is_upgrade_run and not existing_es_config.certs.root_ca.is_demo_cert) else
+      {{ existing_es_config.certs.nodes_dn if is_upgrade_run else
          _epiphany_nodes_dn }}
 
 - name: Restart elasticsearch service

diff --git a/...ible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch.yml b/...ible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch.yml
@@ -47,17 +47,10 @@
     path: /etc/elasticsearch/{{ _existing_es_config['opendistro_security.ssl.transport.pemtrustedcas_filepath'] }}
   register: _root_ca_info
 
-- name: ODFE | Check if demo certificates are in use
+- name: ODFE | Check if demo or epiphany certificates are in use
   set_fact:
     _is_demo_cert_in_use: "{{ 'True' if _root_ca_info.subject.commonName == 'Example Com Inc. Root CA' else 'False' }}"
-
-- name: ODFE | Include tasks that generete Epiphany certificates  # they may be renewed
-  include_role:
-    name: opendistro_for_elasticsearch
-    tasks_from: generate-certs.yml
-  register: generate_certs
-  when: _is_demo_cert_in_use
-     or _root_ca_info.subject.commonName == 'Epiphany Managed ODFE Root CA'
+    _is_epiphany_cert_in_use: "{{ 'True' if _root_ca_info.subject.commonName == 'Epiphany Managed ODFE Root CA' else 'False' }}"
 
 - name: ODFE | Set API endpoint
   set_fact:
@@ -68,24 +61,72 @@
       body_format: json
       validate_certs: false
 
-# TODO: Implement case for custom admin cert (non-demo and non-epiphany) when not '_is_demo_cert_in_use'.
+# TODO: Implement case for custom admin cert (non-demo and non-epiphany) when not '_is_epiphany_cert_in_use'.
 # For now we use workaround for this (upgrade_config.custom_admin_certificate).
 - name: ODFE | Set API access facts
   set_fact:
     uri_client_cert: >-
-      {{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join if (generate_certs is not skipped) else
+      {{ (certificates.dirs.certs, certificates.files.admin.cert.filename) | path_join if _is_epiphany_cert_in_use else
          upgrade_config.custom_admin_certificate.cert_path }}
     uri_client_key: >-
-      {{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join if (generate_certs is not skipped) else
+      {{ (certificates.dirs.certs, certificates.files.admin.key.filename) | path_join if _is_epiphany_cert_in_use else
          upgrade_config.custom_admin_certificate.key_path }}
-  when: generate_certs is not skipped
+  when: _is_epiphany_cert_in_use
      or upgrade_config.custom_admin_certificate.active
 
+# For clustered nodes it's recommended to disable shard allocation for the cluster before restarting node (https://www.elastic.co/guide/en/elasticsearch/reference/current/restart-cluster.html#restart-cluster-rolling).
+
+- name: ODFE | Test API access using custom certificate
+  when: not _is_demo_cert_in_use
+  uri:
+    <<: *uri
+    client_cert: "{{ uri_client_cert | default(omit) }}"
+    client_key:  "{{ uri_client_key  | default(omit) }}"
+    url: "{{ elasticsearch_endpoint }}"
+    method: GET
+  register: test_api_access_using_custom_cert
+  failed_when: false
+  until: test_api_access_using_custom_cert.status == 200
+  retries: 3
+  delay: 1
+
+- name: ODFE | Test API access using demo certificate
+  when: _is_demo_cert_in_use
+  uri:
+    <<: *uri
+    client_cert: /etc/elasticsearch/kirk.pem
+    client_key: /etc/elasticsearch/kirk-key.pem
+    url: "{{ elasticsearch_endpoint }}"
+    method: GET
+  register: test_api_access_using_demo_cert
+  failed_when: false
+  until: test_api_access_using_demo_cert.status == 200
+  retries: 3
+  delay: 1
+
+- name: Fail with informative message
+  fail:
+    msg:
+      - Accessing API at {{ elasticsearch_endpoint }} failed.
+      - It looks like you use custom certificates.
+      - Please refer to "Open Distro for Elasticsearch upgrade" section of How-To docs.
+  when:
+    - (test_api_access_using_custom_cert is not skipped and test_api_access_using_custom_cert.status != 200) or
+      (test_api_access_using_demo_cert is not skipped and test_api_access_using_demo_cert.status != 200)
+
 - name: Pre-upgrade tasks
+  when: (test_api_access_using_custom_cert is not skipped and test_api_access_using_custom_cert.status == 200) or
+        (test_api_access_using_demo_cert is not skipped and test_api_access_using_demo_cert.status == 200)
   module_defaults:
     uri:
-      client_cert: "{{ uri_client_cert | default(omit) }}"
-      client_key:  "{{ uri_client_key  | default(omit) }}"
+      client_cert: >-
+        {{ uri_client_cert | default(omit) if (test_api_access_using_demo_cert is skipped
+                                               or test_api_access_using_demo_cert.status != 200) else
+           '/etc/elasticsearch/kirk.pem' }}
+      client_key: >-
+        {{ uri_client_key | default(omit) if (test_api_access_using_demo_cert is skipped
+                                              or test_api_access_using_demo_cert.status != 200) else
+           '/etc/elasticsearch/kirk-key.pem' }}
   block:
     - name: ODFE | Disable shard allocation for the cluster
       uri:
@@ -95,8 +136,8 @@
         body: '{"persistent":{"cluster.routing.allocation.enable": "primaries"}}'
       register: response_allocation_primaries
       until: response_allocation_primaries.json.acknowledged is defined and response_allocation_primaries.json.acknowledged == true
-      retries: 10
-      delay: 20
+      retries: 20
+      delay: 10
 
     - name: ODFE | Perform a synced flush
       uri:
@@ -127,25 +168,25 @@
       clustered: "{{ 'True' if _existing_es_config['discovery.seed_hosts'] | length > 1 else 'False' }}"
       paths:
         data: "{{ _existing_es_config['path.data'] }}"
-        repo: "{{ _existing_es_config['path.repo'] }}"
+        repo: "{{ _existing_es_config['path.repo'] | default('/var/lib/elasticsearch-snapshots') }}"  # absent in Epiphany v0.6 thus we use default
         logs: "{{ _existing_es_config['path.logs'] }}"
       opendistro_security:
         ssl:
           transport:
             enforce_hostname_verification: "{{ _existing_es_config['opendistro_security.ssl.transport.enforce_hostname_verification'] }}"
     existing_es_config:
       certs:
-        admin_dn_without_demo_user: >-
-          {{ _existing_es_config['opendistro_security.authcz.admin_dn'] | reject('search', 'CN=kirk,OU=client,O=client,L=test') | list }}
+        admin_dn: >-
+          {{ _existing_es_config['opendistro_security.authcz.admin_dn'] | list }}
         node:
           cert:
             filename: "{{ _existing_es_config['opendistro_security.ssl.transport.pemcert_filepath'] }}"
           key:
             filename: "{{ _existing_es_config['opendistro_security.ssl.transport.pemkey_filepath'] }}"
         nodes_dn: "{{ _existing_es_config['opendistro_security.nodes_dn'] | default([]) }}"
         root_ca:
-          filename: "{{ _existing_es_config['opendistro_security.ssl.transport.pemtrustedcas_filepath'] }}"
-          is_demo_cert: "{{ _is_demo_cert_in_use }}"
+          cert:
+            filename: "{{ _existing_es_config['opendistro_security.ssl.transport.pemtrustedcas_filepath'] }}"
 
 - name: ODFE | Include upgrade plugins tasks
   include_tasks: opendistro_for_elasticsearch/upgrade-plugins.yml
@@ -171,8 +212,14 @@
 - name: Post upgrade tasks
   module_defaults:
     uri:
-      client_cert: "{{ uri_client_cert | default(omit) }}"
-      client_key:  "{{ uri_client_key  | default(omit) }}"
+      client_cert: >-
+        {{ uri_client_cert | default(omit) if (test_api_access_using_demo_cert is skipped
+                                               or test_api_access_using_demo_cert.status != 200) else
+           '/etc/elasticsearch/kirk.pem' }}
+      client_key: >-
+        {{ uri_client_key | default(omit) if (test_api_access_using_demo_cert is skipped
+                                              or test_api_access_using_demo_cert.status != 200) else
+           '/etc/elasticsearch/kirk-key.pem' }}
   block:
     - name: ODFE | Wait for 'yellow' or 'green' cluster health status
       uri: