Merge pull request IDR#405 from khaledk2/upgrade_8_secure

Update to Elasticsearch 8 and secure the connection between searchengine and elasticsearch

ansible/group_vars/searchengine-hosts.yml

@@ -3,16 +3,19 @@ apps_folder: /data
 database_port: 5432
 database_name: idr
 database_username: omeroreadonly
-database_user_password:  "{{ idr_secret_postgresql_password_ro | default('omero') }}"
+database_user_password: "{{ idr_secret_postgresql_password_ro | default('omero') }}"
 searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
-search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
-searchengine_docker_image: openmicroscopy/omero-searchengine:0.5.2
+search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
+searchengine_docker_image: openmicroscopy/omero-searchengine:0.5.3
 #ansible_python_interpreter: path/to/bin/python
 searchengine_index: searchengine_index
 cache_rows: 100000
 # I think that the following two variables should be in secret
-searchengine_secret_key: "fagfdssf3fgdnvhg56ghhgfhgfgh45f"
+searchengine_secret_key: "{{ idr_searchengine_secret_key | default('secret_key') }}"
 searchengineurlprefix: "searchengine"
 IDR_TEST_FILE_URL: "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv"
 elasticsearch_no_nodes: 3
 elasticsearch_backup_folder: "/searchengine_backup"
+ca_password: "{{ idr_secret_elastic_ca_password | default('ca_password') }}"
+keystore_password: "{{ idr_secret_elastic_keystore_password | default('keystore_password') }}"
+ELASTIC_PASSWORD: "{{ idr_secret_elastic_password | default('elastic_password') }}"

ansible/idr-elasticsearch.yml

@@ -4,6 +4,8 @@
   hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
   vars:
     elasticsearch_nodes: [ ]
+    instances_nodes: []
+    instances: {}
 
   tasks:
 
@@ -27,6 +29,17 @@
       group: root
       mode: 0755
 
+  - name: Create elasticsearch certs folder
+    become: yes
+    file:
+      path: "{{ apps_folder }}/searchengine/elasticsearch/certs"
+      state: directory
+      # User id in elasticsearch Docker image
+      owner: 1000
+      group: root
+      mode: 0755
+
+
   - name: Create elasticsearch main nodes directories
     become: yes
     file:
@@ -70,6 +83,20 @@
       group: root
       mode: 0755
 
+
+
+  - name: Add elastic nodes to instances_nodes
+    set_fact:
+      instances_nodes: "{{instances_nodes + [( {'name' : 'searchengine_elasticsearch_node'+item, 'dns': ['searchengine_elasticsearch_node'+item,'localhost'],'ip': '127.0.0.1'})] }}"
+
+    with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+
+
+  - name: Add elastic nodes to instances
+    set_fact:
+      instances: "{{instances | combine ( {'instances' : instances_nodes}) }}"
+
+
   #Add all elasticsearch nodes
   - name: Add elastic nodes to elasticsearch_nodes
     set_fact:
@@ -92,6 +119,56 @@
     with_items:
       - { varname: "vm.max_map_count", varvalue: "262144" }
 
+  - name: create CA
+    become: yes
+    docker_container:
+      image: "{{ search_engineelasticsearch_docker_image }}"
+      name: "creat_ca_for_elasticsearch_cluster"
+      cleanup: True
+      command:  "bash -c '/usr/share/elasticsearch/bin/elasticsearch-certutil ca -s  -out /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --pass {{ ca_password }}
+         '"
+          #fi;
+      state: started
+      volumes:
+          - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
+          - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
+
+  - name: Wait for CA file
+    ansible.builtin.wait_for:
+      path: "{{ apps_folder }}/searchengine/elasticsearch/certs/elastic-ca.p12"
+      state: present
+    register: check_ca_file_result
+
+  - name: Show wait CA file result
+    debug:
+      var: check_ca_file_result
+
+  - copy:
+        dest: /tmp/instances.yaml
+        content: |
+          {{ instances   |to_nice_yaml }}
+
+
+
+  - name: Create nodes' cert
+    become: yes
+    docker_container:
+      image: "{{ search_engineelasticsearch_docker_image }}"
+      name: "creat_ca_for_elasticsearch_cluster"
+      cleanup: True
+      command:  "bash -c 'bin/elasticsearch-certutil cert --ca /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --ca-pass {{ ca_password }}  --pass {{ keystore_password }}  --in /tmp/instances.yaml -out /usr/share/elasticsearch/config/certificates/bundle.zip;
+      echo 'done';
+      unzip /usr/share/elasticsearch/config/certificates/bundle.zip -d /usr/share/elasticsearch/config/certificates/;
+         '"
+
+      state: started
+      volumes:
+          - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
+          - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
+          - /tmp/instances.yaml:/tmp/instances.yaml
+
+
+
   - name: Run first docker elasticsearch main node
     become: yes
     docker_container:
@@ -107,14 +184,34 @@
         node.name: searchengine_elasticsearch_node1
         bootstrap.memory_lock: "true"
         network.host: 0.0.0.0
-        cluster.name: searchengine-cluster
+        cluster.name: "searchengine-cluster"
         cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}"
         http.host: 0.0.0.0
         #http.port: 9200
         ES_JAVA_OPTS: "-Xms2g -Xmx2g"
         ingest.geoip.downloader.enabled: "false"
+        ########################################
+        es_api_basic_auth_username: "elastic"
+        ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}"
+        es_validate_certs: "no"
+        es_enable_http_ssl: "true"
+        xpack.security.http.ssl.enabled: "true"
+        xpack.security.enabled: "true"
+        xpack.security.authc.realms.file.file1.order: "0"
+        xpack.security.authc.realms.native.native1.order: "1"
+        xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12"
+        xpack.security.http.ssl.truststore.password: "{{ ca_password }}"
+        xpack.security.http.ssl.keystore.password: "{{ ca_password }}"
+        xpack.security.transport.ssl.enabled: "true"
+        xpack.security.transport.ssl.verification_mode: "certificate"
+        xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12"
+        xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12"
+        xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}"
+        xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}"
+      #########################################################################
       networks:
       - name: "searchengine-net"
+        ipv4_address: 10.11.0.2
       published_ports:
         - "9201:9200"
         - "9301:9300"
@@ -123,8 +220,10 @@
       volumes:
       - "{{ apps_folder }}/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch"
       - "{{ apps_folder }}/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch"
+      - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
       - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}"
 
+
   - name: Run docker elasticsearch for the remaining nodes
     become: yes
     docker_container:
@@ -147,9 +246,29 @@
         #http.port: 9200
         ES_JAVA_OPTS: "-Xms1g -Xmx1g"
         ingest.geoip.downloader.enabled: "false"
+        ####################################################################
+        es_api_basic_auth_username: "elastic"
+        ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}"
+        es_validate_certs: "no"
+        es_enable_http_ssl: "true"
+        xpack.security.http.ssl.enabled: "true"
+        xpack.security.enabled: "true"
+        xpack.security.authc.realms.file.file1.order: "0"
+        xpack.security.authc.realms.native.native1.order: "1"
+        xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12"
+        xpack.security.http.ssl.truststore.password: "{{ ca_password }}"
+        xpack.security.http.ssl.keystore.password: "{{ ca_password }}"
+        xpack.security.transport.ssl.enabled: "true"
+        xpack.security.transport.ssl.verification_mode: "certificate"
+        xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12"
+        xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12"
+        xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}"
+        xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}"
+        ####################################################################
 
       networks:
       - name: "searchengine-net"
+        ipv4_address: 10.11.0.{{ item | int + 1 }}
       published_ports:
         - "920{{ item }}:9200"
         - "930{{ item }}:9300"
@@ -159,4 +278,5 @@
       - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data:/var/lib/elasticsearch"
       - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs:/var/log/elasticsearch"
       - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}"
+      - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
     with_sequence: start=2 count={{ elasticsearch_no_nodes | int -1 }}

ansible/idr-searchengine.yml

@@ -5,7 +5,7 @@
 - name: Deploying search engine
   hosts: "{{ idr_environment | default('idr') }}-searchengine-hosts"
   vars:
-    elasticsearch_nodes: [ ]
+    elasticsearch_nodes_urls: [ ]
 
   tasks:
   - name: Get database host
@@ -41,10 +41,10 @@
       mode: 0755
 
   #Add all elasticsearch nodes
-  - name: Add elastic nodes to elasticsearch_nodes
+  - name: Add elastic nodes to elasticsearch_nodes_urls
     set_fact:
-      elasticsearch_nodes: '{{ elasticsearch_nodes + ["searchengine_elasticsearch_node"+item] }}'
-    with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+      elasticsearch_nodes_urls: '{{ elasticsearch_nodes_urls + ["https://10.11.0."+item+":9200"] }}'
+    with_sequence: start=2 count={{ elasticsearch_no_nodes }}
 
   - name: configure elasticsearch backup folder for docker searchengine
     become: yes
@@ -63,7 +63,7 @@
       image: "{{ searchengine_docker_image }}"
       name: searchengine_elasticsearch
       cleanup: True
-      command: "set_elasticsearch_configuration -e {{ elasticsearch_nodes | join(',') }}"
+      command: "set_elasticsearch_configuration -e {{ elasticsearch_nodes_urls | join(',') }}"
       state: started
       volumes:
         - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
@@ -139,6 +139,18 @@
       volumes:
       - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
 
+  - name: configure elastics search password
+    become: yes
+    docker_container:
+      image: "{{ searchengine_docker_image }}"
+      name: elastics_password
+      cleanup: True
+      #auto_remove: yes
+      command: "set_elasticsearch_password -e {{ ELASTIC_PASSWORD }}"
+      state: started
+      volumes:
+        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
   - name: Run docker searchengine
     become: yes
     docker_container: