OPSEXP-1803 Add Search Enteprise reindex service

.ansible-lint

@@ -1,6 +1,5 @@
 warn_list:
   - experimental
-  - deprecations
 skip_list:
   - 'risky-file-permissions'
   - 'no-changed-when'
@@ -11,7 +10,8 @@ skip_list:
   - 'no-tabs'
   - 'empty-string-compare'
   - 'fqcn-builtins'
-
+  # Allow Jinja templating inside task and play names
+  - name[template]
 kinds:
   - yaml: "**/molecule/**/vars*.yml"
   - yaml: "**/molecule/**/hosts.yml"

.github/workflows/enteprise.yml

@@ -13,6 +13,7 @@ env:
 
 jobs:
   docker:
+    name: ${{ matrix.role.name }} on ${{ matrix.molecule_distro.image }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
@@ -62,6 +63,7 @@ jobs:
           command: cd roles/${{ matrix.role.name }} && pipenv run molecule test
 
   docker_integration:
+    name: ${{ matrix.scenario.name }} on ${{ matrix.molecule_distro.image }}
     if: github.actor != 'dependabot[bot]'
     needs:
       - docker
@@ -70,7 +72,6 @@ jobs:
       fail-fast: false
       matrix:
         molecule_distro:
-          - image: centos:7
           - image: ubuntu:20.04
           - image: rockylinux:8
         scenario:
@@ -108,10 +109,10 @@ jobs:
         run: pipenv run molecule test -s ${{ matrix.scenario.name }}
 
   ec2:
+    name: ${{ matrix.molecule_scenario.desc }}
     if: github.actor != 'dependabot[bot]'
     needs:
       - docker
-    name: ${{ matrix.molecule_scenario.desc }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/Riverside-Healthcare/djLint
-    rev: v1.18.0
+    rev: v1.19.3
     hooks:
       - id: djlint-jinja
 

docs/deployment-guide.md

@@ -97,19 +97,20 @@ Regardless of role and connection type, a consistent folder structure is used. Y
 
 The following systemd services are deployed and can be used to stop and start Alfresco components:
 
-| Service Name                        | Purpose                                                                          |
-|:------------------------------------|:---------------------------------------------------------------------------------|
-| `activemq.service`                  | ActiveMQ Service                                                                 |
-| `postgresql-<version>.service`      | Postgresql DB Service (where `<version>` is 11 for ACS 6.2.N and 13 for ACS 7.x) |
-| `nginx.service`                     | Nginx Service                                                                    |
-| `alfresco-content.service`          | Alfresco Content Service                                                         |
-| `alfresco-search.service`           | Alfresco Search Service                                                          |
-| `alfresco-shared-fs.service`        | Alfresco Shared File Store Controller Service                                    |
-| `alfresco-sync.service`             | Alfresco Sync Service                                                            |
-| `alfresco-tengine-aio.service`      | Alfresco AIO Transform Core Engine                                               |
-| `alfresco-transform-router.service` | Alfresco Transformation Router Service                                           |
-| `elasticsearch-connector.service`   | Alfresco Search Enterprise Service                                               |
-| `elasticsearch.service`             | ElasticSearch Service                                                            |
+| Service Name                              | Purpose                                                                                 |
+|:------------------------------------------|:----------------------------------------------------------------------------------------|
+| `activemq.service`                        | ActiveMQ Service                                                                        |
+| `postgresql-<version>.service`            | Postgresql DB Service (where `<version>` is 11 for ACS 6.2.N and 13 for ACS 7.x)        |
+| `nginx.service`                           | Nginx Service                                                                           |
+| `alfresco-content.service`                | Alfresco Content Service                                                                |
+| `alfresco-search.service`                 | Alfresco Search Service                                                                 |
+| `alfresco-shared-fs.service`              | Alfresco Shared File Store Controller Service                                           |
+| `alfresco-sync.service`                   | Alfresco Sync Service                                                                   |
+| `alfresco-tengine-aio.service`            | Alfresco AIO Transform Core Engine                                                      |
+| `alfresco-transform-router.service`       | Alfresco Transformation Router Service                                                  |
+| `elasticsearch-connector.service`         | Alfresco Search Enterprise Service                                                      |
+| `elasticsearch-connector-reindex.service` | Alfresco Search Enterprise job to force the reindexing of all the contents of the store |
+| `elasticsearch.service`                   | ElasticSearch Service                                                                   |
 
 Please be aware that some configuration changes (e.g. postgres pg_hba,
 properties files, ...) can trigger a service restart and a consequent
@@ -469,7 +470,7 @@ all:
         whatever.mq.eu-west-1.amazonaws.com:
           activemq_username: alfresco
           activemq_port: 61617
-          activemq_protocol: tcp # or ssl
+          activemq_transport: tcp # or ssl
     external:
       children:
         external_activemq:
@@ -794,11 +795,11 @@ Once ACS has initialized access the system using the following URLs with a brows
 
 There are some useful argument you can use with `ansible-playbook` command in many circumstances. Some are highlighted bellow but take a look at [The ansible-playbook documentation](https://docs.ansible.com/ansible/latest/cli/ansible-playbook.html) for complete list of options.
 
-* `-k` : Prompt for SSH password. Usefull when no SSH keys have been deployed but needs to be th same on all hosts (prefer SSH whenever possible)
-* `-K` : Prompt for sudo password. Usefull when the user used to connect to the machine is not root
+* `-k` : Prompt for SSH password. Useful when no SSH keys have been deployed but needs to be th same on all hosts (prefer SSH whenever possible)
+* `-K` : Prompt for sudo password. Useful when the user used to connect to the machine is not root
 * `-e` : Pass an extra variable or override an existing one (read from file with `-e @file`).
 * `-l` : Limit the play to a subset of hosts (either groups or individuals hosts or a mix of both)
-* `-u user` : specify the username to use to cnnect to all targets (Prefer adding the ànsible_ssh_user` to the inventory file in the right scope, e.g. under the `all`group)
+* `-u user` : specify the username to use to connect to all targets (Prefer adding the `ansible_ssh_user` to the inventory file in the right scope, e.g. under the `all`group)
 
 ## ACS cluster
 
@@ -833,6 +834,18 @@ In that case, you can add the `cluster_keepoff` variable to one of the `reposito
 
 > A typical use case is to have a dedicated Solr tracking node. The playbook will then prefer to use that dedicated node - if it finds one - for solr tracking and only use the other as backup server (no load balancing)
 
+## Maintenance
+
+After the initial deploy, may arise the need to execute maintenance tasks that are handled via specific playbooks.
+
+### Search Enterprise Reindexing
+
+You can trigger the reindexing of existing content in Search Enterprise using a dedicated playbook:
+
+```bash
+pipenv run ansible-playbook playbooks/search-enterprise-reindex.yml -i <inventory_file>.yml
+```
+
 ## Cleanup & Uninstallation of ACS
 
 What needs to be removed from a system will depend on your inventory configuration. The steps below presume a cleanup and uninstallation of Alfresco content service after deployment of ansible artifacts by using platform-cleanup.yml playbook and platform-uninstall.yml playbook respectively.
@@ -842,7 +855,7 @@ What needs to be removed from a system will depend on your inventory configurati
 This playbook will remove the temporary artifacts which are stored on the hosts.In order to cleanup the system post deployment run the following command:
 
 ```bash
-pipenv run ansible-playbook playbooks/platform-cleanup.yml -i inventory_ssh.yml
+pipenv run ansible-playbook playbooks/platform-cleanup.yml -i <inventory_file>.yml
 ```
 
 > Note: This playbook can break the idempotency i.e Downloaded artifacts again needs to removed by running cleanup playbook.
@@ -851,7 +864,7 @@ pipenv run ansible-playbook playbooks/platform-cleanup.yml -i inventory_ssh.yml
 
 This playbook will uninstall the sevices which belong to the specific hosts. Below are the services, packages & folders we are removing when uninstalling
 
-1. Stoping and removing the following services:
+1. Stopping and removing the following services:
    * alfresco-transform-router.service
    * alfresco-shared-fs.service
    * alfresco-tengine-aio.service

molecule/elasticsearch/molecule.yml

@@ -43,6 +43,5 @@ provisioner:
       host_vars: host_vars
   playbooks:
     prepare: ../default/prepare.yml
-    verify: ../default/verify.yml
 verifier:
   name: ansible

molecule/elasticsearch/verify.yml

@@ -0,0 +1,54 @@
+- name: Verify
+  hosts: search_enterprise
+  gather_facts: false
+  vars:
+    job_name: elasticsearch-connector.service
+  tasks:
+    - name: Populate service facts
+      ansible.builtin.service_facts:
+
+    - name: Check that Elasticsearch Connector service is up and running
+      ansible.builtin.assert:
+        that:
+          - ansible_facts.services[job_name] is defined
+          - ansible_facts.services[job_name].state == 'running'
+
+    - name: "Check that {{ job_name }} is really up and running"
+      become: true
+      ansible.builtin.shell: journalctl -u {{ job_name }}
+      register: error_log
+      retries: 6
+      delay: 10
+      until: error_log.stdout | regex_search('Started LiveIndexingApp')
+
+- name: Verify Reindex before first run
+  hosts: search_enterprise
+  gather_facts: false
+  vars:
+    job_name: elasticsearch-connector-reindex.service
+  tasks:
+    - name: Check that {{ job_name }} is defined and not running
+      ansible.builtin.assert:
+        that:
+          - ansible_facts.services[job_name] is defined
+          - ansible_facts.services[job_name].state == 'inactive'
+
+- name: Run the Elasticsearch Connector Reindex
+  import_playbook: ../../playbooks/search-enterprise-reindex.yml
+
+- name: Verify Reindex after first run
+  hosts: search_enterprise[0]
+  gather_facts: false
+  vars:
+    job_name: elasticsearch-connector-reindex.service
+  tasks:
+    - name: "Check that {{ job_name }} is really up and running"
+      become: true
+      ansible.builtin.shell: journalctl -u {{ job_name }}
+      register: error_log
+      retries: 6
+      delay: 10
+      until: error_log.stdout | regex_search('reindexByIds.*COMPLETED.*')
+
+- name: Run default verify
+  import_playbook: ../default/verify.yml

molecule/multimachine/prepare.yml

@@ -128,7 +128,7 @@
         mqattrs:
           activemq_username: alfresco
           activemq_port: 61617
-          activemq_protocol: ssl
+          activemq_transport: ssl
       ansible.builtin.template:
         src: hosts.yml.j2
         dest: hosts.yml

playbooks/acs.yml

@@ -186,7 +186,7 @@
         marker_begin: SEARCH_ENTERPRISE_BEGIN
         marker_end: SEARCH_ENTERPRISE_END
   tags:
-    - sfs
+    - search_enterprise
 
 - name: Alfresco Digital Workspace Role
   hosts: adw

playbooks/prerequisite-checks.yml

@@ -16,7 +16,7 @@
     include_tasks: "../roles/helper_modules/tasks/check_port.yml"
     vars:
       checked_host: "{% if groups.activemq | length == 0 %}127.0.0.1{% else %}{{ hostvars[groups.activemq[0]].ansible_host | default('127.0.0.1') }}{% endif %}"
-      checked_port: "{{ ports_cfg.activemq.openwire }}"
+      checked_port: "{{ ports_cfg.activemq[activemq_protocol] }}"
       delegate_target: "{{ groups.activemq | first }}"
   - name: check search connection
     include_tasks: "../roles/helper_modules/tasks/check_port.yml"
@@ -72,7 +72,7 @@
     include_tasks: "../roles/helper_modules/tasks/check_port.yml"
     vars:
       checked_host: "{% if groups.activemq | length == 0 %}127.0.0.1{% else %}{{ hostvars[groups.activemq[0]].ansible_host | default('127.0.0.1') }}{% endif %}"
-      checked_port: "{{ ports_cfg.activemq.openwire }}"
+      checked_port: "{{ ports_cfg.activemq[activemq_protocol] }}"
       delegate_target: "{{ groups.activemq | first }}"
 
 - name: prerequisite checks
@@ -91,7 +91,7 @@
     include_tasks: "../roles/helper_modules/tasks/check_port.yml"
     vars:
       checked_host: "{% if groups.activemq | length == 0 %}127.0.0.1{% else %}{{ hostvars[groups.activemq[0]].ansible_host | default('127.0.0.1') }}{% endif %}"
-      checked_port: "{{ ports_cfg.activemq.openwire }}"
+      checked_port: "{{ ports_cfg.activemq[activemq_protocol] }}"
       delegate_target: "{{ groups.activemq | first }}"
 
 - name: prerequisite checks

playbooks/search-enterprise-reindex.yml

@@ -0,0 +1,9 @@
+- name: Search Enteprise Reindex
+  hosts: search_enterprise[0]
+  gather_facts: false
+  tasks:
+    - name: Run the Elasticsearch Connector Reindex
+      become: true
+      ansible.builtin.systemd:
+        state: started
+        name: elasticsearch-connector-reindex

roles/common/defaults/main.yml

@@ -27,8 +27,9 @@ activemq_host: >-
   {% else %}
   {{- groups.activemq | map('extract', hostvars, ['ansible_default_ipv4','address']) | first -}}
   {% endif %}
-activemq_protocol: >-
-  {{- groups.external_activemq | default([]) | map('extract', hostvars, ['activemq_protocol']) | first | default('tcp') -}}
+activemq_transport: >-
+  {{- groups.external_activemq | default([]) | map('extract', hostvars, ['activemq_transport']) | first | default('tcp') -}}
+activemq_protocol: openwire
 db_host: "{{ groups['database'] | default([]) | map('extract', hostvars, ['ansible_default_ipv4','address']) | first | default('127.0.0.1') }}"
 solr_host: "{{ groups['search'] | default([]) | map('extract', hostvars, ['ansible_default_ipv4','address']) | first | default('127.0.0.1') }}"
 ats_tengine_aio_host: "{{ groups['transformers'] | default([]) | map('extract', hostvars, ['ansible_default_ipv4','address']) | first | default('127.0.0.1') }}"

roles/repository/templates/alfresco-global.properties.j2

@@ -70,7 +70,7 @@ tika.url=${localTransform.core-aio.url}
 transform.misc.url=${localTransform.core-aio.url}
 sfs.url=http://{{ sfs_host }}:{{ ports_cfg.sfs.http }}/
 
-messaging.broker.url=failover:({{ activemq_protocol }}://{{ activemq_host }}:{{ ports_cfg.activemq.openwire }})?timeout=3000
+messaging.broker.url=failover:({{ activemq_transport }}://{{ activemq_host }}:{{ ports_cfg.activemq[activemq_protocol] }})?timeout=3000
 messaging.broker.username={{ activemq_username }}
 messaging.broker.password={{ activemq_password }}
 

roles/search_enterprise/defaults/main.yml

@@ -1,3 +1,5 @@
 ---
 # defaults file for search_enterprise
 search_enterprise_port: 8081
+# see https://docs.alfresco.com/search-enterprise/latest/admin/#alfresco-elasticsearch-connector
+search_enterprise_reindex_options: ''

roles/search_enterprise/molecule/default/host_vars/search-ent-instance.yml

@@ -1 +1,2 @@
 ansible_user: ansible
+search_enterprise_reindex_options: '--alfresco.reindex.concurrentProcessors=6'

roles/search_enterprise/molecule/default/verify.yml

@@ -1,6 +1,6 @@
 ---
 - name: Verify
-  hosts: all
+  hosts: search_enterprise
   gather_facts: false
   tasks:
     - name: Populate service facts
@@ -24,6 +24,28 @@
       ansible.builtin.assert:
         that:
           - not error_log.stdout | regex_search(' WARN ')
-          - not error_log.stderr | regex_search(' ERROR ')
+          - not error_log.stdout | regex_search(' ERROR ')
           - error_log.stdout | regex_search('{{ log_expected_regex }}')
-        msg: "{{ error_log.stdout }} {{ error_log.stderr }}"
+        msg: "{{ error_log.stdout }}"
+
+    - name: Check that Elasticsearch Connector reindex is available
+      ansible.builtin.assert:
+        that:
+          - ansible_facts.services['elasticsearch-connector-reindex.service'] is defined
+          - ansible_facts.services['elasticsearch-connector-reindex.service'].state == 'inactive'
+
+    - name: Retrieve contents of elasticsearch-connector-reindex.service
+      become: true
+      ansible.builtin.slurp:
+        src: /etc/systemd/system/elasticsearch-connector-reindex.service
+      register: service_file
+
+    - name: Check reindex service contains the expected ExecStart line
+      become: true
+      vars:
+        service_file_content: "{{ service_file['content'] | b64decode }}"
+        expected_exec_start: "ExecStart=/opt/openjdk-11.0.15/bin/java -jar /opt/alfresco/enterprise-search-3.1.1/alfresco-elasticsearch-reindexing-3.1.1-app.jar --alfresco.reindex.concurrentProcessors=6"
+      ansible.builtin.assert:
+        that:
+          - "expected_exec_start in service_file_content"
+        msg: "{{ service_file_content }}"

roles/search_enterprise/tasks/main.yml

@@ -8,7 +8,6 @@
     mode: "0644"
     url_username: "{{ nexus_user }}"
     url_password: "{{ nexus_password }}"
-    timeout: 2
 
 - name: Install & configure Search services
   become: true
@@ -34,6 +33,14 @@
         owner: "{{ username }}"
         group: "{{ group_name }}"
 
+    - name: Add elasticsearch-connector reindex service
+      template:
+        src: elasticsearch-connector-reindex.service.j2
+        dest: /etc/systemd/system/elasticsearch-connector-reindex.service
+        owner: root
+        group: root
+        mode: "0640"
+
     - name: Add elasticsearch-connector service
       template:
         src: elasticsearch-connector.service.j2

roles/search_enterprise/templates/elasticsearch-cfg.j2

@@ -0,0 +1,9 @@
+Environment=ALFRESCO_SHAREDFILESTORE_BASEURL=http://{{ sfs_host }}:{{ ports_cfg.sfs.http }}/alfresco/api/-default-/private/sfs/versions/1/file
+Environment=ALFRESCO_ACCEPTED_CONTENT_MEDIA_TYPES_CACHE_BASE_URL=http://{{ trouter_host }}:{{ ports_cfg.transformers.trouter }}/transform/config
+Environment=ELASTICSEARCH_INDEXNAME=alfresco
+Environment=SPRING_ACTIVEMQ_BROKERURL=failover:({{ activemq_transport }}://{{ activemq_host }}:{{ ports_cfg.activemq[activemq_protocol] }})
+Environment=SPRING_ACTIVEMQ_USER={{ activemq_username }}
+Environment=SPRING_ACTIVEMQ_PASSWORD={{ activemq_password }}
+Environment=SPRING_ELASTICSEARCH_REST_URIS={{ elasticsearch_protocol }}://{{ elasticsearch_host }}:{{ ports_cfg.elasticsearch.http }}
+Environment=SPRING_ELASTICSEARCH_REST_USERNAME={{ elasticsearch_username }}
+Environment=SPRING_ELASTICSEARCH_REST_PASSWORD={{ elasticsearch_password }}

roles/search_enterprise/templates/elasticsearch-connector-reindex.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Alfresco Enterprise Search - Reindex job
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=oneshot
+
+User={{ username }}
+
+{% include "elasticsearch-cfg.j2" %}
+
+Environment=SPRING_DATASOURCE_URL=jdbc:postgresql://{{ db_host }}:{{ ports_cfg.postgres.sql }}/{{ repo_db_name }}
+Environment=SPRING_DATASOURCE_USERNAME={{ repo_db_username }}
+Environment=SPRING_DATASOURCE_PASSWORD={{ repo_db_password }}
+
+ExecStart={{ java_home }}/bin/java -jar {{ search_enterprise_binaries_dir }}/alfresco-elasticsearch-reindexing-{{ search_enterprise.version }}-app.jar {{ search_enterprise_reindex_options }}
+
+WorkingDirectory={{ search_enterprise_binaries_dir }}
+
+[Install]
+WantedBy=multi-user.target