Configure Ignite to use fixed ports (#1208)

* Start ignite.service after network.target

* Install Ignite as Anisble block

* Reserve ports from ephemeral range

* Override function findAvailableJmxPort

* Limit number of potentially used ports

core/src/epicli/data/common/ansible/playbooks/roles/ignite/defaults/main.yml

@@ -0,0 +1,33 @@
+---
+ignite_ports_from_ephemeral_range: # strings are required
+  - '47100-47109'
+  - '47500-47509'
+  - '49112'
+
+# The following block is appended to $IGNITE_HOME/bin/include/functions.sh
+block_to_append_to_ignite_functions_script: |
+  #
+  # The function exports JMX_MON variable with Java JMX options.
+  # Overrides original version in order to:
+  # 1) Use fixed ports for JMX (https://github.com/epiphany-platform/epiphany/issues/1181)
+  # 2) Disable direct remote access to unsecured JMX (remote access possible through SSH tunnel)
+  #
+  findAvailableJmxPort() {
+      export IGNITE_JMX_PORT=49112
+
+      JMX_PORT=`"$JAVA" -cp "${IGNITE_LIBS}" org.apache.ignite.internal.util.portscanner.GridJmxPortFinder`
+
+      #
+      # This variable defines parameters for JMX monitoring and management.
+      #
+      if [ -n "$JMX_PORT" ]; then
+          # java.rmi.server.hostname=127.0.0.1 is used to make JMX accessible through SSH tunnel
+          JMX_MON="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${JMX_PORT} \
+              -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false \
+              -Dcom.sun.management.jmxremote.rmi.port=${JMX_PORT} -Djava.rmi.server.hostname=127.0.0.1"
+      else
+          # If JMX port wasn't found do not initialize JMX.
+          echo "$0, WARN: Failed to resolve JMX host (JMX will be disabled): $HOSTNAME"
+          JMX_MON=""
+      fi
+  }

core/src/epicli/data/common/ansible/playbooks/roles/ignite/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-# tasks file for ignite
 - name: Add ignite group
   become: yes
   group:
@@ -27,66 +26,100 @@
   changed_when: false
 
 - name: Set JAVA_HOME environment variable for all users
-  lineinfile: 
-    path: /etc/environment 
-    state: present 
-    regexp: '^JAVA_HOME=' 
+  lineinfile:
+    path: /etc/environment
+    state: present
+    regexp: '^JAVA_HOME='
     line: JAVA_HOME="{{ java_home_location.stdout }}"
 
-- name: Set Apache Ignite file name to install
-  set_fact:
-    ignite_file_name: "{{ specification.file_name }}"
-
 - name: Check if Ignite in current version exists
   stat:
-    path: /opt/ignite_{{ specification.version }}/bin/ignite.sh
-  register: ignite_exists
+    path: /opt/ignite # symlink
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ignite_dir_stat
+
+- name: Install Ignite binaries
+  when: ignite_dir_stat.stat.lnk_source is not defined
+     or ignite_dir_stat.stat.lnk_source != ignite_dest_path
+  block:
+    - name: Download Ignite binaries
+      include_role:
+        name: download
+        tasks_from: download_file
+      vars:
+        file_name: "{{ ignite_file_name }}"
+
+    - name: Create temp directory
+      file:
+        path: /tmp/ignite_{{ specification.version }}
+        state: directory
+        owner: ignite
+        group: ignite
+
+    - name: Uncompress {{ ignite_file_name }} to temp directory
+      unarchive:
+        remote_src: yes
+        src: "{{ download_directory }}/{{ ignite_file_name }}"
+        dest: /tmp/ignite_{{ specification.version }}
+        owner: ignite
+        group: ignite
+        list_files: yes
+      register: archive_contents
+
+    - name: Create {{ ignite_dest_path }} directory
+      file:
+        path: "{{ ignite_dest_path }}"
+        state: directory
+        owner: ignite
+        group: ignite
+
+    - name: Copy Ignite files from tmp location
+      copy:
+        remote_src: yes
+        src: "/tmp/ignite_{{ specification.version }}/{{ archive_contents.files[0].split('/')[0] }}/"
+        dest: "{{ ignite_dest_path }}"
+
+    - name: Link /opt/ignite to the right version
+      file:
+        src: "{{ ignite_dest_path }}"
+        dest: /opt/ignite
+        state: link
+      register: link_ignite_version
 
-- name: Download Ignite binaries
-  include_role:
-    name: download
-    tasks_from: download_file
   vars:
-    file_name: "{{ ignite_file_name }}"
-  when: not ignite_exists.stat.exists
-
-- name: Create temp directory
-  file:
-    path: /tmp/ignite_{{ specification.version }}
-    state: directory
-    owner: ignite
-    group: ignite
-
-- name: Uncompress {{ ignite_file_name }} to temp directory 
-  unarchive:
-    remote_src: yes
-    src: "{{ download_directory }}/{{ ignite_file_name }}"
-    dest: /tmp/ignite_{{ specification.version }}
-    owner: ignite
-    group: ignite
-    list_files: yes
-  register: archive_contents
-  when: not ignite_exists.stat.exists
-
-- name: Create /opt/ignite_{{ specification.version }} directory
-  file:
-    path: /opt/ignite_{{ specification.version }}
-    state: directory
-    owner: ignite
-    group: ignite
-
-- name: Copy Ignite files from tmp location
-  copy:
-    remote_src: yes
-    src: "/tmp/ignite_{{ specification.version }}/{{ archive_contents.files[0].split('/')[0] }}/"
-    dest: /opt/ignite_{{ specification.version }}
-  when: not ignite_exists.stat.exists
-
-- name: Link /opt/ignite to the right version
-  file:
-    dest: /opt/ignite
-    state: link
-    src: /opt/ignite_{{ specification.version }}
+    ignite_file_name: "{{ specification.file_name }}"
+    ignite_dest_path: /opt/ignite_{{ specification.version }}
+
+- name: Reserve ports from ephemeral range
+  when: link_ignite_version.changed
+  block:
+    - name: Get net.ipv4.ip_local_reserved_ports
+      command: cat /proc/sys/net/ipv4/ip_local_reserved_ports
+      register: ip_local_reserved_ports
+      changed_when: false
+
+    - name: Reserve ports from ephemeral range
+      sysctl:
+        name: net.ipv4.ip_local_reserved_ports
+        value: "{{ (reserved_ports + ignite_ports_from_ephemeral_range) | sort | unique | join(',') }}"
+        sysctl_set: yes
+        state: present
+        reload: yes
+      vars:
+        reserved_ports: "{{ ip_local_reserved_ports.stdout.split(',') | reject('equalto', '') | list }}"
+
+- name: Append block to /opt/ignite/bin/include/functions.sh
+  blockinfile:
+    path: /opt/ignite/bin/include/functions.sh
+    marker: "# {mark} ANSIBLE MANAGED BLOCK"
+    insertafter: EOF
+    backup: yes
+    block: "{{ block_to_append_to_ignite_functions_script }}"
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
 
 - name: Copy Ignite enabled plugins
   copy:
@@ -132,7 +165,7 @@
 
 - name: Restart Ignite service
   become: yes
-  systemd: 
-    name: ignite 
-    state: restarted 
+  systemd:
+    name: ignite
+    state: restarted
   when: plugins_installed.changed or ignite_configuration_created.changed

core/src/epicli/data/common/ansible/playbooks/roles/ignite/templates/ignite.service.j2

@@ -1,5 +1,6 @@
 [Unit]
 Description=Apache Ignite Server
+After=network.target
 
 [Service]
 Type=simple

core/src/epicli/data/common/defaults/configuration/ignite.yml

@@ -54,7 +54,7 @@ specification:
               <property name="walArchivePath" value="/wal/archive"/>
             </bean>
           </property>
-        
+
           <property name="discoverySpi">
             <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
               <property name="ipFinder">
@@ -64,7 +64,35 @@ specification:
                   </property>
                 </bean>
               </property>
+              <property name="localPort" value="47500"/>
+              <!-- Limit number of potentially used ports from 100 to 10 -->
+              <property name="localPortRange" value="10"/>
+            </bean>
+          </property>
+
+          <property name="communicationSpi">
+            <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+              <property name="localPort" value="47100"/>
+              <!-- Limit number of potentially used ports from 100 to 10 -->
+              <property name="localPortRange" value="10"/>
             </bean>
           </property>
+
+          <property name="clientConnectorConfiguration">
+            <bean class="org.apache.ignite.configuration.ClientConnectorConfiguration">
+              <property name="port" value="10800"/>
+              <!-- Limit number of potentially used ports from 100 to 10 -->
+              <property name="portRange" value="10"/>
+            </bean>
+          </property>
+
+          <property name="connectorConfiguration">
+            <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
+              <property name="port" value="11211"/>
+              <!-- Limit number of potentially used ports from 100 to 10 -->
+              <property name="portRange" value="10"/>
+            </bean>
+          </property>
+
         </bean>
     </beans>