elastic 7.x support

- Update JVM startup options
- Add node name to log4j2.properties
- Rename augeas resource to prevent dependency loop
- Add boolean to enable/disable xpack and required features
- Add self to contributors list
- Add check for required certificate variables
- Add version check for xpack related settings
  Maintains backwards compatibility
- Resolve variable scope warnings
- Update unit tests
- Remove ubuntu 14.04 acceptance tests - Not supported
  https://www.elastic.co/support/matrix

.travis.yml

@@ -1,5 +1,5 @@
 sudo: required
-group: deprecated-2017Q4
+group: deprecated-2017q4
 services:
 - docker
 cache:
@@ -11,11 +11,6 @@ language: ruby
 script: travis_retry bundle exec rake $TASK
 jobs:
   allow_failures:
-  - env:
-    - TASK=beaker:ubuntu-server-1404-x64:snapshot
-  - env:
-    - OSS_PACKAGE=true
-    - TASK=beaker:ubuntu-server-1404-x64:snapshot
   - env:
     - TASK=beaker:ubuntu-server-1604-x64:snapshot
   - env:
@@ -97,24 +92,13 @@ jobs:
     - TASK=beaker:debian-9-x64:acceptance[5.6.9]
   - env:
     - TASK=beaker:debian-9-x64:acceptance[2.4.6]
-  - env:
-    - TASK=beaker:ubuntu-server-1404-x64:acceptance
-  - env:
-    - TASK=beaker:ubuntu-server-1404-x64:acceptance[5.6.9]
-  - env:
-    - TASK=beaker:ubuntu-server-1404-x64:acceptance[2.4.6]
   - env:
     - TASK=beaker:ubuntu-server-1604-x64:acceptance
   - env:
     - TASK=beaker:ubuntu-server-1604-x64:acceptance[5.6.9]
   - env:
     - TASK=beaker:ubuntu-server-1604-x64:acceptance[2.4.6]
   - stage: snapshots
-    env:
-    - TASK=beaker:ubuntu-server-1404-x64:snapshot
-    env:
-    - OSS_PACKAGE=true
-    - TASK=beaker:ubuntu-server-1404-x64:snapshot
   - env:
     - TASK=beaker:ubuntu-server-1604-x64:snapshot
   - env:

CONTRIBUTORS

@@ -33,3 +33,4 @@ Matthias Baur (baurmatt)
 Gavin Williams (fatmcgav)
 Giedrius Statkevičius (GiedriusS)
 Jakub Pieńkowski (Jakski)
+Simon Hulme (uberjew666)

data/common.yaml

@@ -65,3 +65,4 @@ elasticsearch::templates: {}
 elasticsearch::users: {}
 elasticsearch::validate_tls: true
 elasticsearch::version: false
+elasticsearch::xpack: false

lib/puppet/type/elasticsearch_keystore.rb

@@ -59,6 +59,6 @@ def change_to_s(currentvalue, newvalue_raw)
   end
 
   autorequire(:augeas) do
-    "defaults_#{self[:name]}"
+    "ks_defaults_#{self[:name]}"
   end
 end

manifests/init.pp

@@ -297,6 +297,9 @@
 # @param version
 #   To set the specific version you want to install.
 #
+# @param xpack
+#   Enable x-pack security. Requires ca_certificate, certificate and private key.
+#
 # @author Richard Pijnenburg <[email protected]>
 # @author Tyler Langlois <[email protected]>
 #
@@ -371,6 +374,7 @@
   Hash                                            $users,
   Boolean                                         $validate_tls,
   Variant[String, Boolean]                        $version,
+  Boolean                                         $xpack,
   Boolean $restart_config_change  = $restart_on_change,
   Boolean $restart_package_change = $restart_on_change,
   Boolean $restart_plugin_change  = $restart_on_change,

manifests/instance.pp

@@ -125,6 +125,9 @@
 #   Source for the Shield system key. Valid values are any that are
 #   supported for the file resource `source` parameter.
 #
+# @param xpack
+#   Enable xpack security features. Requires ca_certificate, certificate and private_key.
+#
 # @author Richard Pijnenburg <[email protected]>
 # @author Tyler Langlois <[email protected]>
 #
@@ -162,6 +165,7 @@
   Boolean                            $ssl                           = false,
   Elasticsearch::Status              $status                        = $elasticsearch::status,
   Optional[String]                   $system_key                    = $elasticsearch::system_key,
+  Boolean                            $xpack                         = $elasticsearch::xpack,
 ) {
 
   File {
@@ -273,6 +277,10 @@
         fail('keystore_password required')
       }
 
+      if (($ca_certificate == undef) or ($certificate == undef) or ($private_key == undef)) {
+        fail('ca_certificate, certificate and private_key required')
+      }
+
       if ($keystore_path == undef) {
         $_keystore_path = "${configdir}/${security_plugin}/${name}.ks"
       } else {
@@ -287,11 +295,23 @@
           'shield.ssl.keystore.password' => $keystore_password,
         }
       } elsif $security_plugin == 'x-pack' {
-        $tls_config = {
-          'xpack.security.transport.ssl.enabled' => true,
-          'xpack.security.http.ssl.enabled'      => true,
-          'xpack.ssl.keystore.path'              => $_keystore_path,
-          'xpack.ssl.keystore.password'          => $keystore_password,
+        if $elasticsearch::version.split(/\./)[0] == '7' {
+          $tls_config = {
+            'xpack.security.transport.ssl.enabled'           => true,
+            'xpack.security.transport.ssl.keystore.path'     => $_keystore_path,
+            'xpack.security.transport.ssl.keystore.password' => $keystore_password,
+            'xpack.security.http.ssl.enabled'                => true,
+            'xpack.security.http.ssl.keystore.path'          => $_keystore_path,
+            'xpack.security.http.ssl.keystore.password'      => $keystore_password,
+          }
+        }
+        else {
+          $tls_config = {
+            'xpack.security.transport.ssl.enabled' => true,
+            'xpack.security.http.ssl.enabled'      => true,
+            'xpack.ssl.keystore.path'              => $_keystore_path,
+            'xpack.ssl.keystore.password'          => $keystore_password,
+          }
         }
       }
 
@@ -440,13 +460,51 @@
       }
     }
 
+    if $xpack {
+      if (($ca_certificate == undef) or ($certificate == undef) or ($private_key == undef)) {
+        fail('ca_certificate, certificate and private_key required')
+      }
+
+      file { "${configdir}/ca_certificate.pem":
+        ensure => 'file',
+        source => "file://${ca_certificate}",
+        owner  => $elasticsearch::elasticsearch_user,
+        group  => undef,
+        mode   => '0640',
+      }
+      file { "${configdir}/certificate.pem":
+        ensure => 'file',
+        source => "file://${certificate}",
+        owner  => $elasticsearch::elasticsearch_user,
+        group  => undef,
+        mode   => '0640',
+      }
+      file { "${configdir}/private_key.key":
+        ensure => 'file',
+        source => "file://${private_key}",
+        owner  => $elasticsearch::elasticsearch_user,
+        group  => undef,
+        mode   => '0600',
+      }
+
+      $xpack_config = {
+        'xpack.security.enabled'                               => true,
+        'xpack.security.transport.ssl.enabled'                 => true,
+        'xpack.security.transport.ssl.verification_mode'       => 'certificate',
+        'xpack.security.transport.ssl.key'                     => "${configdir}/private_key.key",
+        'xpack.security.transport.ssl.certificate'             => "${configdir}/certificate.pem",
+        'xpack.security.transport.ssl.certificate_authorities' => "${configdir}/ca_certificate.pem",
+      }
+    } else { $xpack_config = {} }
+
     # build up new config
     $instance_conf = merge(
       $main_config,
       $instance_node_name,
       $instance_datadir_config,
       { 'path.logs' => $logdir },
       $tls_config,
+      $xpack_config,
       $instance_config
     )
 

manifests/package.pp

@@ -182,7 +182,7 @@
 
     exec { 'remove_plugin_dir':
       refreshonly => true,
-      command     => "rm -rf ${elasticsearch::_plugindir}",
+      command     => "rm -rf ${::elasticsearch::_plugindir}",
     }
 
 

manifests/plugin.pp

@@ -137,7 +137,7 @@
     plugin_dir                 => $::elasticsearch::_plugindir,
     plugin_path                => $module_dir,
   }
-  -> file { "${elasticsearch::_plugindir}/${_module_dir}":
+  -> file { "${::elasticsearch::_plugindir}/${_module_dir}":
     ensure  => $_file_ensure,
     mode    => 'o+Xr',
     recurse => true,

spec/defines/005_elasticsearch_instance_spec.rb

@@ -790,30 +790,35 @@ class { 'elasticsearch':
             should contain_file('/etc/elasticsearch/es-instance/jvm.options')
               .with_content(%r{
                 -Dfile.encoding=UTF-8.
+                -Dio.netty.allocator.numDirectArenas=0.
                 -Dio.netty.noKeySetOptimization=true.
                 -Dio.netty.noUnsafe=true.
                 -Dio.netty.recycler.maxCapacityPerThread=0.
                 -Djava.awt.headless=true.
+                -Djava.io.tmpdir=\${ES_TMPDIR}.
                 -Djna.nosys=true.
                 -Dlog4j.shutdownHookEnabled=false.
                 -Dlog4j2.disable.jmx=true.
                 -XX:\+AlwaysPreTouch.
                 -XX:\+HeapDumpOnOutOfMemoryError.
-                -XX:\+PrintGCDateStamps.
-                -XX:\+PrintGCDetails.
-                -XX:\+PrintTenuringDistribution.
                 -XX:\+UseCMSInitiatingOccupancyOnly.
-                -XX:\+UseConcMarkSweepGC.
-                -XX:\+UseGCLogFileRotation.
                 -XX:-OmitStackTraceInFastThrow.
                 -XX:CMSInitiatingOccupancyFraction=75.
-                -XX:GCLogFileSize=64m.
-                -XX:NumberOfGCLogFiles=32.
-                -Xloggc:\/var\/log\/elasticsearch\/es-instance\/gc.log.
                 -Xms4g.
                 -Xmx4g.
                 -Xss1m.
                 -server.
+                8:-XX:\+PrintGCApplicationStoppedTime.
+                8:-XX:\+PrintGCDateStamps.
+                8:-XX:\+PrintGCDetails.
+                8:-XX:\+PrintTenuringDistribution.
+                8:-XX:\+UseConcMarkSweepGC.
+                8:-XX:\+UseGCLogFileRotation.
+                8:-XX:GCLogFileSize=64m.
+                8:-XX:NumberOfGCLogFiles=5.
+                8:-Xloggc:/var/log/elasticsearch/es-instance/gc.log.
+                9-:-Djava.locale.providers=COMPAT.
+                9-:-Xlog:gc\*,gc\+age=trace,safepoint:file=/var/log/elasticsearch/es-instance/gc.log:utctime,pid,tags:filecount=5,filesize=64m.
               }xm)
           end
         end
@@ -832,30 +837,35 @@ class { 'elasticsearch':
             should contain_file('/etc/elasticsearch/es-instance/jvm.options')
               .with_content(%r{
                 -Dfile.encoding=UTF-8.
+                -Dio.netty.allocator.numDirectArenas=0.
                 -Dio.netty.noKeySetOptimization=true.
                 -Dio.netty.noUnsafe=true.
                 -Dio.netty.recycler.maxCapacityPerThread=0.
                 -Djava.awt.headless=true.
+                -Djava.io.tmpdir=\${ES_TMPDIR}.
                 -Djna.nosys=true.
                 -Dlog4j.shutdownHookEnabled=false.
                 -Dlog4j2.disable.jmx=true.
                 -XX:\+AlwaysPreTouch.
                 -XX:\+HeapDumpOnOutOfMemoryError.
-                -XX:\+PrintGCDateStamps.
-                -XX:\+PrintGCDetails.
-                -XX:\+PrintTenuringDistribution.
                 -XX:\+UseCMSInitiatingOccupancyOnly.
-                -XX:\+UseConcMarkSweepGC.
-                -XX:\+UseGCLogFileRotation.
                 -XX:-OmitStackTraceInFastThrow.
                 -XX:CMSInitiatingOccupancyFraction=75.
-                -XX:GCLogFileSize=64m.
-                -XX:NumberOfGCLogFiles=32.
-                -Xloggc:\/var\/log\/elasticsearch\/es-instance\/gc.log.
                 -Xms8g.
                 -Xmx8g.
                 -Xss1m.
                 -server.
+                8:-XX:\+PrintGCApplicationStoppedTime.
+                8:-XX:\+PrintGCDateStamps.
+                8:-XX:\+PrintGCDetails.
+                8:-XX:\+PrintTenuringDistribution.
+                8:-XX:\+UseConcMarkSweepGC.
+                8:-XX:\+UseGCLogFileRotation.
+                8:-XX:GCLogFileSize=64m.
+                8:-XX:NumberOfGCLogFiles=5.
+                8:-Xloggc:/var/log/elasticsearch/es-instance/gc.log.
+                9-:-Djava.locale.providers=COMPAT.
+                9-:-Xlog:gc\*,gc\+age=trace,safepoint:file=/var/log/elasticsearch/es-instance/gc.log:utctime,pid,tags:filecount=5,filesize=64m.
               }xm)
           end
         end

templates/etc/elasticsearch/jvm.options.erb

@@ -9,7 +9,7 @@ end
 defaults = {
   '-Xms' => '-Xms2g',
   '-Xmx' => '-Xmx2g',
-  'UseConcMarkSweepGC' => '-XX:+UseConcMarkSweepGC',
+  'UseConcMarkSweepGC' => '8:-XX:+UseConcMarkSweepGC',
   'CMSInitiatingOccupancyFraction=' => '-XX:CMSInitiatingOccupancyFraction=75',
   'UseCMSInitiatingOccupancyOnly' => '-XX:+UseCMSInitiatingOccupancyOnly',
   'AlwaysPreTouch' => '-XX:+AlwaysPreTouch',
@@ -22,16 +22,21 @@ defaults = {
   '-Dio.netty.noUnsafe' => '-Dio.netty.noUnsafe=true',
   '-Dio.netty.noKeySetOptimization' => '-Dio.netty.noKeySetOptimization=true',
   '-Dio.netty.recycler.maxCapacityPerThread' => '-Dio.netty.recycler.maxCapacityPerThread=0',
+  '-Dio.netty.allocator.numDirectArenas' => '-Dio.netty.allocator.numDirectArenas=0',
   '-Dlog4j.shutdownHookEnabled' => '-Dlog4j.shutdownHookEnabled=false',
   '-Dlog4j2.disable.jmx' => '-Dlog4j2.disable.jmx=true',
+  '-Djava.io.tmpdir' => '-Djava.io.tmpdir=${ES_TMPDIR}',
   'HeapDumpOnOutOfMemoryError' => '-XX:+HeapDumpOnOutOfMemoryError',
-  'PrintGCDetails' => '-XX:+PrintGCDetails',
-  'PrintGCDateStamps' => '-XX:+PrintGCDateStamps',
-  'PrintTenuringDistribution' => '-XX:+PrintTenuringDistribution',
-  'Xloggc' => "-Xloggc:#{@logdir}/gc.log",
-  'UseGCLogFileRotation' => '-XX:+UseGCLogFileRotation',
-  'NumberOfGCLogFiles' => '-XX:NumberOfGCLogFiles=32',
-  'GCLogFileSize' => '-XX:GCLogFileSize=64m',
+  'PrintGCDetails' => '8:-XX:+PrintGCDetails',
+  'PrintGCDateStamps' => '8:-XX:+PrintGCDateStamps',
+  'PrintTenuringDistribution' => '8:-XX:+PrintTenuringDistribution',
+  'PrintGCApplicationStoppedTime' => '8:-XX:+PrintGCApplicationStoppedTime',
+  'Xloggc' => "8:-Xloggc:#{@logdir}/gc.log",
+  'UseGCLogFileRotation' => '8:-XX:+UseGCLogFileRotation',
+  'NumberOfGCLogFiles' => '8:-XX:NumberOfGCLogFiles=5',
+  'GCLogFileSize' => '8:-XX:GCLogFileSize=64m',
+  'Xlog:gc' => "9-:-Xlog:gc*,gc+age=trace,safepoint:file=#{@logdir}/gc.log:utctime,pid,tags:filecount=5,filesize=64m",
+  'Djava.locale.providers' => '9-:-Djava.locale.providers=COMPAT',
 }
 defaults.each {|k,v| set_default(@jvm_options, k, v)}
 

templates/etc/elasticsearch/log4j2.properties.erb

@@ -7,20 +7,20 @@ logger.action.level = debug
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%m%n
 
 <%- if @file_rolling_type == 'file' -%>
 appender.fixed.type = File
 appender.fixed.name = fixed
 appender.fixed.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.fixed.layout.type = PatternLayout
-appender.fixed.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.fixed.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%.-10000m%n
 <%- else -%>
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%.-10000m%n
 <%- if @file_rolling_type == 'dailyRollingFile' -%>
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
 appender.rolling.policies.type = Policies
@@ -49,7 +49,7 @@ appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%.-10000m%n
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -70,7 +70,7 @@ appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker%.-10000m%n
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -86,7 +86,7 @@ appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker%.-10000m%n
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy