diff --git a/.travis.yml b/.travis.yml index 8e03da626..f440042a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -group: deprecated-2017Q4 +group: deprecated-2017q4 services: - docker cache: @@ -11,11 +11,6 @@ language: ruby script: travis_retry bundle exec rake $TASK jobs: allow_failures: - - env: - - TASK=beaker:ubuntu-server-1404-x64:snapshot - - env: - - OSS_PACKAGE=true - - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: @@ -97,12 +92,6 @@ jobs: - TASK=beaker:debian-9-x64:acceptance[5.6.9] - env: - TASK=beaker:debian-9-x64:acceptance[2.4.6] - - env: - - TASK=beaker:ubuntu-server-1404-x64:acceptance - - env: - - TASK=beaker:ubuntu-server-1404-x64:acceptance[5.6.9] - - env: - - TASK=beaker:ubuntu-server-1404-x64:acceptance[2.4.6] - env: - TASK=beaker:ubuntu-server-1604-x64:acceptance - env: @@ -110,11 +99,6 @@ jobs: - env: - TASK=beaker:ubuntu-server-1604-x64:acceptance[2.4.6] - stage: snapshots - env: - - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - - OSS_PACKAGE=true - - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: diff --git a/CONTRIBUTORS b/CONTRIBUTORS index a7984751a..dab0a04cf 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -33,3 +33,4 @@ Matthias Baur (baurmatt) Gavin Williams (fatmcgav) Giedrius Statkevičius (GiedriusS) Jakub Pieńkowski (Jakski) +Simon Hulme (uberjew666) diff --git a/data/common.yaml b/data/common.yaml index bb17a2138..fa3035f38 100644 --- a/data/common.yaml +++ b/data/common.yaml @@ -65,3 +65,4 @@ elasticsearch::templates: {} elasticsearch::users: {} elasticsearch::validate_tls: true elasticsearch::version: false +elasticsearch::xpack: false diff --git a/lib/puppet/type/elasticsearch_keystore.rb b/lib/puppet/type/elasticsearch_keystore.rb index 46f5d7d3e..a65653894 100644 --- a/lib/puppet/type/elasticsearch_keystore.rb +++ b/lib/puppet/type/elasticsearch_keystore.rb @@ -59,6 +59,6 @@ def change_to_s(currentvalue, newvalue_raw) end autorequire(:augeas) do - "defaults_#{self[:name]}" + "ks_defaults_#{self[:name]}" end end diff --git a/manifests/init.pp b/manifests/init.pp index 0e4bd0060..e9e81ffcd 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -297,6 +297,9 @@ # @param version # To set the specific version you want to install. # +# @param xpack +# Enable x-pack security. Requires ca_certificate, certificate and private key. +# # @author Richard Pijnenburg # @author Tyler Langlois # @@ -371,6 +374,7 @@ Hash $users, Boolean $validate_tls, Variant[String, Boolean] $version, + Boolean $xpack, Boolean $restart_config_change = $restart_on_change, Boolean $restart_package_change = $restart_on_change, Boolean $restart_plugin_change = $restart_on_change, diff --git a/manifests/instance.pp b/manifests/instance.pp index 813136f81..41547e5d0 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -125,6 +125,9 @@ # Source for the Shield system key. Valid values are any that are # supported for the file resource `source` parameter. # +# @param xpack +# Enable xpack security features. Requires ca_certificate, certificate and private_key. +# # @author Richard Pijnenburg # @author Tyler Langlois # @@ -162,6 +165,7 @@ Boolean $ssl = false, Elasticsearch::Status $status = $elasticsearch::status, Optional[String] $system_key = $elasticsearch::system_key, + Boolean $xpack = $elasticsearch::xpack, ) { File { @@ -273,6 +277,10 @@ fail('keystore_password required') } + if (($ca_certificate == undef) or ($certificate == undef) or ($private_key == undef)) { + fail('ca_certificate, certificate and private_key required') + } + if ($keystore_path == undef) { $_keystore_path = "${configdir}/${security_plugin}/${name}.ks" } else { @@ -287,11 +295,23 @@ 'shield.ssl.keystore.password' => $keystore_password, } } elsif $security_plugin == 'x-pack' { - $tls_config = { - 'xpack.security.transport.ssl.enabled' => true, - 'xpack.security.http.ssl.enabled' => true, - 'xpack.ssl.keystore.path' => $_keystore_path, - 'xpack.ssl.keystore.password' => $keystore_password, + if $elasticsearch::version.split(/\./)[0] == '7' { + $tls_config = { + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.transport.ssl.keystore.path' => $_keystore_path, + 'xpack.security.transport.ssl.keystore.password' => $keystore_password, + 'xpack.security.http.ssl.enabled' => true, + 'xpack.security.http.ssl.keystore.path' => $_keystore_path, + 'xpack.security.http.ssl.keystore.password' => $keystore_password, + } + } + else { + $tls_config = { + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.http.ssl.enabled' => true, + 'xpack.ssl.keystore.path' => $_keystore_path, + 'xpack.ssl.keystore.password' => $keystore_password, + } } } @@ -440,6 +460,43 @@ } } + if $xpack { + if (($ca_certificate == undef) or ($certificate == undef) or ($private_key == undef)) { + fail('ca_certificate, certificate and private_key required') + } + + file { "${configdir}/ca_certificate.pem": + ensure => 'file', + source => "file://${ca_certificate}", + owner => $elasticsearch::elasticsearch_user, + group => undef, + mode => '0640', + } + file { "${configdir}/certificate.pem": + ensure => 'file', + source => "file://${certificate}", + owner => $elasticsearch::elasticsearch_user, + group => undef, + mode => '0640', + } + file { "${configdir}/private_key.key": + ensure => 'file', + source => "file://${private_key}", + owner => $elasticsearch::elasticsearch_user, + group => undef, + mode => '0600', + } + + $xpack_config = { + 'xpack.security.enabled' => true, + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.transport.ssl.verification_mode' => 'certificate', + 'xpack.security.transport.ssl.key' => "${configdir}/private_key.key", + 'xpack.security.transport.ssl.certificate' => "${configdir}/certificate.pem", + 'xpack.security.transport.ssl.certificate_authorities' => "${configdir}/ca_certificate.pem", + } + } else { $xpack_config = {} } + # build up new config $instance_conf = merge( $main_config, @@ -447,6 +504,7 @@ $instance_datadir_config, { 'path.logs' => $logdir }, $tls_config, + $xpack_config, $instance_config ) diff --git a/manifests/package.pp b/manifests/package.pp index 0bf8cd762..109d48a38 100644 --- a/manifests/package.pp +++ b/manifests/package.pp @@ -182,7 +182,7 @@ exec { 'remove_plugin_dir': refreshonly => true, - command => "rm -rf ${elasticsearch::_plugindir}", + command => "rm -rf ${::elasticsearch::_plugindir}", } diff --git a/manifests/plugin.pp b/manifests/plugin.pp index 12b74a1be..cb2551670 100644 --- a/manifests/plugin.pp +++ b/manifests/plugin.pp @@ -137,7 +137,7 @@ plugin_dir => $::elasticsearch::_plugindir, plugin_path => $module_dir, } - -> file { "${elasticsearch::_plugindir}/${_module_dir}": + -> file { "${::elasticsearch::_plugindir}/${_module_dir}": ensure => $_file_ensure, mode => 'o+Xr', recurse => true, diff --git a/spec/defines/005_elasticsearch_instance_spec.rb b/spec/defines/005_elasticsearch_instance_spec.rb index 57a1d93c6..d7908ff3b 100644 --- a/spec/defines/005_elasticsearch_instance_spec.rb +++ b/spec/defines/005_elasticsearch_instance_spec.rb @@ -790,30 +790,35 @@ class { 'elasticsearch': should contain_file('/etc/elasticsearch/es-instance/jvm.options') .with_content(%r{ -Dfile.encoding=UTF-8. + -Dio.netty.allocator.numDirectArenas=0. -Dio.netty.noKeySetOptimization=true. -Dio.netty.noUnsafe=true. -Dio.netty.recycler.maxCapacityPerThread=0. -Djava.awt.headless=true. + -Djava.io.tmpdir=\${ES_TMPDIR}. -Djna.nosys=true. -Dlog4j.shutdownHookEnabled=false. -Dlog4j2.disable.jmx=true. -XX:\+AlwaysPreTouch. -XX:\+HeapDumpOnOutOfMemoryError. - -XX:\+PrintGCDateStamps. - -XX:\+PrintGCDetails. - -XX:\+PrintTenuringDistribution. -XX:\+UseCMSInitiatingOccupancyOnly. - -XX:\+UseConcMarkSweepGC. - -XX:\+UseGCLogFileRotation. -XX:-OmitStackTraceInFastThrow. -XX:CMSInitiatingOccupancyFraction=75. - -XX:GCLogFileSize=64m. - -XX:NumberOfGCLogFiles=32. - -Xloggc:\/var\/log\/elasticsearch\/es-instance\/gc.log. -Xms4g. -Xmx4g. -Xss1m. -server. + 8:-XX:\+PrintGCApplicationStoppedTime. + 8:-XX:\+PrintGCDateStamps. + 8:-XX:\+PrintGCDetails. + 8:-XX:\+PrintTenuringDistribution. + 8:-XX:\+UseConcMarkSweepGC. + 8:-XX:\+UseGCLogFileRotation. + 8:-XX:GCLogFileSize=64m. + 8:-XX:NumberOfGCLogFiles=5. + 8:-Xloggc:/var/log/elasticsearch/es-instance/gc.log. + 9-:-Djava.locale.providers=COMPAT. + 9-:-Xlog:gc\*,gc\+age=trace,safepoint:file=/var/log/elasticsearch/es-instance/gc.log:utctime,pid,tags:filecount=5,filesize=64m. }xm) end end @@ -832,30 +837,35 @@ class { 'elasticsearch': should contain_file('/etc/elasticsearch/es-instance/jvm.options') .with_content(%r{ -Dfile.encoding=UTF-8. + -Dio.netty.allocator.numDirectArenas=0. -Dio.netty.noKeySetOptimization=true. -Dio.netty.noUnsafe=true. -Dio.netty.recycler.maxCapacityPerThread=0. -Djava.awt.headless=true. + -Djava.io.tmpdir=\${ES_TMPDIR}. -Djna.nosys=true. -Dlog4j.shutdownHookEnabled=false. -Dlog4j2.disable.jmx=true. -XX:\+AlwaysPreTouch. -XX:\+HeapDumpOnOutOfMemoryError. - -XX:\+PrintGCDateStamps. - -XX:\+PrintGCDetails. - -XX:\+PrintTenuringDistribution. -XX:\+UseCMSInitiatingOccupancyOnly. - -XX:\+UseConcMarkSweepGC. - -XX:\+UseGCLogFileRotation. -XX:-OmitStackTraceInFastThrow. -XX:CMSInitiatingOccupancyFraction=75. - -XX:GCLogFileSize=64m. - -XX:NumberOfGCLogFiles=32. - -Xloggc:\/var\/log\/elasticsearch\/es-instance\/gc.log. -Xms8g. -Xmx8g. -Xss1m. -server. + 8:-XX:\+PrintGCApplicationStoppedTime. + 8:-XX:\+PrintGCDateStamps. + 8:-XX:\+PrintGCDetails. + 8:-XX:\+PrintTenuringDistribution. + 8:-XX:\+UseConcMarkSweepGC. + 8:-XX:\+UseGCLogFileRotation. + 8:-XX:GCLogFileSize=64m. + 8:-XX:NumberOfGCLogFiles=5. + 8:-Xloggc:/var/log/elasticsearch/es-instance/gc.log. + 9-:-Djava.locale.providers=COMPAT. + 9-:-Xlog:gc\*,gc\+age=trace,safepoint:file=/var/log/elasticsearch/es-instance/gc.log:utctime,pid,tags:filecount=5,filesize=64m. }xm) end end diff --git a/templates/etc/elasticsearch/jvm.options.erb b/templates/etc/elasticsearch/jvm.options.erb index c9b0dded9..31349f9f6 100644 --- a/templates/etc/elasticsearch/jvm.options.erb +++ b/templates/etc/elasticsearch/jvm.options.erb @@ -9,7 +9,7 @@ end defaults = { '-Xms' => '-Xms2g', '-Xmx' => '-Xmx2g', - 'UseConcMarkSweepGC' => '-XX:+UseConcMarkSweepGC', + 'UseConcMarkSweepGC' => '8:-XX:+UseConcMarkSweepGC', 'CMSInitiatingOccupancyFraction=' => '-XX:CMSInitiatingOccupancyFraction=75', 'UseCMSInitiatingOccupancyOnly' => '-XX:+UseCMSInitiatingOccupancyOnly', 'AlwaysPreTouch' => '-XX:+AlwaysPreTouch', @@ -22,16 +22,21 @@ defaults = { '-Dio.netty.noUnsafe' => '-Dio.netty.noUnsafe=true', '-Dio.netty.noKeySetOptimization' => '-Dio.netty.noKeySetOptimization=true', '-Dio.netty.recycler.maxCapacityPerThread' => '-Dio.netty.recycler.maxCapacityPerThread=0', + '-Dio.netty.allocator.numDirectArenas' => '-Dio.netty.allocator.numDirectArenas=0', '-Dlog4j.shutdownHookEnabled' => '-Dlog4j.shutdownHookEnabled=false', '-Dlog4j2.disable.jmx' => '-Dlog4j2.disable.jmx=true', + '-Djava.io.tmpdir' => '-Djava.io.tmpdir=${ES_TMPDIR}', 'HeapDumpOnOutOfMemoryError' => '-XX:+HeapDumpOnOutOfMemoryError', - 'PrintGCDetails' => '-XX:+PrintGCDetails', - 'PrintGCDateStamps' => '-XX:+PrintGCDateStamps', - 'PrintTenuringDistribution' => '-XX:+PrintTenuringDistribution', - 'Xloggc' => "-Xloggc:#{@logdir}/gc.log", - 'UseGCLogFileRotation' => '-XX:+UseGCLogFileRotation', - 'NumberOfGCLogFiles' => '-XX:NumberOfGCLogFiles=32', - 'GCLogFileSize' => '-XX:GCLogFileSize=64m', + 'PrintGCDetails' => '8:-XX:+PrintGCDetails', + 'PrintGCDateStamps' => '8:-XX:+PrintGCDateStamps', + 'PrintTenuringDistribution' => '8:-XX:+PrintTenuringDistribution', + 'PrintGCApplicationStoppedTime' => '8:-XX:+PrintGCApplicationStoppedTime', + 'Xloggc' => "8:-Xloggc:#{@logdir}/gc.log", + 'UseGCLogFileRotation' => '8:-XX:+UseGCLogFileRotation', + 'NumberOfGCLogFiles' => '8:-XX:NumberOfGCLogFiles=5', + 'GCLogFileSize' => '8:-XX:GCLogFileSize=64m', + 'Xlog:gc' => "9-:-Xlog:gc*,gc+age=trace,safepoint:file=#{@logdir}/gc.log:utctime,pid,tags:filecount=5,filesize=64m", + 'Djava.locale.providers' => '9-:-Djava.locale.providers=COMPAT', } defaults.each {|k,v| set_default(@jvm_options, k, v)} diff --git a/templates/etc/elasticsearch/log4j2.properties.erb b/templates/etc/elasticsearch/log4j2.properties.erb index 4e4d831f7..2ddb0e7de 100644 --- a/templates/etc/elasticsearch/log4j2.properties.erb +++ b/templates/etc/elasticsearch/log4j2.properties.erb @@ -7,20 +7,20 @@ logger.action.level = debug appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%m%n <%- if @file_rolling_type == 'file' -%> appender.fixed.type = File appender.fixed.name = fixed appender.fixed.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log appender.fixed.layout.type = PatternLayout -appender.fixed.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.fixed.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%.-10000m%n <%- else -%> appender.rolling.type = RollingFile appender.rolling.name = rolling appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%.-10000m%n <%- if @file_rolling_type == 'dailyRollingFile' -%> appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz appender.rolling.policies.type = Policies @@ -49,7 +49,7 @@ appender.deprecation_rolling.type = RollingFile appender.deprecation_rolling.name = deprecation_rolling appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log appender.deprecation_rolling.layout.type = PatternLayout -appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%.-10000m%n appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz appender.deprecation_rolling.policies.type = Policies appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy @@ -70,7 +70,7 @@ appender.index_search_slowlog_rolling.type = RollingFile appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log appender.index_search_slowlog_rolling.layout.type = PatternLayout -appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker%.-10000m%n appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log appender.index_search_slowlog_rolling.policies.type = Policies appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy @@ -86,7 +86,7 @@ appender.index_indexing_slowlog_rolling.type = RollingFile appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log appender.index_indexing_slowlog_rolling.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker%.-10000m%n appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log appender.index_indexing_slowlog_rolling.policies.type = Policies appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy