diff --git a/README.md b/README.md index 284bc3f2..da29fe3d 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,6 @@ In addition to es_config, the following parameters allow the customization of th * ```update_java``` Updates Java to the latest version. (true or false (default)) * ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144. * ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. -* ```es_max_threads``` the maximum number of threads the process can start. Defaults to 2048 (the minimum required by elasticsearch). * ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds. * ```es_use_repository``` Setting this to `false` will stop Ansible from using the official Elastic package from any repository configured on the system. * ```es_add_repository``` Setting this to `false` will stop Ansible to add the official Elastic package repositories (if es_use_repository is true) if you want to use a repo already present. diff --git a/defaults/main.yml b/defaults/main.yml index 3bade03a..35d1ec3f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -20,7 +20,9 @@ es_templates: false es_user: elasticsearch es_group: elasticsearch es_config: {} -es_config_log4j2: log4j2.properties.j2 +es_config_default: "elasticsearch.j2" +es_config_log4j2: "" +es_config_jvm: "jvm.options.j2" #Need to provide default directories es_conf_dir: "/etc/elasticsearch" es_pid_dir: "/var/run/elasticsearch" @@ -29,7 +31,6 @@ es_data_dirs: es_log_dir: "/var/log/elasticsearch" es_action_auto_create_index: true es_max_open_files: 65536 -es_max_threads: 8192 es_max_map_count: 262144 es_allow_downgrades: false es_xpack_features: [] diff --git a/files/logging/log4j2.properties.custom.j2 b/files/logging/log4j2.properties.custom.j2 deleted file mode 100644 index 9a2a60f8..00000000 --- a/files/logging/log4j2.properties.custom.j2 +++ /dev/null @@ -1,76 +0,0 @@ -#CUSTOM LOG4J FILE - -status = error - -# log action execution errors for easier debugging -logger.action.name = org.elasticsearch.action -logger.action.level = info - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -appender.rolling.type = RollingFile -appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs}.log -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log -appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval = 1 -appender.rolling.policies.time.modulate = true - -rootLogger.level = debug -rootLogger.appenderRef.console.ref = console -rootLogger.appenderRef.rolling.ref = rolling - -appender.deprecation_rolling.type = RollingFile -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log -appender.deprecation_rolling.layout.type = PatternLayout -appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz -appender.deprecation_rolling.policies.type = Policies -appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling.policies.size.size = 10mb -appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling.strategy.max = 4 - -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = debug -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.additivity = false - -appender.index_search_slowlog_rolling.type = RollingFile -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log -appender.index_search_slowlog_rolling.layout.type = PatternLayout -appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log -appender.index_search_slowlog_rolling.policies.type = Policies -appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_search_slowlog_rolling.policies.time.interval = 1 -appender.index_search_slowlog_rolling.policies.time.modulate = true - -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = debug -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.additivity = false - -appender.index_indexing_slowlog_rolling.type = RollingFile -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log -appender.index_indexing_slowlog_rolling.policies.type = Policies -appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling.policies.time.interval = 1 -appender.index_indexing_slowlog_rolling.policies.time.modulate = true - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = debug -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.additivity = false diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index c8bc1cf7..796730f0 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -4,17 +4,32 @@ #Create conf directory - name: Create Configuration Directory become: yes - file: path={{ es_conf_dir }} state=directory owner=root group={{ es_group }} mode=2750 + file: + path: "{{ es_conf_dir }}" + state: directory + owner: root + group: "{{ es_group }}" + mode: "2750" #Create pid directory - name: Create PID Directory become: yes - file: path={{ es_pid_dir }} state=directory owner={{ es_user }} group={{ es_group }} mode=0755 + file: + path: "{{ es_pid_dir }}" + state: directory + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: "755" #Create required directories - name: Create Others Directories become: yes - file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} mode=2750 + file: + path: "{{ item }}" + state: directory + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: "2750" with_items: - "{{ es_log_dir }}" - "{{ es_data_dirs }}" @@ -22,14 +37,26 @@ #Copy the config template - name: Copy Configuration File become: yes - template: src=elasticsearch.yml.j2 dest={{ es_conf_dir }}/elasticsearch.yml owner=root group={{ es_group }} mode=0660 force=yes + template: + src: elasticsearch.yml.j2 + dest: "{{ es_conf_dir }}/elasticsearch.yml" + owner: root + group: "{{ es_group }}" + mode: "660" + force: yes register: system_change notify: restart elasticsearch #Copy the default file - name: Copy Default File become: yes - template: src=elasticsearch.j2 dest={{ default_file }} owner=root group={{ es_group }} mode=0660 force=yes + template: + src: "{{ es_config_default }}" + dest: "{{ default_file }}" + owner: root + group: "{{ es_group }}" + mode: "660" + force: yes notify: restart elasticsearch #Copy the systemd specific file if systemd is installed @@ -37,21 +64,41 @@ become: yes block: - name: Make sure destination dir exists - file: path={{ sysd_config_file | dirname }} state=directory mode=0755 + file: + path: "{{ sysd_config_file | dirname }}" + state: directory + mode: "755" - name: Copy specific ElasticSearch Systemd config file - ini_file: path={{ sysd_config_file }} section=Service option=LimitMEMLOCK value=infinity mode=0644 + ini_file: + path: "{{ sysd_config_file }}" + section: Service + option: LimitMEMLOCK + value: infinity + mode: "644" notify: - reload systemd configuration - restart elasticsearch -#Copy the logging.yml -- name: Copy log4j2.properties File +- name: Copy jvm.options File become: yes - template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner=root group={{ es_group }} mode=0660 force=yes + template: + src: "{{ es_config_jvm }}" + dest: "{{ es_conf_dir }}/jvm.options" + owner: root + group: "{{ es_group }}" + mode: "660" + force: yes notify: restart elasticsearch -- name: Copy jvm.options File +- name: Copy log4j2.properties File become: yes - template: src=jvm.options.j2 dest={{ es_conf_dir }}/jvm.options owner=root group={{ es_group }} mode=0660 force=yes + template: + src: "{{ es_config_log4j2 }}" + dest: "{{ es_conf_dir }}/log4j2.properties" + owner: root + group: "{{ es_group }}" + mode: "660" + force: yes notify: restart elasticsearch + when: es_config_log4j2 != '' diff --git a/tasks/elasticsearch-ssl.yml b/tasks/elasticsearch-ssl.yml index b4d2212d..70ac7b5a 100644 --- a/tasks/elasticsearch-ssl.yml +++ b/tasks/elasticsearch-ssl.yml @@ -13,7 +13,7 @@ state: directory owner: root group: "{{ es_group }}" - mode: 0750 + mode: "750" - name: Upload SSL/TLS keystore copy: @@ -21,7 +21,7 @@ dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}" owner: "{{ es_user }}" group: "{{ es_group }}" - mode: 0640 + mode: "640" when: es_ssl_keystore and es_ssl_truststore notify: restart elasticsearch register: copy_keystore @@ -32,7 +32,7 @@ dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}" owner: "{{ es_user }}" group: "{{ es_group }}" - mode: 0640 + mode: "640" when: es_ssl_keystore and es_ssl_truststore notify: restart elasticsearch register: copy_truststore @@ -43,7 +43,7 @@ dest: "{{ es_ssl_certificate_path }}/{{ item | basename }}" owner: "{{ es_user }}" group: "{{ es_group }}" - mode: 0640 + mode: "640" with_items: - "{{ es_ssl_key }}" - "{{ es_ssl_certificate }}" @@ -58,7 +58,7 @@ dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}" owner: "{{ es_user }}" group: "{{ es_group }}" - mode: 0640 + mode: "640" #Restart if this changes notify: restart elasticsearch when: es_ssl_certificate_authority | bool diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml index a04097f6..e7669324 100644 --- a/tasks/elasticsearch-template.yml +++ b/tasks/elasticsearch-template.yml @@ -6,7 +6,7 @@ state: directory owner: root group: "{{ es_group }}" - mode: 2750 + mode: "2750" - name: Copy templates to elasticsearch copy: src={{ item }} dest={{ es_conf_dir }}/templates owner=root group={{ es_group }} mode=0660 diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index 912cc383..84c4d758 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -9,30 +9,23 @@ ES_HOME={{es_home}} #JAVA_HOME= # Elasticsearch configuration directory -CONF_DIR={{ es_conf_dir }} ES_PATH_CONF={{ es_conf_dir }} -# Elasticsearch data directory -DATA_DIR={{ es_data_dirs | array_to_str }} - -# Elasticsearch logs directory -LOG_DIR={{ es_log_dir }} - # Elasticsearch PID directory PID_DIR={{ es_pid_dir }} -ES_JVM_OPTIONS={{ es_conf_dir }}/jvm.options +# Additional Java OPTS +ES_JAVA_OPTS= # Configure restart on package upgrade (true, every other setting will lead to not restarting) -#ES_RESTART_ON_UPGRADE=true - -# Path to the GC log file -#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log +#RESTART_ON_UPGRADE=true ################################ # Elasticsearch service ################################ +# SysV init.d +# # The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process ES_STARTUP_SLEEP_TIME=5 @@ -43,17 +36,17 @@ ES_STARTUP_SLEEP_TIME=5 # Specifies the maximum file descriptor number that can be opened by this process # When using Systemd, this setting is ignored and the LimitNOFILE defined in # /usr/lib/systemd/system/elasticsearch.service takes precedence +#MAX_OPEN_FILES=65535 {% if es_max_open_files is defined %} -#MAX_OPEN_FILES MAX_OPEN_FILES={{es_max_open_files}} {% endif %} # The maximum number of bytes of memory that may be locked into RAM # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option -# in elasticsearch.yml -# When using Systemd, the LimitMEMLOCK property must be set -# in /usr/lib/systemd/system/elasticsearch.service -#MAX_LOCKED_MEMORY= +# in elasticsearch.yml. +# When using systemd, LimitMEMLOCK must be set in a unit file such as +# /etc/systemd/system/elasticsearch.service.d/override.conf. +#MAX_LOCKED_MEMORY=unlimited {% if m_lock_enabled %} MAX_LOCKED_MEMORY=unlimited {% endif %} @@ -65,9 +58,3 @@ MAX_LOCKED_MEMORY=unlimited {% if es_max_map_count is defined %} MAX_MAP_COUNT={{es_max_map_count}} {% endif %} - -# Specifies the maximum number of threads that can be started. -# Elasticsearch requires a minimum of 2048. -{% if es_max_threads is defined %} -MAX_THREADS={{ es_max_threads }} -{% endif %} diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 index 98327824..dd908d70 100644 --- a/templates/jvm.options.j2 +++ b/templates/jvm.options.j2 @@ -41,6 +41,28 @@ -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly +## G1GC Configuration +# NOTE: G1GC is only supported on JDK version 10 or later. +# To use G1GC uncomment the lines below. +# 10-:-XX:-UseConcMarkSweepGC +# 10-:-XX:-UseCMSInitiatingOccupancyOnly +# 10-:-XX:+UseG1GC +{% if es_version is version('7.3.0', '<=') %} +# 10-:-XX:InitiatingHeapOccupancyPercent=75 +{% else %} +# 10-:-XX:G1ReservePercent=25 +# 10-:-XX:InitiatingHeapOccupancyPercent=30 +{% endif %} + +## DNS cache policy +# cache ttl in seconds for positive DNS lookups noting that this overrides the +# JDK security property networkaddress.cache.ttl; set to -1 to cache forever +-Des.networkaddress.cache.ttl=60 +# cache ttl in seconds for negative DNS lookups noting that this overrides the +# JDK security property networkaddress.cache.negative ttl; set to -1 to cache +# forever +-Des.networkaddress.cache.negative.ttl=10 + ## optimizations # pre-touch memory pages used by the JVM during initialization @@ -48,8 +70,8 @@ ## basic -# force the server VM --server +# explicitly set the stack size +-Xss1m # set to headless, just in case -Djava.awt.headless=true @@ -60,18 +82,23 @@ # use our provided JNA always versus the system one -Djna.nosys=true -# use old-style file permissions on JDK9 --Djdk.io.permissionsUseCanonicalPath=true +# turn off a JDK optimization that throws away stack traces for common +# exceptions because stack traces are important for debugging +-XX:-OmitStackTraceInFastThrow # flags to configure Netty -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 +{% if es_version is version('7.4.0', '>=') %} +-Dio.netty.allocator.numDirectArenas=0 +{% endif %} # log4j 2 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true --Dlog4j.skipJansi=true + +-Djava.io.tmpdir=${ES_TMPDIR} ## heap dumps @@ -79,30 +106,35 @@ # heap dumps are created in the working directory of the JVM -XX:+HeapDumpOnOutOfMemoryError -# specify an alternative path for heap dumps -# ensure the directory exists and has sufficient space -#-XX:HeapDumpPath=${heap.dump.path} - -## GC logging - -#-XX:+PrintGCDetails -#-XX:+PrintGCTimeStamps -#-XX:+PrintGCDateStamps -#-XX:+PrintClassHistogram -#-XX:+PrintTenuringDistribution -#-XX:+PrintGCApplicationStoppedTime - -# log GC status to a file with time stamps -# ensure the directory exists -#-Xloggc:${loggc} - +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath=${heap.dump.path} + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile={{ es_log_dir }}/hs_err_pid%p.log + +## JDK 8 GC logging + +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:{{ es_log_dir }}/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ es_log_dir }}/gc.log:utctime,pid,tags:filecount=32,filesize=64m +# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise +# time/date parsing will break in an incompatible way for some date patterns and locals +9-:-Djava.locale.providers=COMPAT + +{% if es_major_version == "6.x" %} +# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512 +10-:-XX:UseAVX=2 +{% endif %} -# By default, the GC log file will not rotate. -# By uncommenting the lines below, the GC log file -# will be rotated every 128MB at most 32 times. -#-XX:+UseGCLogFileRotation -#-XX:NumberOfGCLogFiles=32 -#-XX:GCLogFileSize=128M {% if es_jvm_custom_parameters !='' %} {% for item in es_jvm_custom_parameters %} {{ item }} diff --git a/templates/log4j2.properties.j2 b/templates/log4j2.properties.j2 deleted file mode 100644 index b4754c1a..00000000 --- a/templates/log4j2.properties.j2 +++ /dev/null @@ -1,85 +0,0 @@ -status = error - -# log action execution errors for easier debugging -logger.action.name = org.elasticsearch.action -logger.action.level = debug - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -appender.rolling.type = RollingFile -appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz -appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval = 1 -appender.rolling.policies.time.modulate = true -appender.rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling.policies.size.size = 128MB -appender.rolling.strategy.type = DefaultRolloverStrategy -appender.rolling.strategy.fileIndex = nomax -appender.rolling.strategy.action.type = Delete -appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} -appender.rolling.strategy.action.condition.type = IfFileName -appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* -appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize -appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB -rootLogger.level = info -rootLogger.appenderRef.console.ref = console -rootLogger.appenderRef.rolling.ref = rolling - -appender.deprecation_rolling.type = RollingFile -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log -appender.deprecation_rolling.layout.type = PatternLayout -appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz -appender.deprecation_rolling.policies.type = Policies -appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling.policies.size.size = 1GB -appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling.strategy.max = 4 - -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = warn -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.additivity = false - -appender.index_search_slowlog_rolling.type = RollingFile -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log -appender.index_search_slowlog_rolling.layout.type = PatternLayout -appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log -appender.index_search_slowlog_rolling.policies.type = Policies -appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_search_slowlog_rolling.policies.time.interval = 1 -appender.index_search_slowlog_rolling.policies.time.modulate = true - -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = trace -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.additivity = false - -appender.index_indexing_slowlog_rolling.type = RollingFile -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log -appender.index_indexing_slowlog_rolling.policies.type = Policies -appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling.policies.time.interval = 1 -appender.index_indexing_slowlog_rolling.policies.time.modulate = true - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = trace -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.additivity = false diff --git a/test/integration/files/custom_config/elasticsearch b/test/integration/files/custom_config/elasticsearch new file mode 100644 index 00000000..36540c8a --- /dev/null +++ b/test/integration/files/custom_config/elasticsearch @@ -0,0 +1,7 @@ +# Elasticsearch CUSTOM FILE + +ES_HOME=/usr/share/elasticsearch +ES_PATH_CONF=/etc/elasticsearch +PID_DIR=/var/run/elasticsearch +ES_JAVA_OPTS= +ES_STARTUP_SLEEP_TIME=5 diff --git a/test/integration/files/custom_config/jvm.options b/test/integration/files/custom_config/jvm.options new file mode 100644 index 00000000..71ce100a --- /dev/null +++ b/test/integration/files/custom_config/jvm.options @@ -0,0 +1,34 @@ +## JVM configuration CUSTOM FILE + +-Xms2g +-Xmx2g +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly +-Des.networkaddress.cache.ttl=60 +-Des.networkaddress.cache.negative.ttl=10 +-XX:+AlwaysPreTouch +-Xss1m +-Djava.awt.headless=true +-Dfile.encoding=UTF-8 +-Djna.nosys=true +-XX:-OmitStackTraceInFastThrow +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true +-Dio.netty.recycler.maxCapacityPerThread=0 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Djava.io.tmpdir=${ES_TMPDIR} +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=${heap.dump.path} +-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:/var/log/elasticsearch/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m +9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m +9-:-Djava.locale.providers=COMPAT diff --git a/test/integration/files/custom_config/log4j2.properties b/test/integration/files/custom_config/log4j2.properties new file mode 100644 index 00000000..1573120e --- /dev/null +++ b/test/integration/files/custom_config/log4j2.properties @@ -0,0 +1,86 @@ +{% raw %} +# Log4j CUSTOM FILE + +status = error +logger.action.name = org.elasticsearch.action +logger.action.level = debug +appender.rolling.type = Console +appender.rolling.name = rolling +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server +rootLogger.level = info +rootLogger.appenderRef.rolling.ref = rolling +appender.deprecation_rolling.type = Console +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type_name = deprecation +appender.deprecation_rolling.layout.esmessagefields=x-opaque-id +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false +appender.index_search_slowlog_rolling.type = Console +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog +appender.index_search_slowlog_rolling.layout.esmessagefields=message,took,took_millis,total_hits,stats,search_type,total_shards,source,id +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false +appender.index_indexing_slowlog_rolling.type = Console +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog +appender.index_indexing_slowlog_rolling.layout.esmessagefields=message,took,took_millis,doc_type,id,routing,source +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false +appender.audit_rolling.type = Console +appender.audit_rolling.name = audit_rolling +appender.audit_rolling.layout.type = PatternLayout +appender.audit_rolling.layout.pattern = {\ +"type": "audit", \ +"timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}"\ +%varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\ +%varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\ +%varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\ +%varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\ +%varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\ +%varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\ +%varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\ +%varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\ +%varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\ +%varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\ +%varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\ +%varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\ +%varsNotEmpty{, "user.roles":%map{user.roles}}\ +%varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\ +%varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\ +%varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\ +%varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\ +%varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}\ +%varsNotEmpty{, "request.method":"%enc{%map{request.method}}{JSON}"}\ +%varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}\ +%varsNotEmpty{, "request.id":"%enc{%map{request.id}}{JSON}"}\ +%varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}\ +%varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\ +%varsNotEmpty{, "indices":%map{indices}}\ +%varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\ +%varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\ +%varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\ +%varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\ +%varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\ +}%n +logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail +logger.xpack_security_audit_logfile.level = info +logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling +logger.xpack_security_audit_logfile.additivity = false +logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature +logger.xmlsig.level = error +logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter +logger.samlxml_decrypt.level = fatal +logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter +logger.saml2_decrypt.level = fatal +{% endraw %} diff --git a/test/integration/helpers/serverspec/oss_spec.rb b/test/integration/helpers/serverspec/oss_spec.rb index 0f4ff000..f07ce889 100644 --- a/test/integration/helpers/serverspec/oss_spec.rb +++ b/test/integration/helpers/serverspec/oss_spec.rb @@ -1,13 +1,20 @@ require 'spec_helper' +require 'shared_spec' shared_examples 'oss::init' do |vars| describe file("/etc/elasticsearch/log4j2.properties") do it { should be_file } it { should be_owned_by 'root' } - it { should_not contain 'CUSTOM LOG4J FILE' } + it { should contain 'Log4j CUSTOM FILE' } end describe file("/etc/elasticsearch/jvm.options") do it { should be_file } it { should be_owned_by 'root' } + it { should contain 'JVM configuration CUSTOM FILE' } + end + describe file($family['defaults_path']) do + it { should be_file } + it { should be_owned_by 'root' } + it { should contain 'Elasticsearch CUSTOM FILE' } end end diff --git a/test/integration/helpers/serverspec/shared_spec.rb b/test/integration/helpers/serverspec/shared_spec.rb index 0522fe49..4bf94e98 100644 --- a/test/integration/helpers/serverspec/shared_spec.rb +++ b/test/integration/helpers/serverspec/shared_spec.rb @@ -2,7 +2,7 @@ require 'json' vars = JSON.parse(File.read('/tmp/vars.json')) -families = { +$families = { 'Debian' => { 'shell' => '/bin/false', 'password' => '*', @@ -15,7 +15,7 @@ } } -family = families[vars['ansible_os_family']] +$family = $families[vars['ansible_os_family']] es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}" username = vars['es_api_basic_auth_username'] @@ -88,9 +88,9 @@ it { should belong_to_group vars['es_group'] } it { should have_uid vars['es_user_id'] } if vars.key?('es_user_id') - it { should have_login_shell family['shell'] } + it { should have_login_shell $family['shell'] } - its(:encrypted_password) { should eq(family['password']) } + its(:encrypted_password) { should eq($family['password']) } end describe package(vars['es_package_name']) do @@ -129,7 +129,7 @@ end end - describe file(family['defaults_path']) do + describe file($family['defaults_path']) do its(:content) { should match '' } end diff --git a/test/integration/oss.yml b/test/integration/oss.yml index f94d5852..d7ab9b72 100644 --- a/test/integration/oss.yml +++ b/test/integration/oss.yml @@ -10,5 +10,8 @@ es_heap_size: "1g" es_plugins: - plugin: ingest-attachment + es_config_default: "test/integration/files/custom_config/elasticsearch" + es_config_log4j2: "test/integration/files/custom_config/log4j2.properties" + es_config_jvm: "test/integration/files/custom_config/jvm.options" #Do not add tests here. This test is run twice and confirms idempotency.