fix inconsitent handling of config dir and path settings and add test

When running bin/elasticsearch config dir and config file can be passed as a parameter
independently with -Des.config and -Des.path.conf. If the config file is not provided
then it is assumed to be {path.conf}/elasticearch.yml

This was not so for running as a service. For deb the CONF_FILE still pointed
to the default path even if a custom CONF_DIR was defined.
For rpm the CONF_FILE parameter was not passed on at all, see elastic#5329 .

Custom config path and config file now work as follows for all services except systemd:

CONF_DIR and CONF_FILE undefined: CONF_FILE points to default
CONF_DIR defined but CONF_FILE undefined: CONF_FILE points to CONF_DIR/elasticsearch.yml
CONF_FILE defined: CONF_FILE must point to an absolute path

For systemd this commit only fixes that the service could be started if only a CONF_DIR
is defined. However, now a custom CONF_FILE cannot be defined anymore which seemed
the lesser evil to me.

relates to elastic#12712 and elastic#12954
closes elastic#5329

distribution/deb/src/main/packaging/init.d/elasticsearch

@@ -85,9 +85,6 @@ DATA_DIR=/var/lib/$NAME
 # Elasticsearch configuration directory
 CONF_DIR=/etc/$NAME
 
-# Elasticsearch configuration file (elasticsearch.yml)
-CONF_FILE=$CONF_DIR/elasticsearch.yml
-
 # Maximum number of VMA (Virtual Memory Areas) a process can own
 MAX_MAP_COUNT=262144
 
@@ -104,6 +101,12 @@ if [ -f "$DEFAULT" ]; then
 	. "$DEFAULT"
 fi
 
+# Elasticsearch configuration file (elasticsearch.yml)
+if [ -z "$CONF_FILE" ]; then
+    # file was not set by the user, use the configured conf path
+    CONF_FILE=$CONF_DIR/elasticsearch.yml
+fi
+
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch

distribution/rpm/src/main/packaging/init.d/elasticsearch

@@ -40,7 +40,7 @@ MAX_MAP_COUNT=${packaging.os.max.map.count}
 LOG_DIR="${packaging.elasticsearch.log.dir}"
 DATA_DIR="${packaging.elasticsearch.data.dir}"
 CONF_DIR="${packaging.elasticsearch.conf.dir}"
-CONF_FILE="${packaging.elasticsearch.conf.dir}/elasticsearch.yml"
+
 PID_DIR="${packaging.elasticsearch.pid.dir}"
 
 # Source the default env file
@@ -49,6 +49,12 @@ if [ -f "$ES_ENV_FILE" ]; then
     . "$ES_ENV_FILE"
 fi
 
+# Elasticsearch configuration file (elasticsearch.yml)
+if [ -z "$CONF_FILE" ]; then
+    # file was not set by the user, use the configured conf path
+    CONF_FILE=$CONF_DIR/elasticsearch.yml
+fi
+
 exec="$ES_HOME/bin/elasticsearch"
 prog="elasticsearch"
 pidfile="$PID_DIR/${prog}.pid"
@@ -110,7 +116,7 @@ start() {
     cd $ES_HOME
     echo -n $"Starting $prog: "
     # if not running, start it up here, usually something like "daemon $exec"
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR
+    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.config=$CONF_FILE -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR
     retval=$?
     echo
     [ $retval -eq 0 ] && touch $lockfile

distribution/src/main/packaging/systemd/elasticsearch.service

@@ -7,7 +7,6 @@ After=network-online.target
 [Service]
 Environment=ES_HOME=${packaging.elasticsearch.home.dir}
 Environment=CONF_DIR=${packaging.elasticsearch.conf.dir}
-Environment=CONF_FILE=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
 Environment=DATA_DIR=${packaging.elasticsearch.data.dir}
 Environment=LOG_DIR=${packaging.elasticsearch.log.dir}
 Environment=PID_DIR=${packaging.elasticsearch.pid.dir}
@@ -23,7 +22,7 @@ ExecStart=${packaging.elasticsearch.bin.dir}/elasticsearch \
                                                 -Des.default.path.home=${ES_HOME} \
                                                 -Des.default.path.logs=${LOG_DIR} \
                                                 -Des.default.path.data=${DATA_DIR} \
-                                                -Des.default.config=${CONF_FILE} \
+                                                -Des.default.config=${CONF_DIR}/elasticsearch.yml \
                                                 -Des.default.path.conf=${CONF_DIR}
 
 # Connects standard output to /dev/null

docs/reference/setup/as-a-service.asciidoc

@@ -72,9 +72,9 @@ sudo service elasticsearch start
 
 
 [float]
-===== Using systemd
+==== Using systemd
 
-Distributions like SUSE do not use the `chkconfig` tool to register services, but rather `systemd` and its command `/bin/systemctl` to start and stop services (at least in newer versions, otherwise use the `chkconfig` commands above). The configuration file is also placed at `/etc/sysconfig/elasticsearch`. After installing the RPM, you have to change the systemd configuration and then start up elasticsearch
+Distributions like SUSE do not use the `chkconfig` tool to register services, but rather `systemd` and its command `/bin/systemctl` to start and stop services (at least in newer versions, otherwise use the `chkconfig` commands above). The configuration file is also placed at `/etc/sysconfig/elasticsearch` if the system is rpm based and `/etc/default/elasticsearch` if it is deb. After installing the RPM, you have to change the systemd configuration and then start up elasticsearch
 
 [source,sh]
 --------------------------------------------------

qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash

@@ -239,12 +239,25 @@ clean_before_test() {
 start_elasticsearch_service() {
     local desiredStatus=${1:-green}
 
+# Set the CONF_DIR setting in case we start as a service
+    if [ ! -z "$CONF_DIR" ] ; then
+        if is_dpkg ; then
+            echo "CONF_DIR=$CONF_DIR" >> /etc/default/elasticsearch;
+        elif is_rpm; then
+            echo "CONF_DIR=$CONF_DIR" >> /etc/sysconfig/elasticsearch;
+        fi
+    fi
+
     if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
+        if [ -z "$CONF_DIR" ]; then
+            local CONF_DIR=""
+        fi
         # su and the Elasticsearch init script work together to break bats.
         # sudo isolates bats enough from the init script so everything continues
         # to tick along
         sudo -u elasticsearch /tmp/elasticsearch/bin/elasticsearch -d \
-            -p /tmp/elasticsearch/elasticsearch.pid
+            -p /tmp/elasticsearch/elasticsearch.pid \
+            -Des.path.conf="$CONF_DIR";
     elif is_systemd; then
         run systemctl daemon-reload
         [ "$status" -eq 0 ]
@@ -322,7 +335,7 @@ wait_for_elasticsearch_status() {
               if [ -e '/var/log/elasticsearch/elasticsearch.log' ]; then
                   cat /var/log/elasticsearch/elasticsearch.log
               else
-                  echo "The elasticsearch log doesn't exist. Maybe /vag/log/messages has something:"
+                  echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:"
                   tail -n20 /var/log/messages
               fi
           fi

qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash

@@ -109,6 +109,9 @@ fi
     move_config
 
     CONF_DIR="$ESCONFIG" install_jvm_example
+    CONF_DIR="$ESCONFIG" start_elasticsearch_service
+    diff  <(curl -s localhost:9200/_cat/configured_example | sed 's/ //g') <(echo "foo")
+    stop_elasticsearch_service
     CONF_DIR="$ESCONFIG" remove_jvm_example
 }