Issue 174 - Logging

activemq/Dockerfile

@@ -20,6 +20,8 @@ EXPOSE 61616 5672 61613 1883 61614 8161
 
 # Defaults environment variables to be overloaded.
 ENV \
+    ACTIVEMQ_AUDIT_LOG_LEVEL=INFO \
+    ACTIVEMQ_LOG_LEVEL=INFO \
     ACTIVEMQ_PASSWORD=password \
     ACTIVEMQ_USER=admin \
     ACTIVEMQ_WEB_ADMIN_NAME=admin \

activemq/README.md

@@ -40,18 +40,20 @@ additional settings, volumes, ports, etc.
 
 ## Settings
 
-| Environment Variable        | Confd Key                     | Default  | Description                              |
-| :-------------------------- | :--------------------------- | :------- | :--------------------------------------- |
-| ACTIVEMQ_USER               | /activemq/user               | admin    | See [Security]: credentials.properties   |
-| ACTIVEMQ_PASSWORD           | /activemq/password           | password | See [Security]: credentials.properties   |
-| ACTIVEMQ_WEB_ADMIN_NAME     | /activemq/web/admin/name     | admin    | See [WebConsole]: jetty-realm.properties |
-| ACTIVEMQ_WEB_ADMIN_PASSWORD | /activemq/web/admin/password | password | See [WebConsole]: jetty-realm.properties |
-| ACTIVEMQ_WEB_ADMIN_ROLES    | /activemq/web/admin/roles    | admin    | See [WebConsole]: jetty-realm.properties |
+| Environment Variable        | Confd Key                    | Default  | Description                                                                    |
+| :-------------------------- | :--------------------------- | :------- | :----------------------------------------------------------------------------- |
+| ACTIVEMQ_AUDIT_LOG_LEVEL    | /activemq/audit/log/level    | INFO     | Log level. Possible Values: OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE or ALL |
+| ACTIVEMQ_LOG_LEVEL          | /activemq/log/level          | INFO     | Log level. Possible Values: OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE or ALL |
+| ACTIVEMQ_PASSWORD           | /activemq/password           | password | See [Security]: credentials.properties                                         |
+| ACTIVEMQ_USER               | /activemq/user               | admin    | See [Security]: credentials.properties                                         |
+| ACTIVEMQ_WEB_ADMIN_NAME     | /activemq/web/admin/name     | admin    | See [WebConsole]: jetty-realm.properties                                       |
+| ACTIVEMQ_WEB_ADMIN_PASSWORD | /activemq/web/admin/password | password | See [WebConsole]: jetty-realm.properties                                       |
+| ACTIVEMQ_WEB_ADMIN_ROLES    | /activemq/web/admin/roles    | admin    | See [WebConsole]: jetty-realm.properties                                       |
 
 Additional users/groups/etc can be defined by adding more environment variables,
 following the above conventions:
 
-| Environment Variable              | Confd Key                           | Description                              |
+| Environment Variable              | Confd Key                          | Description                              |
 | :-------------------------------- | :--------------------------------- | :--------------------------------------- |
 | ACTIVEMQ_USER_{USER}_NAME         | /activemq/user/{USER}/name         | See [Security]: users.properties         |
 | ACTIVEMQ_USER_{USER}_PASSWORD     | /activemq/user/{USER}/password     | See [Security]: users.properties         |
@@ -66,19 +68,16 @@ following the above conventions:
 For example to add a new user `someone` to the [WebConsole] you would need to
 define the following:
 
-| Environment Variable               | Confd Key                            | Value    |
+| Environment Variable               | Confd Key                           | Value    |
 | :--------------------------------- | :---------------------------------- | :------- |
 | ACTIVEMQ_WEB_USER_SOMEONE_NAME     | /activemq/web/user/someone/name     | someone  |
 | ACTIVEMQ_WEB_USER_SOMEONE_PASSWORD | /activemq/web/user/someone/password | password |
 | ACTIVEMQ_WEB_USER_SOMEONE_ROLES    | /activemq/web/user/someone/roles    | admin    |
 
 ## Logs
 
-| Path                            | Description    |
-| :------------------------------ | :------------- |
-| STDOUT                          | [ActiveMQ Log] |
-| /opt/activemq/data/activemq.log | [ActiveMQ Log] |
-| /opt/activemq/data/audit.log    | [Audit Log]    |
+- [ActiveMQ Log]
+- [Audit Log]
 
 [ActiveMQ Documentation]: https://activemq.apache.org/components/classic/documentation
 [ActiveMQ Log]: https://activemq.apache.org/how-do-i-change-the-logging

activemq/rootfs/etc/confd/conf.d/log4j.properties.toml

@@ -0,0 +1,7 @@
+[template]
+src = "log4j.properties.tmpl"
+dest = "/opt/activemq/conf/log4j.properties"
+uid = 100
+gid = 1000
+mode = "0640"
+keys = [ "/" ]

activemq/rootfs/etc/confd/templates/log4j.properties.tmpl

@@ -0,0 +1,67 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements.  See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License.  You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+#
+# This file controls most of the logging in ActiveMQ which is mainly based around
+# the commons logging API.
+#
+log4j.rootLogger={{ getenv "ACTIVEMQ_LOG_LEVEL" }}, console
+log4j.logger.org.apache.activemq.spring=WARN
+log4j.logger.org.apache.activemq.web.handler=WARN
+log4j.logger.org.springframework=WARN
+log4j.logger.org.apache.xbean=WARN
+log4j.logger.org.apache.camel=INFO
+log4j.logger.org.eclipse.jetty=WARN
+
+# When debugging or reporting problems to the ActiveMQ team,
+# comment out the above lines and uncomment the next.
+
+#log4j.rootLogger=DEBUG, logfile, console
+
+# Or for more fine grained debug logging uncomment one of these
+#log4j.logger.org.apache.activemq=DEBUG
+#log4j.logger.org.apache.camel=DEBUG
+
+# Console appender
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d | %-5p | %m | %c | %t%n
+log4j.appender.console.threshold={{ getenv "ACTIVEMQ_LOG_LEVEL" }}
+
+# use some of the following patterns to see MDC logging data
+#
+# %X{activemq.broker}
+# %X{activemq.connector}
+# %X{activemq.destination}
+#
+# e.g.
+#
+# log4j.appender.logfile.layout.ConversionPattern=%d | %-20.20X{activemq.connector} | %-5p | %m | %c | %t%n
+
+log4j.throwableRenderer=org.apache.log4j.EnhancedThrowableRenderer
+
+###########
+# Audit log
+###########
+
+log4j.additivity.org.apache.activemq.audit=false
+log4j.logger.org.apache.activemq.audit={{ getenv "ACTIVEMQ_AUDIT_LOG_LEVEL" }}, audit
+
+log4j.appender.audit=org.apache.log4j.ConsoleAppender
+log4j.appender.audit.layout=org.apache.log4j.PatternLayout
+log4j.appender.audit.layout.ConversionPattern=%d | %-5p | %m | %t%n
+log4j.appender.audit.threshold={{ getenv "ACTIVEMQ_AUDIT_LOG_LEVEL" }}

activemq/rootfs/etc/cont-init.d/02-activemq-setup.sh

@@ -1,13 +1,6 @@
 #!/usr/bin/with-contenv bash
 set -e
 
-# Change log files to redirect to stdout/stderr
-ln -sf /dev/stdout /opt/activemq/data/activemq.log
-chown activemq:activemq /opt/activemq/data/activemq.log
-
-ln -sf /dev/stdout /opt/activemq/data/audit.log
-chown activemq:activemq /opt/activemq/data/audit.log
-
 # When bind mounting we need to ensure that we
 # actually can write to the folder.
 chown activemq:activemq /opt/activemq/data

alpaca/README.md

@@ -23,7 +23,7 @@ additional settings, volumes, ports, etc.
 
 ## Settings
 
-| Environment Variable                       | Confd Key                                   | Default                                              | Description                                                                                                                                                 |
+| Environment Variable                       | Confd Key                                  | Default                                              | Description                                                                                                                                                 |
 | :----------------------------------------- | :----------------------------------------- | :--------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | ALPACA_ACTIVEMQ_PASSWORD                   | /alpaca/activemq/password                  | password                                             | Password to authenticate with                                                                                                                               |
 | ALPACA_ACTIVEMQ_URL                        | /alpaca/activemq/url                       | tcp://broker:61616                                   | The url for connecting to the ActiveMQ broker, shared by all components                                                                                     |
@@ -61,13 +61,6 @@ additional settings, volumes, ports, etc.
 | ALPACA_OCR_REDELIVERIES                    | /alpaca/ocr/redeliveries                   | 10                                                   | Number of attempts to redeliver if an exception occurs                                                                                                      |
 | ALPACA_OCR_SERVICE                         | /alpaca/ocr/service                        | http://hypercube:8000                                | Url of micro-service                                                                                                                                        |
 
-## Logs
-
-| Path                              | Description   |
-| :-------------------------------- | :------------ |
-| /opt/karaf/data/log/camel.log     | Camel Log     |
-| /opt/karaf/data/log/islandora.log | Islandora Log |
-
 [Alpaca Documentation]: https://islandora.github.io/documentation/
 [Alpaca]: https://github.com/Islandora/Alpaca
 [JMX]: https://karaf.apache.org/manual/latest/#_monitoring_and_management_using_jmx

alpaca/rootfs/etc/confd/templates/org.ops4j.pax.logging.cfg.tmpl

@@ -1,34 +1,18 @@
 # Root logger
 log4j.rootLogger={{ getenv "ALPACA_LOGGER_ROOT_LEVEL" }}, out, osgi:*
 log4j.throwableRenderer=org.apache.log4j.OsgiThrowableRenderer
-
-# File appender
-log4j.appender.out=org.apache.log4j.RollingFileAppender
+log4j.appender.out=org.apache.log4j.ConsoleAppender
 log4j.appender.out.layout=org.apache.log4j.PatternLayout
 log4j.appender.out.layout.ConversionPattern=%d{ISO8601} | %-5.5p | %-16.16t | %-32.32c{1} | %X{bundle.id} - %X{bundle.name} - %X{bundle.version} | %m%n
-log4j.appender.out.file=${karaf.data}/log/karaf.log
-log4j.appender.out.append=true
-log4j.appender.out.maxFileSize=1MB
-log4j.appender.out.maxBackupIndex=10
 
 # Camel Logger
-log4j.appender.camel=org.apache.log4j.RollingFileAppender
+log4j.logger.org.apache.camel={{ getenv "ALPACA_LOGGER_CAMEL_LEVEL" }}, camel
+log4j.appender.camel=org.apache.log4j.ConsoleAppender
 log4j.appender.camel.layout=org.apache.log4j.PatternLayout
 log4j.appender.camel.layout.ConversionPattern=%d{ISO8601} | %-5.5p | %-16.16t | %-32.32c{1} | %X{bundle.id} - %X{bundle.name} - %X{bundle.version} | %m%n
-log4j.appender.camel.file=${karaf.data}/log/camel.log
-log4j.appender.camel.append=false
-log4j.appender.camel.maxFileSize=1MB
-log4j.appender.camel.maxBackupIndex=10
-
-log4j.logger.org.apache.camel={{ getenv "ALPACA_LOGGER_CAMEL_LEVEL" }}, camel
 
 # Islandora Logger
-log4j.appender.islandora=org.apache.log4j.RollingFileAppender
+log4j.logger.ca.islandora.camel={{ getenv "ALPACA_LOGGER_ISLANDORA_LEVEL" }}, islandora
+log4j.appender.islandora=org.apache.log4j.ConsoleAppender
 log4j.appender.islandora.layout=org.apache.log4j.PatternLayout
 log4j.appender.islandora.layout.ConversionPattern=%d{ISO8601} | %-5.5p | %-16.16t | %-32.32c{1} | %X{bundle.id} - %X{bundle.name} - %X{bundle.version} | %m%n
-log4j.appender.islandora.file=${karaf.data}/log/islandora.log
-log4j.appender.islandora.append=false
-log4j.appender.islandora.maxFileSize=1MB
-log4j.appender.islandora.maxBackupIndex=10
-
-log4j.logger.ca.islandora.camel={{ getenv "ALPACA_LOGGER_ISLANDORA_LEVEL" }}, islandora

base/rootfs/etc/cont-init.d/00-tty.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+# If the container is started without allocating a tty, i.e. without `-t`.
+# It can cause issues for non-root processes that want to write directly to
+# standard out.
+#
+# If a tty is allocated /dev/stdout will indirectly point to it /dev/pts/0.
+# This file allows members of the tty group to write to it.
+#
+# If no tty is allocated /dev/stdout will point to /proc/self/fd/1 which 
+# will be a pipe to the hosts users active terminal. This pipe is owned 
+# root with read/write access only permitted to the root user.
+#
+# To permit the containers to be started without `tty` we allow all users
+# to read/write to the stdout,stderr,stdin pipes.
+chmod o+rw /dev/std{in,out,err}

blazegraph/README.md

@@ -23,15 +23,5 @@ additional settings, volumes, ports, etc.
 | :---- | :--------------------------- |
 | /data | Location of the backing file |
 
-## Logs
-
-| Path                               | Description |
-| :--------------------------------- | :---------- |
-| /opt/tomcat/logs/rules.log         |             |
-| /opt/tomcat/logs/queryLog.csv      |             |
-| /opt/tomcat/logs/queryRunState.log |             |
-| /opt/tomcat/logs/solutions.csv     |             |
-| /opt/tomcat/logs/sparql.txt        |             |
-
 [Blazegraph Documentation]: https://github.com/blazegraph/database/wiki/About_Blazegraph
 [Blazegraph]: https://blazegraph.com/

blazegraph/rootfs/etc/cont-init.d/03-blazegraph-setup.sh

@@ -1,22 +1,6 @@
 #!/usr/bin/with-contenv bash
 set -e
 
-# Change log files to redirect to stdout/stderr
-ln -sf /dev/stdout /opt/tomcat/logs/rules.log
-chown tomcat:tomcat /opt/tomcat/logs/rules.log
-
-ln -sf /dev/stdout /opt/tomcat/logs/queryLog.csv
-chown tomcat:tomcat /dev/stdout /opt/tomcat/logs/queryLog.csv
-
-ln -sf /dev/stdout /opt/tomcat/logs/queryRunState.log
-chown tomcat:tomcat /dev/stdout /opt/tomcat/logs/queryRunState.log
-
-ln -sf /dev/stdout /opt/tomcat/logs/solutions.csv
-chown tomcat:tomcat /dev/stdout /opt/tomcat/logs/solutions.csv
-
-ln -sf /dev/stdout /opt/tomcat/logs/sparql.txt
-chown tomcat:tomcat /dev/stdout /opt/tomcat/logs/sparql.txt
-
 # When bind mounting we need to ensure that we
 # actually can write to the folder.
 chown tomcat:tomcat /data

blazegraph/rootfs/opt/tomcat/webapps/bigdata/WEB-INF/classes/log4j.properties

@@ -34,10 +34,8 @@ log4j.appender.dest2.layout.ConversionPattern=%-5p: %r %X{hostname} %X{serviceUU
 # Rule execution log. This is a formatted log file (comma delimited).
 log4j.logger.com.bigdata.relation.rule.eval.RuleLog=INFO,ruleLog
 log4j.additivity.com.bigdata.relation.rule.eval.RuleLog=false
-log4j.appender.ruleLog=org.apache.log4j.FileAppender
+log4j.appender.ruleLog=org.apache.log4j.ConsoleAppender
 log4j.appender.ruleLog.Threshold=ALL
-log4j.appender.ruleLog.File=/opt/tomcat/logs/rules.log
-log4j.appender.ruleLog.Append=true
 # I find that it is nicer to have this unbuffered since you can see what
 # is going on and to make sure that I have complete rule evaluation logs
 # on shutdown.
@@ -49,10 +47,8 @@ log4j.appender.ruleLog.layout.ConversionPattern=%m
 # Summary query evaluation log (tab delimited file). Uncomment the next line to enable.
 #log4j.logger.com.bigdata.bop.engine.QueryLog=INFO,queryLog
 log4j.additivity.com.bigdata.bop.engine.QueryLog=false
-log4j.appender.queryLog=org.apache.log4j.FileAppender
+log4j.appender.queryLog=org.apache.log4j.ConsoleAppender
 log4j.appender.queryLog.Threshold=ALL
-log4j.appender.queryLog.File=/opt/tomcat/logs/queryLog.csv
-log4j.appender.queryLog.Append=true
 # I find that it is nicer to have this unbuffered since you can see what
 # is going on and to make sure that I have complete rule evaluation logs
 # on shutdown.
@@ -64,10 +60,8 @@ log4j.appender.queryLog.layout.ConversionPattern=%m
 # BOp run state trace (tab delimited file).  Uncomment the next line to enable.
 #log4j.logger.com.bigdata.bop.engine.RunState$TableLog=INFO,queryRunStateLog
 log4j.additivity.com.bigdata.bop.engine.RunState$TableLog=false
-log4j.appender.queryRunStateLog=org.apache.log4j.FileAppender
+log4j.appender.queryRunStateLog=org.apache.log4j.ConsoleAppender
 log4j.appender.queryRunStateLog.Threshold=ALL
-log4j.appender.queryRunStateLog.File=/opt/tomcat/logs/queryRunState.log
-log4j.appender.queryRunStateLog.Append=true
 # I find that it is nicer to have this unbuffered since you can see what
 # is going on and to make sure that I have complete rule evaluation logs
 # on shutdown.
@@ -80,10 +74,8 @@ log4j.appender.queryRunStateLog.layout.ConversionPattern=%m
 #log4j.logger.com.bigdata.bop.engine.SolutionsLog=INFO,solutionsLog
 log4j.additivity.com.bigdata.bop.engine.SolutionsLog=false
 log4j.appender.solutionsLog=org.apache.log4j.ConsoleAppender
-#log4j.appender.solutionsLog=org.apache.log4j.FileAppender
+#log4j.appender.solutionsLog=org.apache.log4j.ConsoleAppender
 log4j.appender.solutionsLog.Threshold=ALL
-log4j.appender.solutionsLog.File=/opt/tomcat/logs/solutions.csv
-log4j.appender.solutionsLog.Append=true
 # I find that it is nicer to have this unbuffered since you can see what
 # is going on and to make sure that I have complete rule evaluation logs
 # on shutdown.
@@ -97,10 +89,8 @@ log4j.logger.com.bigdata.rdf.sparql.ast.eval.ASTEvalHelper=WARN
 #log4j.logger.com.bigdata.rdf.sparql.ast.eval.ASTEvalHelper=INFO,sparqlLog
 log4j.additivity.com.bigdata.rdf.sparql.ast.eval.ASTEvalHelper=false
 log4j.appender.sparqlLog=org.apache.log4j.ConsoleAppender
-#log4j.appender.sparqlLog=org.apache.log4j.FileAppender
+#log4j.appender.sparqlLog=org.apache.log4j.ConsoleAppender
 log4j.appender.sparqlLog.Threshold=ALL
-log4j.appender.sparqlLog.File=/opt/tomcat/logs/sparql.txt
-log4j.appender.sparqlLog.Append=true
 # I find that it is nicer to have this unbuffered since you can see what
 # is going on and to make sure that I have complete rule evaluation logs
 # on shutdown.

download/rootfs/usr/local/bin/create-service-user.sh

@@ -75,6 +75,8 @@ function main {
     addgroup ${group}
     # Users that run services should permit login / do not require passwords.
     adduser --system --disabled-password --no-create-home --ingroup ${group} --shell /sbin/nologin --home ${install_directory} ${user}
+    # User also needs to be a member of tty to write directly to /dev/stdout, etc.
+    addgroup ${user} tty
     chown ${user}:${group} ${install_directory}
 }
 main

fcrepo/rootfs/etc/cont-init.d/03-fcrepo-setup.sh

@@ -56,14 +56,7 @@ function setup_persistence_backend {
     esac
 }
 
-# Change log files to redirect to stdout/stderr
-function redirect_logs_to_stdout { 
-    ln -sf /dev/stdout /opt/tomcat/logs/velocity.log
-    chown tomcat:tomcat /opt/tomcat/logs/velocity.log
-}
-
 function main {
-    redirect_logs_to_stdout
     setup_persistence_backend
     # When bind mounting we need to ensure that we
     # actually can write to the folder.

fcrepo/rootfs/opt/tomcat/bin/setenv.sh

@@ -2,7 +2,7 @@
 export JAVA_OPTS="${TOMCAT_JAVA_OPTS}"
 export CATALINA_OPTS="${TOMCAT_CATALINA_OPTS}"
 export CATALINA_OPTS="${CATALINA_OPTS} -Dfcrepo.home=/data/home"
-export CATALINA_OPTS="${CATALINA_OPTS} -Dfcrepo.velocity.runtime.log=/opt/tomcat/logs/velocity.log"
+export CATALINA_OPTS="${CATALINA_OPTS} -Dfcrepo.velocity.runtime.log=/dev/stdout"
 export CATALINA_OPTS="${CATALINA_OPTS} -Dfcrepo.modeshape.configuration=${FCREPO_MODESHAPE_CONFIGURATION} -Dfcrepo.jms.baseUrl=http://${HOSTNAME}/fcrepo/rest"
 if [[ "${FCREPO_DISABLE_SYN}" == "true" ]]; then
   export CATALINA_OPTS="${CATALINA_OPTS} -Dfcrepo.properties.management=relaxed"