Skip to content

Commit

Permalink
[Backport] Add cli option & system property to enable json only loggi…
Browse files Browse the repository at this point in the history
…ng for the json api (#11717)

* Add cli option & system property to enable json only logging for http json service (#9725)

changelog_begin
- [Ledger HTTP Json Service] Logging output can now be in JSON either via providing the cli option `--log-encoder json` or via setting the env var `LOG_FORMAT_JSON=true`
changelog_end

* Load the correct logback file for the http json service respecting the deployment situation (#9938)

* Load the correct logback file for the http json service respecting the deployment situation

changelog_begin

[HTTP-JSON]
- fixed that json log output could not be enabled via cli options except via usage of env vars

changelog_end

* Move import statement, remove some braces and reformat

* Move system prop dependend logic to cliopts Logging.scala

* Remove PathKind type in cliopts Logging.scala as it's not necessary anymore
  • Loading branch information
realvictorprm authored Nov 18, 2021
1 parent 17cacb0 commit b7fc305
Show file tree
Hide file tree
Showing 34 changed files with 320 additions and 76 deletions.
3 changes: 2 additions & 1 deletion bazel-java-deps.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ def install_java_deps():
"javax.ws.rs:javax.ws.rs-api:2.1",
"junit:junit:4.12",
"junit:junit-dep:4.10",
"net.logstash.logback:logstash-logback-encoder:6.3",
"net.logstash.logback:logstash-logback-encoder:6.6",
"org.codehaus.janino:janino:3.1.4",
"org.apache.commons:commons-lang3:3.9",
"org.apache.commons:commons-text:1.4",
"org.awaitility:awaitility:3.1.6",
Expand Down
1 change: 1 addition & 0 deletions daml-script/test/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ da_scala_test_suite(
"//language-support/scala/bindings-akka",
"//ledger-api/rs-grpc-bridge",
"//ledger-api/testing-utils",
"//ledger-service/cli-opts",
"//ledger-service/http-json",
"//ledger-service/jwt",
"//ledger-service/lf-value-json",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import akka.actor.ActorSystem
import akka.http.scaladsl.Http.ServerBinding
import akka.stream.Materializer
import com.daml.bazeltools.BazelRunfiles._
import com.daml.cliopts.Logging.LogEncoder
import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
import com.daml.http.util.Logging.{InstanceUUID, instanceUUIDLogCtx}
import com.daml.http.HttpService
Expand Down Expand Up @@ -150,6 +151,7 @@ trait JsonApiFixture
override val accessTokenFile = Some(jsonAccessTokenFile)
override val allowNonHttps = true
override val logLevel = None
override val logEncoder = LogEncoder.Plain
}
HttpService
.start(config)(
Expand Down
2 changes: 1 addition & 1 deletion daml-script/test/src/main/resources/logback-test.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
8 changes: 8 additions & 0 deletions ledger-service/cli-opts/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,15 @@ da_scala_library(
],
scalacopts = lf_scalacopts,
tags = ["maven_coordinates=com.daml:http-json-cli-opts:__VERSION__"],
versioned_scala_deps = {
"2.12": [
"@maven//:org_scala_lang_modules_scala_collection_compat",
],
},
visibility = ["//visibility:public"],
runtime_deps = [
"@maven//:org_codehaus_janino_janino",
],
deps = [
"//ledger/ledger-api-common",
"@maven//:ch_qos_logback_logback_classic",
Expand Down
71 changes: 70 additions & 1 deletion ledger-service/cli-opts/src/main/scala/cliopts/Logging.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,47 @@ import ch.qos.logback.classic.{Level => LogLevel}

object Logging {

def reconfigure(clazz: Class[_]): Unit = {
// Try reconfiguring the library
import ch.qos.logback.core.joran.spi.JoranException
import ch.qos.logback.classic.LoggerContext
import ch.qos.logback.classic.joran.JoranConfigurator
import org.slf4j.LoggerFactory
import scala.util.Using
import java.io.InputStream
import java.io.FileInputStream
def reloadConfig(path: String, openStream: String => InputStream): Unit =
Using.resource(openStream(path)) { stream =>
try {
val context = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]
val configurator = new JoranConfigurator()
configurator.setContext(context)
context.reset()
configurator.doConfigure(stream)
} catch {
case je: JoranException =>
// Fallback to System.err.println because the logger won't work in any way anymore.
System.err.println(
s"reconfigured failed using url $path: $je"
)
je.printStackTrace(System.err)
} finally stream.close()
}
System.getProperty("logback.configurationFile") match {
case null => reloadConfig("logback.xml", clazz.getClassLoader.getResource(_).openStream())
case path => reloadConfig(path, new FileInputStream(_))
}
}

private val KnownLogLevels = Set("ERROR", "WARN", "INFO", "DEBUG", "TRACE")

sealed trait LogEncoder

object LogEncoder {
case object Plain extends LogEncoder
case object Json extends LogEncoder
}

private implicit val scoptLogLevel: scopt.Read[LogLevel] = scopt.Read.reads { level =>
Either
.cond(
Expand All @@ -19,9 +58,19 @@ object Logging {
.getOrElse(throw new java.lang.IllegalArgumentException(s"Unknown logging level $level"))
}

private implicit val scoptLogEncoder: scopt.Read[LogEncoder] =
scopt.Read.reads { encoder =>
encoder.toLowerCase match {
case "plain" => LogEncoder.Plain
case "json" => LogEncoder.Json
case _ =>
throw new java.lang.IllegalArgumentException(s"Unrecognized logging encoder $encoder")
}
}

/** Parse in the cli option for the logging level.
*/
def loggingLevelParse[C](
def logLevelParse[C](
parser: scopt.OptionParser[C]
)(logLevel: Setter[C, Option[LogLevel]]): Unit = {
import parser.opt
Expand All @@ -34,4 +83,24 @@ object Logging {
)
()
}

def logEncoderParse[C](
parser: scopt.OptionParser[C]
)(logEncoder: Setter[C, LogEncoder]): Unit = {
import parser.opt

opt[LogEncoder]("log-encoder")
.optional()
.action {
case (LogEncoder.Plain, c) => c
case (encoder, c) => logEncoder(_ => encoder, c)
}
.text("Which encoder to use: plain|json")
()
}

def setUseJsonLogEncoderSystemProp(): Unit = {
System.setProperty("LOG_FORMAT_JSON", "true")
()
}
}
1 change: 1 addition & 0 deletions ledger-service/http-json-testing/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ da_scala_library(
"//bazel_tools/runfiles:scala_runfiles",
"//language-support/scala/bindings-akka",
"//ledger-api/rs-grpc-bridge",
"//ledger-service/cli-opts",
"//ledger-service/http-json",
"//ledger-service/jwt",
"//ledger-service/utils",
Expand Down
1 change: 1 addition & 0 deletions ledger-service/http-json/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ da_scala_library(
visibility = ["//visibility:public"],
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
"@maven//:org_codehaus_janino_janino",
],
deps = [
"//daml-lf/data",
Expand Down
2 changes: 1 addition & 1 deletion ledger-service/http-json/src/failure/resources/logback.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
2 changes: 1 addition & 1 deletion ledger-service/http-json/src/it/resources/logback.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
20 changes: 14 additions & 6 deletions ledger-service/http-json/src/main/resources/logback.xml
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
<if condition='isDefined("LOG_FORMAT_JSON")'>
<then>
<encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
</then>
<else>
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</else>
</if>
</appender>

<logger name="io.netty" level="WARN" />
Expand All @@ -14,4 +22,4 @@
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
</configuration>
</configuration>
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ object Cli extends StrictLogging {
.valueName(WebsocketConfig.usage)
.text(s"Optional websocket configuration string. ${WebsocketConfig.help}")

cliopts.Logging.loggingLevelParse(this)((f, c) => c.copy(logLevel = f(c.logLevel)))
cliopts.Logging.logLevelParse(this)((f, c) => c.copy(logLevel = f(c.logLevel)))
cliopts.Logging.logEncoderParse(this)((f, c) => c.copy(logEncoder = f(c.logEncoder)))

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import scala.concurrent.duration._
import scala.util.Try

import ch.qos.logback.classic.{Level => LogLevel}
import com.daml.cliopts.Logging.LogEncoder

// The internal transient scopt structure *and* StartSettings; external `start`
// users should extend StartSettings or DefaultStartSettings themselves
Expand All @@ -38,6 +39,7 @@ private[http] final case class Config(
accessTokenFile: Option[Path] = None,
wsConfig: Option[WebsocketConfig] = None,
logLevel: Option[LogLevel] = None, // the default is in logback.xml
logEncoder: LogEncoder = LogEncoder.Plain,
) extends HttpService.StartSettings

private[http] object Config {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import scala.concurrent.{Await, ExecutionContext, Future}
import scala.util.control.NonFatal

import ch.qos.logback.classic.{Level => LogLevel}
import com.daml.cliopts.Logging.LogEncoder

object HttpService {

Expand Down Expand Up @@ -83,6 +84,7 @@ object HttpService {
val maxInboundMessageSize: Int
val healthTimeoutSeconds: Int
val logLevel: Option[LogLevel]
val logEncoder: LogEncoder
}

trait DefaultStartSettings extends StartSettings {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ import java.nio.file.Path
import akka.actor.ActorSystem
import akka.http.scaladsl.Http.ServerBinding
import akka.stream.Materializer
import com.daml.cliopts.Logging.LogEncoder
import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
import com.daml.scalautil.Statement.discard
import com.daml.http.dbbackend.ContractDao
import scalaz.{-\/, \/, \/-}
import scalaz.std.anyVal._
import scalaz.std.option._
import scalaz.syntax.show._
import com.daml.cliopts.GlobalLogLevel
import com.daml.cliopts.{GlobalLogLevel, Logging}
import com.daml.http.util.Logging.{InstanceUUID, instanceUUIDLogCtx}
import com.daml.logging.{ContextualizedLogger, LoggingContextOf}

Expand All @@ -32,10 +33,27 @@ object Main {
val StartupError = 101
}

def adjustAndReloadLoggingOptions(
config: Config
): Unit = {
// If the system property was explicitly set before application startup and the config option was provided,
// the prior value will be overridden here.
config.logEncoder match {
case LogEncoder.Plain => () // This is the default
case LogEncoder.Json =>
Logging.setUseJsonLogEncoderSystemProp()
Logging.reconfigure(getClass)
}
// Here we set all things which are related to logging but not to
// any env vars in the logback.xml file.
config.logLevel.foreach(GlobalLogLevel.set("Ledger HTTP-JSON API"))
}

def main(args: Array[String]): Unit = {
instanceUUIDLogCtx(implicit lc =>
Cli.parseConfig(args) match {
case Some(config) =>
adjustAndReloadLoggingOptions(config)
main(config)
case None =>
// error is printed out by scopt
Expand All @@ -45,7 +63,6 @@ object Main {
}

private def main(config: Config)(implicit lc: LoggingContextOf[InstanceUUID]): Unit = {
config.logLevel.foreach(GlobalLogLevel.set("Ledger HTTP-JSON API"))
logger.info(
s"Config(ledgerHost=${config.ledgerHost: String}, ledgerPort=${config.ledgerPort: Int}" +
s", address=${config.address: String}, httpPort=${config.httpPort: Int}" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%level %logger{10}: %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
2 changes: 1 addition & 1 deletion ledger/ledger-on-sql/src/test/resources/logback-test.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%level %logger{10}: %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<included>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %logger{5} - %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%level %logger{10}: %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %logger{5} - %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %level %logger{10}: %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
2 changes: 1 addition & 1 deletion ledger/sandbox-common/src/main/resources/logback.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%level: %msg%n</pattern>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
</encoder>
</appender>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ class CommonCli(name: LedgerName) {
.text("This flag is deprecated -- please use --sql-backend-jdbcurl.")
.action((url, config) => config.copy(jdbcUrl = Some(url)))

com.daml.cliopts.Logging.loggingLevelParse(this)((f, c) => c.copy(logLevel = f(c.logLevel)))
com.daml.cliopts.Logging.logLevelParse(this)((f, c) => c.copy(logLevel = f(c.logLevel)))

opt[Unit]("eager-package-loading")
.optional()
Expand Down
Loading

0 comments on commit b7fc305

Please sign in to comment.