Add cli option & system property to enable json only logging for http…

… json service

changelog_begin
- [Ledger HTTP Json Service] Logging output can now be in JSON either via providing the cli option `--log-encoder json` or via setting the env var `LOG_FORMAT_JSON=true`
changelog_end

bazel-java-deps.bzl

@@ -135,7 +135,8 @@ def install_java_deps():
             "javax.ws.rs:javax.ws.rs-api:2.1",
             "junit:junit:4.12",
             "junit:junit-dep:4.10",
-            "net.logstash.logback:logstash-logback-encoder:6.3",
+            "net.logstash.logback:logstash-logback-encoder:6.6",
+            "org.codehaus.janino:janino:3.1.4",
             "org.apache.commons:commons-lang3:3.9",
             "org.apache.commons:commons-text:1.4",
             "org.awaitility:awaitility:3.1.6",

daml-script/test/BUILD.bazel

@@ -184,6 +184,7 @@ da_scala_test_suite(
         "//language-support/scala/bindings-akka",
         "//ledger-api/rs-grpc-bridge",
         "//ledger-api/testing-utils",
+        "//ledger-service/cli-opts",
         "//ledger-service/http-json:http-json-ce",
         "//ledger-service/http-json-cli:ce",
         "//ledger-service/jwt",

daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/JsonApiIt.scala

@@ -12,6 +12,7 @@ import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.server.Directives._
 import akka.stream.Materializer
 import com.daml.bazeltools.BazelRunfiles._
+import com.daml.cliopts.Logging.LogEncoder
 import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
 import com.daml.http.util.Logging.{InstanceUUID, instanceUUIDLogCtx}
 import com.daml.http.{HttpService, StartSettings, nonrepudiation}
@@ -157,6 +158,7 @@ trait JsonApiFixture
                 override val allowNonHttps = true
                 override val nonRepudiation = nonrepudiation.Configuration.Cli.Empty
                 override val logLevel = None
+                override val logEncoder = LogEncoder.Plain
               }
               HttpService
                 .start(config)(

daml-script/test/src/main/resources/logback-test.xml

@@ -2,7 +2,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger-service/cli-opts/BUILD.bazel

@@ -16,7 +16,15 @@ da_scala_library(
     ],
     scalacopts = lf_scalacopts,
     tags = ["maven_coordinates=com.daml:http-json-cli-opts:__VERSION__"],
+    versioned_scala_deps = {
+        "2.12": [
+            "@maven//:org_scala_lang_modules_scala_collection_compat",
+        ],
+    },
     visibility = ["//visibility:public"],
+    runtime_deps = [
+        "@maven//:org_codehaus_janino_janino",
+    ],
     deps = [
         "//ledger/ledger-api-common",
         "@maven//:ch_qos_logback_logback_classic",

ledger-service/cli-opts/src/main/scala/cliopts/Logging.scala

@@ -7,8 +7,41 @@ import ch.qos.logback.classic.{Level => LogLevel}
 
 object Logging {
 
+  def reconfigure(clazz: Class[_], pathToLogbackFileInJarFile: String): Unit = {
+    // Try reconfiguring the library
+    import ch.qos.logback.core.joran.spi.JoranException
+    import ch.qos.logback.classic.LoggerContext
+    import ch.qos.logback.classic.joran.JoranConfigurator
+    import org.slf4j.LoggerFactory
+    import scala.util.Using
+    Using.resource(clazz.getClassLoader.getResource(pathToLogbackFileInJarFile).openStream()) {
+      stream =>
+        try {
+          val context = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]
+          val configurator = new JoranConfigurator
+          configurator.setContext(context)
+          context.reset()
+          configurator.doConfigure(stream)
+        } catch {
+          case je: JoranException =>
+            // Fallback to System.err.println because the logger won't work in any way anymore.
+            System.err.println(s"reconfigured failed using url $pathToLogbackFileInJarFile: $je")
+            je.printStackTrace(System.err)
+        } finally {
+          stream.close()
+        }
+    }
+  }
+
   private val KnownLogLevels = Set("ERROR", "WARN", "INFO", "DEBUG", "TRACE")
 
+  sealed trait LogEncoder
+
+  object LogEncoder {
+    case object Plain extends LogEncoder
+    case object Json extends LogEncoder
+  }
+
   private implicit val scoptLogLevel: scopt.Read[LogLevel] = scopt.Read.reads { level =>
     Either
       .cond(
@@ -19,9 +52,19 @@ object Logging {
       .getOrElse(throw new java.lang.IllegalArgumentException(s"Unknown logging level $level"))
   }
 
+  private implicit val scoptLogEncoder: scopt.Read[LogEncoder] =
+    scopt.Read.reads { encoder =>
+      encoder.toLowerCase match {
+        case "plain" => LogEncoder.Plain
+        case "json" => LogEncoder.Json
+        case _ =>
+          throw new java.lang.IllegalArgumentException(s"Unrecognized logging encoder $encoder")
+      }
+    }
+
   /** Parse in the cli option for the logging level.
     */
-  def loggingLevelParse[C](
+  def logLevelParse[C](
       parser: scopt.OptionParser[C]
   )(logLevel: Setter[C, Option[LogLevel]]): Unit = {
     import parser.opt
@@ -34,4 +77,24 @@ object Logging {
       )
     ()
   }
+
+  def logEncoderParse[C](
+      parser: scopt.OptionParser[C]
+  )(logEncoder: Setter[C, LogEncoder]): Unit = {
+    import parser.opt
+
+    opt[LogEncoder]("log-encoder")
+      .optional()
+      .action {
+        case (LogEncoder.Plain, c) => c
+        case (encoder, c) => logEncoder(_ => encoder, c)
+      }
+      .text("Which encoder to use: plain|json")
+    ()
+  }
+
+  def setUseJsonLogEncoderSystemProp(): Unit = {
+    System.setProperty("LOG_FORMAT_JSON", "true")
+    ()
+  }
 }

ledger-service/http-json-cli/BUILD.bazel

@@ -38,6 +38,7 @@ da_scala_library(
         exports = [":base"],
         deps = [
             ":base",
+            "//ledger-service/cli-opts",
             "//ledger/ledger-api-common",
         ],
     )

ledger-service/http-json-cli/src/main/scala/com/daml/http/CliBase.scala

@@ -3,9 +3,8 @@
 
 package com.daml.http
 
-import com.typesafe.scalalogging.StrictLogging
+trait CliBase {
 
-trait CliBase extends StrictLogging {
   private[http] def parseConfig(
       args: collection.Seq[String],
       supportedJdbcDriverNames: Set[String],

ledger-service/http-json-cli/src/main/scala/com/daml/http/Config.scala

@@ -18,6 +18,7 @@ import scala.concurrent.duration._
 import scala.util.Try
 
 import ch.qos.logback.classic.{Level => LogLevel}
+import com.daml.cliopts.Logging.LogEncoder
 
 // The internal transient scopt structure *and* StartSettings; external `start`
 // users should extend StartSettings or DefaultStartSettings themselves
@@ -39,6 +40,7 @@ private[http] final case class Config(
     wsConfig: Option[WebsocketConfig] = None,
     nonRepudiation: nonrepudiation.Configuration.Cli = nonrepudiation.Configuration.Cli.Empty,
     logLevel: Option[LogLevel] = None, // the default is in logback.xml
+    logEncoder: LogEncoder = LogEncoder.Plain,
 ) extends StartSettings
 
 private[http] object Config {

ledger-service/http-json-cli/src/main/scala/com/daml/http/OptionParser.scala

@@ -166,5 +166,7 @@ class OptionParser(getEnvVar: String => Option[String], supportedJdbcDriverNames
     .valueName(WebsocketConfig.usage)
     .text(s"Optional websocket configuration string. ${WebsocketConfig.help}")
 
-  cliopts.Logging.loggingLevelParse(this)((f, c) => c.copy(logLevel = f(c.logLevel)))
+  cliopts.Logging.logLevelParse(this)((f, c) => c.copy(logLevel = f(c.logLevel)))
+  cliopts.Logging.logEncoderParse(this)((f, c) => c.copy(logEncoder = f(c.logEncoder)))
+
 }

ledger-service/http-json-cli/src/main/scala/com/daml/http/StartSettings.scala

@@ -10,6 +10,7 @@ import com.daml.ledger.api.tls.TlsConfiguration
 import scala.concurrent.duration.FiniteDuration
 
 import ch.qos.logback.classic.{Level => LogLevel}
+import com.daml.cliopts.Logging.LogEncoder
 
 // defined separately from Config so
 //  1. it is absolutely lexically apparent what `import startSettings._` means
@@ -31,6 +32,7 @@ trait StartSettings {
   val healthTimeoutSeconds: Int
   val nonRepudiation: nonrepudiation.Configuration.Cli
   val logLevel: Option[LogLevel]
+  val logEncoder: LogEncoder
 }
 
 object StartSettings {

ledger-service/http-json-testing/BUILD.bazel

@@ -49,6 +49,7 @@ hj_scalacopts = lf_scalacopts + [
             "//ledger-api/rs-grpc-bridge",
             "//ledger-service/http-json:http-json-{}".format(edition),
             "//ledger-service/http-json-cli:{}".format(edition),
+            "//ledger-service/cli-opts",
             "//ledger-service/jwt",
             "//ledger-service/utils",
             "//ledger/caching",

ledger-service/http-json/BUILD.bazel

@@ -47,6 +47,7 @@ hj_scalacopts = lf_scalacopts + [
         visibility = ["//visibility:public"],
         runtime_deps = [
             "@maven//:ch_qos_logback_logback_classic",
+            "@maven//:org_codehaus_janino_janino",
         ],
         deps = [
             "//runtime-components/jdbc-drivers:jdbc-drivers-{}".format(edition),
@@ -183,7 +184,6 @@ daml_compile(
             "@maven//:com_chuusai_shapeless",
             "@maven//:com_lihaoyi_sourcecode",
             "@maven//:com_typesafe_akka_akka_http_core",
-            "@maven//:com_typesafe_scala_logging_scala_logging",
             "@maven//:io_spray_spray_json",
             "@maven//:org_scala_lang_modules_scala_collection_compat",
             "@maven//:org_scalacheck_scalacheck",

ledger-service/http-json/src/failure/resources/logback.xml

@@ -3,7 +3,7 @@
         <!-- encoders are assigned the type
              ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
         <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger-service/http-json/src/it/resources/logback.xml

@@ -3,7 +3,7 @@
         <!-- encoders are assigned the type
              ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
         <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger-service/http-json/src/main/resources/logback.xml

@@ -1,10 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <!-- encoders are assigned the type
-             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
-        </encoder>
+        <if condition='isDefined("LOG_FORMAT_JSON")'>
+            <then>
+                <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+            </then>
+            <else>
+                <!-- encoders are assigned the type
+                    ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+                <encoder>
+                    <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
+                </encoder>
+            </else>
+        </if>
     </appender>
 
     <logger name="io.netty" level="WARN" />
@@ -14,4 +22,4 @@
     <root level="INFO">
         <appender-ref ref="STDOUT" />
     </root>
-</configuration>
+</configuration>

ledger-service/http-json/src/main/scala/com/digitalasset/http/Main.scala

@@ -7,6 +7,7 @@ import java.nio.file.Path
 import akka.actor.ActorSystem
 import akka.http.scaladsl.Http.ServerBinding
 import akka.stream.Materializer
+import com.daml.cliopts.Logging.LogEncoder
 import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
 import com.daml.runtime.JdbcDrivers
 import com.daml.scalautil.Statement.discard
@@ -15,7 +16,7 @@ import scalaz.{-\/, \/, \/-}
 import scalaz.std.anyVal._
 import scalaz.std.option._
 import scalaz.syntax.show._
-import com.daml.cliopts.GlobalLogLevel
+import com.daml.cliopts.{GlobalLogLevel, Logging}
 import com.daml.http.util.Logging.{InstanceUUID, instanceUUIDLogCtx}
 import com.daml.logging.{ContextualizedLogger, LoggingContextOf}
 
@@ -33,13 +34,30 @@ object Main {
     val StartupError = 101
   }
 
+  def adjustAndReloadLoggingOptions(
+      config: Config
+  ): Unit = {
+    // If the system property was explicitly set before application startup and the config option was provided,
+    // the prior value will be overridden here.
+    config.logEncoder match {
+      case LogEncoder.Plain => () // This is the default
+      case LogEncoder.Json =>
+        Logging.setUseJsonLogEncoderSystemProp()
+        Logging.reconfigure(getClass, "logback.xml")
+    }
+    // Here we set all things which are related to logging but not to
+    // any env vars in the logback.xml file.
+    config.logLevel.foreach(GlobalLogLevel.set("Ledger HTTP-JSON API"))
+  }
+
   def main(args: Array[String]): Unit = {
     instanceUUIDLogCtx(implicit lc =>
       Cli.parseConfig(
         args,
         ContractDao.supportedJdbcDriverNames(JdbcDrivers.availableJdbcDriverNames),
       ) match {
         case Some(config) =>
+          adjustAndReloadLoggingOptions(config)
           main(config)
         case None =>
           // error is printed out by scopt
@@ -49,7 +67,6 @@ object Main {
   }
 
   private def main(config: Config)(implicit lc: LoggingContextOf[InstanceUUID]): Unit = {
-    config.logLevel.foreach(GlobalLogLevel.set("Ledger HTTP-JSON API"))
     logger.info(
       s"Config(ledgerHost=${config.ledgerHost: String}, ledgerPort=${config.ledgerPort: Int}" +
         s", address=${config.address: String}, httpPort=${config.httpPort: Int}" +

ledger/indexer-benchmark/src/main/resources/logback.xml

@@ -7,7 +7,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%d{HH:mm:ss.SSS} %-5level %logger{5} - %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger/ledger-on-memory/src/test/resources/logback-test.xml

@@ -2,7 +2,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%level %logger{10}: %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger/ledger-on-sql/src/test/resources/logback-test.xml

@@ -2,7 +2,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%level %logger{10}: %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger/participant-integration-api/src/test/resources/logback-test.xml

@@ -2,7 +2,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger/participant-state/kvutils/app/src/main/resources/com/daml/ledger/participant/state/kvutils/app/logback.base.xml

@@ -7,7 +7,7 @@
 <included>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%d{HH:mm:ss.SSS} %-5level %logger{5} - %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger/participant-state/kvutils/src/test/resources/logback-test.xml

@@ -2,7 +2,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%level %logger{10}: %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>
 

ledger/participant-state/kvutils/tools/integrity-check/src/main/resources/logback.xml

@@ -7,7 +7,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%d{HH:mm:ss.SSS} %-5level %logger{5} - %msg%n</pattern>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %replace(, context: %marker){', context: $', ''} %n</pattern>
         </encoder>
     </appender>