From 675b1c1e646a571f572d5471af2d22cfeb1f9004 Mon Sep 17 00:00:00 2001
From: David Furey <david.furey@guardian.co.uk>
Date: Wed, 13 Dec 2023 14:05:10 +0000
Subject: [PATCH] Use DevX logs instead of kinesis-logback-appender

This decouples the application from the logging, which avoids the application falling over if there is ever an issue pushing logging events to kinesis.

Also attempts to standardise logback.xml a little
---
 app/AppComponents.scala                  |  4 +-
 app/com/gu/viewer/logging/Loggable.scala | 49 +-----------------------
 build.sbt                                |  3 +-
 conf/logback.xml                         | 34 ++++++----------
 riff-raff.yaml                           |  2 +-
 5 files changed, 16 insertions(+), 76 deletions(-)

diff --git a/app/AppComponents.scala b/app/AppComponents.scala
index 6751817..9729ed2 100644
--- a/app/AppComponents.scala
+++ b/app/AppComponents.scala
@@ -8,7 +8,7 @@ import com.gu.pandomainauth.PanDomainAuthSettingsRefresher
 import com.gu.viewer.aws.AwsInstanceTags
 import com.gu.viewer.config.AppConfig
 import com.gu.viewer.controllers.{Application, Email, Management, Proxy}
-import com.gu.viewer.logging.{LogStash, RequestLoggingFilter}
+import com.gu.viewer.logging.RequestLoggingFilter
 import com.gu.viewer.proxy.{LiveProxy, PreviewProxy, ProxyClient}
 import controllers.AssetsComponents
 import play.api.{BuiltInComponentsFromContext, Mode}
@@ -42,8 +42,6 @@ class AppComponents(context: Context)
   val tags = new AwsInstanceTags(ec2Client)
   val config = new AppConfig(tags, context.initialConfiguration)
 
-  if (context.environment.mode != Mode.Dev) LogStash.init(config, tags, region)
-
   val panDomainSettings: PanDomainAuthSettingsRefresher = new PanDomainAuthSettingsRefresher(
     domain = config.pandaDomain,
     system = "viewer",
diff --git a/app/com/gu/viewer/logging/Loggable.scala b/app/com/gu/viewer/logging/Loggable.scala
index 0197b92..c82ec96 100644
--- a/app/com/gu/viewer/logging/Loggable.scala
+++ b/app/com/gu/viewer/logging/Loggable.scala
@@ -1,52 +1,5 @@
 package com.gu.viewer.logging
 
-import ch.qos.logback.classic.filter.ThresholdFilter
-import ch.qos.logback.classic.{Logger => LogbackLogger}
-import ch.qos.logback.classic.spi.ILoggingEvent
-import com.amazonaws.auth.InstanceProfileCredentialsProvider
-import com.amazonaws.regions.Regions
-import com.gu.logback.appender.kinesis.KinesisAppender
-import com.gu.viewer.aws.AwsInstanceTags
-import com.gu.viewer.config.AppConfig
-import net.logstash.logback.layout.LogstashLayout
-import org.slf4j.{LoggerFactory, Logger => SLFLogger}
-
 trait Loggable {
   val log = play.api.Logger(getClass)
-}
-
-object LogStash {
-  private val rootLogger = LoggerFactory.getLogger(SLFLogger.ROOT_LOGGER_NAME).asInstanceOf[LogbackLogger]
-
-  def init(config: AppConfig, tags: AwsInstanceTags, region: Regions): Unit = {
-    for {
-      stack <- tags.readTag("Stack")
-      app <- tags.readTag("App")
-      stage <- tags.readTag("Stage")
-      stream <- config.logstashKinesisStream
-    } {
-      val context = rootLogger.getLoggerContext
-
-      val layout = new LogstashLayout()
-      layout.setContext(context)
-      layout.setCustomFields(s"""{"stack":"$stack","app":"$app","stage":"$stage"}""")
-      layout.start()
-
-      val filter = new ThresholdFilter()
-      filter.setLevel("INFO")
-      filter.start()
-
-      val appender = new KinesisAppender[ILoggingEvent]()
-      appender.setBufferSize(1000)
-      appender.setRegion(region.getName)
-      appender.setStreamName(stream)
-      appender.setContext(context)
-      appender.setLayout(layout)
-      appender.setCredentialsProvider(InstanceProfileCredentialsProvider.getInstance())
-      appender.addFilter(filter)
-      appender.start()
-
-      rootLogger.addAppender(appender)
-    }
-  }
-}
+}
\ No newline at end of file
diff --git a/build.sbt b/build.sbt
index eb224b3..a692603 100644
--- a/build.sbt
+++ b/build.sbt
@@ -33,8 +33,7 @@ libraryDependencies ++= Seq(
   "com.amazonaws" % "aws-java-sdk-ec2" % awsVersion,
   "com.amazonaws" % "aws-java-sdk-ses" % awsVersion,
   "com.gu" %% "pan-domain-auth-play_2-8" % "1.0.6",
-  "net.logstash.logback" % "logstash-logback-encoder" % "4.5.1",
-  "com.gu" % "kinesis-logback-appender" % "1.3.0",
+  "net.logstash.logback" % "logstash-logback-encoder" % "7.2",
   ws,
   "com.typesafe.play" %% "play-iteratees" % "2.6.1",
   "com.google.guava" % "guava" % "27.0-jre"
diff --git a/conf/logback.xml b/conf/logback.xml
index af7ae4f..0691cac 100644
--- a/conf/logback.xml
+++ b/conf/logback.xml
@@ -2,41 +2,31 @@
 
   <conversionRule conversionWord="coloredLevel" converterClass="play.api.libs.logback.ColoredLevel" />
 
-  <!-- Log only WARN and ERROR to stderr-->
-  <appender name="STDERR" class="ch.qos.logback.core.ConsoleAppender">
-    <target>System.err</target>
-    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-      <level>WARN</level>
-    </filter>
-    <encoder>
-      <pattern>%coloredLevel - %logger - %message%n%xException</pattern>
-    </encoder>
-  </appender>
-
   <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-    <file>logs/viewer.log</file>
+    <file>logs/application.log</file>
 
     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-      <fileNamePattern>logs/viewer.log.%d{yyyy-MM-dd}.gz</fileNamePattern>
+      <fileNamePattern>logs/application.log.%d{yyyy-MM-dd}.gz</fileNamePattern>
       <maxHistory>2</maxHistory>
     </rollingPolicy>
 
     <encoder>
-      <pattern>%date [%thread{10}] %-5level %logger{20} - %msg%n MDC[%mdc]%n MARKER[%marker]%n%xException{20}%n</pattern>
+      <pattern>%date - [%level] - from %logger in %thread %n%message%n%xException%n</pattern>
     </encoder>
   </appender>
 
-  <!--
-    The logger name is typically the Java/Scala package name.
-    This configures the log level to log at for a package and its children packages.
-  -->
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
+  </appender>
+  <appender name="ASYNCSTDOUT" class="ch.qos.logback.classic.AsyncAppender">
+    <appender-ref ref="STDOUT" />
+  </appender>
   <logger name="play" level="INFO" />
   <logger name="application" level="DEBUG" />
   <logger name="com.gu.viewer.logging" level="DEBUG" />
 
   <root level="INFO">
-    <appender-ref ref="STDERR" />
-    <appender-ref ref="LOGFILE" />
+    <appender-ref ref="LOGFILE"/>
+    <appender-ref ref="ASYNCSTDOUT"/>
   </root>
-
-</configuration>
+</configuration>
\ No newline at end of file
diff --git a/riff-raff.yaml b/riff-raff.yaml
index cb13413..a6c156a 100644
--- a/riff-raff.yaml
+++ b/riff-raff.yaml
@@ -10,7 +10,7 @@ deployments:
       type: ami-cloudformation-parameter
       parameters:
         amiTags:
-          Recipe: editorial-tools-focal-java8-ARM
+          Recipe: editorial-tools-focal-java8-ARM-WITH-cdk-base
           AmigoStage: PROD
           BuiltBy: amigo
         amiEncrypted: true