Added new knet-log4j.properties for standard logging, updated KNetCLI…

…, KNetConnect and KNetPS to use less intrusive logs (#386)
masesgroup · Jan 27, 2024 · e33a6f2 · e33a6f2
1 parent 955b362
commit e33a6f2
Show file tree

Hide file tree

Showing 7 changed files with 168 additions and 13 deletions.
diff --git a/src/config/knet-log4j.properties b/src/config/knet-log4j.properties
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Unspecified loggers and loggers with additivity=true output to server.log and stdout
+# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
+log4j.rootLogger=INFO, kafkaAppender
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
+log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
+log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
+log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
+log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
+log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Change the line below to adjust ZK client logging
+log4j.logger.org.apache.zookeeper=INFO
+
+# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
+log4j.logger.kafka=INFO
+log4j.logger.org.apache.kafka=INFO
+
+# Change to DEBUG or TRACE to enable request logging
+log4j.logger.kafka.request.logger=WARN, requestAppender
+log4j.additivity.kafka.request.logger=false
+
+# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
+# related to the handling of requests
+#log4j.logger.kafka.network.Processor=TRACE, requestAppender
+#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
+#log4j.additivity.kafka.server.KafkaApis=false
+log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
+log4j.additivity.kafka.network.RequestChannel$=false
+
+# Change the line below to adjust KRaft mode controller logging
+log4j.logger.org.apache.kafka.controller=INFO, controllerAppender
+log4j.additivity.org.apache.kafka.controller=false
+
+# Change the line below to adjust ZK mode controller logging
+log4j.logger.kafka.controller=TRACE, controllerAppender
+log4j.additivity.kafka.controller=false
+
+log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
+log4j.additivity.kafka.log.LogCleaner=false
+
+log4j.logger.state.change.logger=INFO, stateChangeAppender
+log4j.additivity.state.change.logger=false
+
+# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses
+log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
+log4j.additivity.kafka.authorizer.logger=false
+
diff --git a/src/net/KNet/InternalConst.cs b/src/net/KNet/InternalConst.cs
@@ -48,13 +48,21 @@ public class Const
         /// </summary>
         public const string DefaultScalaVersion = "2.13.6";
         /// <summary>
+        /// Default path location of configuration files
+        /// </summary>
+        public static readonly string DefaultConfigurationPath = Path.Combine(AssemblyLocation, "config");
+        /// <summary>
+        /// Default path location of Jars files
+        /// </summary>
+        public static readonly string DefaultJarsPath = Path.Combine(AssemblyLocation, "jars");
+        /// <summary>
         /// Default root path, i.e. consider installation within bin folder
         /// </summary>
-        public static readonly string DefaultRootPath = Path.Combine(AssemblyLocation, "jars") + Path.DirectorySeparatorChar;
+        public static readonly string DefaultRootPath = DefaultJarsPath + Path.DirectorySeparatorChar;
         /// <summary>
-        /// Default log4j path, i.e. consider installation within bin folder
+        /// Default log4j configuration file, i.e. considering a relative location to <see cref="DefaultConfigurationPath"/>
         /// </summary>
-        public static readonly string DefaultLog4JPath = Path.Combine(AssemblyLocation, "config", "log4j.properties");
+        public static readonly string DefaultLog4JConfigurationPath = Path.Combine(DefaultConfigurationPath, "knet-log4j.properties");
         /// <summary>
         /// Default log path, i.e. consider installation within bin folder
         /// </summary>

diff --git a/src/net/KNet/KNet.csproj b/src/net/KNet/KNet.csproj
@@ -14,6 +14,25 @@
 		<RunAnalyzersDuringBuild>False</RunAnalyzersDuringBuild>
 		<RunAnalyzersDuringLiveAnalysis>False</RunAnalyzersDuringLiveAnalysis>
 	</PropertyGroup>
+
+	<ItemGroup Condition="True">
+		<knetcore_jars Include="$(ProjectDir)\..\..\..\jars\*.*" />
+		<knetcore_config Include="$(ProjectDir)\..\..\config\*.*" />
+		<knetcore_kraft Include="$(ProjectDir)\..\..\config\kraft\*.*" />
+	</ItemGroup>
+
+	<Target Name="CopyCustomContent" AfterTargets="AfterBuild" Condition="'$(GITHUB_ACTIONS)' != 'true'">
+		<Copy SourceFiles="@(knetcore_jars)" DestinationFolder="$(OutDir)\jars" />
+		<Copy SourceFiles="@(knetcore_config)" DestinationFolder="$(OutDir)\config" />
+		<Copy SourceFiles="@(knetcore_kraft)" DestinationFolder="$(OutDir)\config\kraft" />
+	</Target>
+	<Target Name="CopyCustomContentOnPublish" AfterTargets="Publish" Condition="'$(GITHUB_ACTIONS)' != 'true'">
+		<Copy SourceFiles="@(knetcore_jars)" DestinationFolder="$(OutDir)\jars" />
+		<Copy SourceFiles="@(knetcore_config)" DestinationFolder="$(OutDir)\config" />
+		<Copy SourceFiles="@(knetcore_kraft)" DestinationFolder="$(OutDir)\config\kraft" />
+	</Target>
+
+
 	<ItemGroup>
 		<None Include="mases.knet.targets" Pack="true" PackagePath="build" />
 		<None Include="..\..\..\jars\*.jar" Pack="true" PackagePath="build\jars" />

diff --git a/src/net/KNet/KNetCore.cs b/src/net/KNet/KNetCore.cs
@@ -66,7 +66,7 @@ public override IEnumerable<IArgumentMetadata> CommandLineArguments
                     new ArgumentMetadata<string>()
                     {
                         Name = CLIParam.Log4JConfiguration,
-                        Default = Const.DefaultLog4JPath,
+                        Default = DefaultLog4JConfiguration(),
                         Help = "The file containing the configuration of log4j.",
                     },
                     new ArgumentMetadata<string>()
@@ -99,6 +99,16 @@ public override IEnumerable<IArgumentMetadata> CommandLineArguments
                 return lst;
             }
         }
+        /// <summary>
+        /// Returns the default configuration file to use when initializing command line defaults
+        /// </summary>
+        /// <returns>The configuration file to use for logging</returns>
+        /// <remarks>Overrides in derived classes to give another default file</remarks>
+        protected virtual string DefaultLog4JConfiguration()
+        {
+            return Const.DefaultLog4JConfigurationPath;
+        }
+
         /// <summary>
         /// Public initializer
         /// </summary>
@@ -115,6 +125,18 @@ protected override string[] ProcessCommandLine()
             _classToRun = ParsedArgs.Get<string>(CLIParam.ClassToRun);
             _JarRootPath = ParsedArgs.Get<string>(CLIParam.KafkaLocation);
             _log4JPath = ParsedArgs.Get<string>(CLIParam.Log4JConfiguration);
+            if (!Path.IsPathRooted(_log4JPath)) // it is not a full path
+            {
+                var absolutePath = Path.Combine(Const.DefaultConfigurationPath, _log4JPath);
+                if (File.Exists(absolutePath))
+                {
+                    _log4JPath = absolutePath;
+                }
+                else
+                {
+                    throw new ArgumentException($"{_log4JPath} is not an absolute path and there is no file under {Const.DefaultConfigurationPath} whose absolute path is {absolutePath}");
+                }
+            }
             _logPath = ParsedArgs.Get<string>(CLIParam.LogPath);
             _scalaVersion = ParsedArgs.Get<string>(CLIParam.ScalaVersion);
             _disableJMX = ParsedArgs.Exist(CLIParam.DisableJMX);

diff --git a/src/net/KNetCLI/KNetCLICore.cs b/src/net/KNetCLI/KNetCLICore.cs
@@ -29,6 +29,11 @@ namespace MASES.KNetCLI
     public class KNetCLICore<T> : KNetCore<T>
         where T : KNetCLICore<T>
     {
+        protected override string DefaultLog4JConfiguration()
+        {
+            return Path.Combine(Const.DefaultConfigurationPath, "tools-log4j.properties");
+        }
+
         protected override string[] ProcessCommandLine()
         {
             var result = base.ProcessCommandLine();

diff --git a/src/net/KNetConnect/KNetConnect.csproj b/src/net/KNetConnect/KNetConnect.csproj
@@ -18,20 +18,20 @@
 	</PropertyGroup>
 
 	<ItemGroup Condition="True">
-		<knetcli_jars Include="$(ProjectDir)\..\..\..\jars\*.*" />
-		<knetcli_config Include="$(ProjectDir)\..\..\config\*.*" />
-		<knetcli_kraft Include="$(ProjectDir)\..\..\config\kraft\*.*" />
+		<knetconnect_jars Include="$(ProjectDir)\..\..\..\jars\*.*" />
+		<knetconnect_config Include="$(ProjectDir)\..\..\config\*.*" />
+		<knetconnect_kraft Include="$(ProjectDir)\..\..\config\kraft\*.*" />
 	</ItemGroup>
 
 	<Target Name="CopyCustomContent" AfterTargets="AfterBuild">
-		<Copy SourceFiles="@(knetcli_jars)" DestinationFolder="$(OutDir)\jars" />
-		<Copy SourceFiles="@(knetcli_config)" DestinationFolder="$(OutDir)\config" />
-		<Copy SourceFiles="@(knetcli_kraft)" DestinationFolder="$(OutDir)\config\kraft" />
+		<Copy SourceFiles="@(knetconnect_jars)" DestinationFolder="$(OutDir)\jars" />
+		<Copy SourceFiles="@(knetconnect_config)" DestinationFolder="$(OutDir)\config" />
+		<Copy SourceFiles="@(knetconnect_kraft)" DestinationFolder="$(OutDir)\config\kraft" />
 	</Target>
 	<Target Name="CopyCustomContentOnPublish" AfterTargets="Publish">
-		<Copy SourceFiles="@(knetcli_jars)" DestinationFolder="$(OutDir)\jars" />
-		<Copy SourceFiles="@(knetcli_config)" DestinationFolder="$(OutDir)\config" />
-		<Copy SourceFiles="@(knetcli_kraft)" DestinationFolder="$(OutDir)\config\kraft" />
+		<Copy SourceFiles="@(knetconnect_jars)" DestinationFolder="$(OutDir)\jars" />
+		<Copy SourceFiles="@(knetconnect_config)" DestinationFolder="$(OutDir)\config" />
+		<Copy SourceFiles="@(knetconnect_kraft)" DestinationFolder="$(OutDir)\config\kraft" />
 	</Target>
 
 	<ItemGroup>

diff --git a/src/net/KNetConnect/KNetConnectCore.cs b/src/net/KNetConnect/KNetConnectCore.cs
@@ -31,6 +31,11 @@ namespace MASES.KNetCLI
     /// </summary>
     internal class KNetConnectCore : KNetCore<KNetConnectCore>
     {
+        protected override string DefaultLog4JConfiguration()
+        {
+            return Path.Combine(Const.DefaultConfigurationPath, "tools-log4j.properties");
+        }
+
         public override IEnumerable<IArgumentMetadata> CommandLineArguments
         {
             get