From e33a6f2308904625958cf0097695646fa6006a4f Mon Sep 17 00:00:00 2001 From: MASES Public Developers Team <94312179+masesdevelopers@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:45:11 +0100 Subject: [PATCH] Added new knet-log4j.properties for standard logging, updated KNetCLI, KNetConnect and KNetPS to use less intrusive logs (#386) --- src/config/knet-log4j.properties | 96 ++++++++++++++++++++++++++ src/net/KNet/InternalConst.cs | 14 +++- src/net/KNet/KNet.csproj | 19 +++++ src/net/KNet/KNetCore.cs | 24 ++++++- src/net/KNetCLI/KNetCLICore.cs | 5 ++ src/net/KNetConnect/KNetConnect.csproj | 18 ++--- src/net/KNetConnect/KNetConnectCore.cs | 5 ++ 7 files changed, 168 insertions(+), 13 deletions(-) create mode 100644 src/config/knet-log4j.properties diff --git a/src/config/knet-log4j.properties b/src/config/knet-log4j.properties new file mode 100644 index 0000000000..e789eaa953 --- /dev/null +++ b/src/config/knet-log4j.properties @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unspecified loggers and loggers with additivity=true output to server.log and stdout +# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise +log4j.rootLogger=INFO, kafkaAppender + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log +log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Change the line below to adjust ZK client logging +log4j.logger.org.apache.zookeeper=INFO + +# Change the two lines below to adjust the general broker logging level (output to server.log and stdout) +log4j.logger.kafka=INFO +log4j.logger.org.apache.kafka=INFO + +# Change to DEBUG or TRACE to enable request logging +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output +# related to the handling of requests +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +# Change the line below to adjust KRaft mode controller logging +log4j.logger.org.apache.kafka.controller=INFO, controllerAppender +log4j.additivity.org.apache.kafka.controller=false + +# Change the line below to adjust ZK mode controller logging +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=INFO, stateChangeAppender +log4j.additivity.state.change.logger=false + +# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses +log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender +log4j.additivity.kafka.authorizer.logger=false + diff --git a/src/net/KNet/InternalConst.cs b/src/net/KNet/InternalConst.cs index c6bab8d512..42fd17d099 100644 --- a/src/net/KNet/InternalConst.cs +++ b/src/net/KNet/InternalConst.cs @@ -48,13 +48,21 @@ public class Const /// public const string DefaultScalaVersion = "2.13.6"; /// + /// Default path location of configuration files + /// + public static readonly string DefaultConfigurationPath = Path.Combine(AssemblyLocation, "config"); + /// + /// Default path location of Jars files + /// + public static readonly string DefaultJarsPath = Path.Combine(AssemblyLocation, "jars"); + /// /// Default root path, i.e. consider installation within bin folder /// - public static readonly string DefaultRootPath = Path.Combine(AssemblyLocation, "jars") + Path.DirectorySeparatorChar; + public static readonly string DefaultRootPath = DefaultJarsPath + Path.DirectorySeparatorChar; /// - /// Default log4j path, i.e. consider installation within bin folder + /// Default log4j configuration file, i.e. considering a relative location to /// - public static readonly string DefaultLog4JPath = Path.Combine(AssemblyLocation, "config", "log4j.properties"); + public static readonly string DefaultLog4JConfigurationPath = Path.Combine(DefaultConfigurationPath, "knet-log4j.properties"); /// /// Default log path, i.e. consider installation within bin folder /// diff --git a/src/net/KNet/KNet.csproj b/src/net/KNet/KNet.csproj index cc0fb2a884..e3657c4c88 100644 --- a/src/net/KNet/KNet.csproj +++ b/src/net/KNet/KNet.csproj @@ -14,6 +14,25 @@ False False + + + + + + + + + + + + + + + + + + + diff --git a/src/net/KNet/KNetCore.cs b/src/net/KNet/KNetCore.cs index efb7283a77..e1e8620f79 100644 --- a/src/net/KNet/KNetCore.cs +++ b/src/net/KNet/KNetCore.cs @@ -66,7 +66,7 @@ public override IEnumerable CommandLineArguments new ArgumentMetadata() { Name = CLIParam.Log4JConfiguration, - Default = Const.DefaultLog4JPath, + Default = DefaultLog4JConfiguration(), Help = "The file containing the configuration of log4j.", }, new ArgumentMetadata() @@ -99,6 +99,16 @@ public override IEnumerable CommandLineArguments return lst; } } + /// + /// Returns the default configuration file to use when initializing command line defaults + /// + /// The configuration file to use for logging + /// Overrides in derived classes to give another default file + protected virtual string DefaultLog4JConfiguration() + { + return Const.DefaultLog4JConfigurationPath; + } + /// /// Public initializer /// @@ -115,6 +125,18 @@ protected override string[] ProcessCommandLine() _classToRun = ParsedArgs.Get(CLIParam.ClassToRun); _JarRootPath = ParsedArgs.Get(CLIParam.KafkaLocation); _log4JPath = ParsedArgs.Get(CLIParam.Log4JConfiguration); + if (!Path.IsPathRooted(_log4JPath)) // it is not a full path + { + var absolutePath = Path.Combine(Const.DefaultConfigurationPath, _log4JPath); + if (File.Exists(absolutePath)) + { + _log4JPath = absolutePath; + } + else + { + throw new ArgumentException($"{_log4JPath} is not an absolute path and there is no file under {Const.DefaultConfigurationPath} whose absolute path is {absolutePath}"); + } + } _logPath = ParsedArgs.Get(CLIParam.LogPath); _scalaVersion = ParsedArgs.Get(CLIParam.ScalaVersion); _disableJMX = ParsedArgs.Exist(CLIParam.DisableJMX); diff --git a/src/net/KNetCLI/KNetCLICore.cs b/src/net/KNetCLI/KNetCLICore.cs index 905c883059..2e78bd2bf0 100644 --- a/src/net/KNetCLI/KNetCLICore.cs +++ b/src/net/KNetCLI/KNetCLICore.cs @@ -29,6 +29,11 @@ namespace MASES.KNetCLI public class KNetCLICore : KNetCore where T : KNetCLICore { + protected override string DefaultLog4JConfiguration() + { + return Path.Combine(Const.DefaultConfigurationPath, "tools-log4j.properties"); + } + protected override string[] ProcessCommandLine() { var result = base.ProcessCommandLine(); diff --git a/src/net/KNetConnect/KNetConnect.csproj b/src/net/KNetConnect/KNetConnect.csproj index 582c649842..c1c2664e6c 100644 --- a/src/net/KNetConnect/KNetConnect.csproj +++ b/src/net/KNetConnect/KNetConnect.csproj @@ -18,20 +18,20 @@ - - - + + + - - - + + + - - - + + + diff --git a/src/net/KNetConnect/KNetConnectCore.cs b/src/net/KNetConnect/KNetConnectCore.cs index 658bc7de54..e5beb5f7f2 100644 --- a/src/net/KNetConnect/KNetConnectCore.cs +++ b/src/net/KNetConnect/KNetConnectCore.cs @@ -31,6 +31,11 @@ namespace MASES.KNetCLI /// internal class KNetConnectCore : KNetCore { + protected override string DefaultLog4JConfiguration() + { + return Path.Combine(Const.DefaultConfigurationPath, "tools-log4j.properties"); + } + public override IEnumerable CommandLineArguments { get