Skip to content

Commit

Permalink
perf: do not spam the logs with config defs (#3044)
Browse files Browse the repository at this point in the history
* perf: do not spam the logs with config defs

Disable a load of Config classes from logging their config which just fills up the logs, and some other classes that likewise add a lot of noise to the logs.

Full build:

used to produce 53.7MB of logs, it now produces 2.3MB.
used to take 9:40 on my machine, with this change its down to 7:22 (less spooling to disk!)
This will also reduce storage costs of build logs.
  • Loading branch information
big-andy-coates authored Jul 3, 2019
1 parent 86ad8fc commit 94904a3
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 0 deletions.
48 changes: 48 additions & 0 deletions ksql-test-util/src/main/resources/log4j.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#
# Copyright 2019 Confluent Inc.
#
# Licensed under the Confluent Community License (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy of the
# License at
#
# http://www.confluent.io/confluent-community-license
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#

log4j.rootLogger=INFO,stdout

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n

# Disable INFO logs from Config classes, which log out their config on each creation:
log4j.logger.io.confluent.connect.avro.AvroConverterConfig=WARN
log4j.logger.io.confluent.connect.avro.AvroDataConfig=WARN
log4j.logger.io.confluent.kafka.serializers.KafkaAvroDeserializerConfig=WARN
log4j.logger.io.confluent.kafka.serializers.KafkaAvroSerializerConfig=WARN
log4j.logger.io.confluent.kafka.serializers.KafkaJsonDeserializerConfig=WARN
log4j.logger.io.confluent.kafka.serializers.KafkaJsonSerializerConfig=WARN
log4j.logger.io.confluent.ksql.logging.processing.ProcessingLogConfig=WARN
log4j.logger.io.confluent.ksql.rest.server.KsqlRestConfig=WARN,stdout
log4j.logger.io.confluent.ksql.util.KsqlConfig=WARN,stdout
log4j.logger.kafka.server.KafkaConfig=WARN
log4j.logger.org.apache.kafka.clients.admin.AdminClientConfig=WARN
log4j.logger.org.apache.kafka.clients.consumer.ConsumerConfig=WARN
log4j.logger.org.apache.kafka.clients.producer.ProducerConfig=WARN
log4j.logger.org.apache.kafka.connect.json.JsonConverterConfig=WARN
log4j.logger.org.apache.kafka.streams.StreamsConfig=WARN

# Disable INFO logging from the UDF loader, which logs every UDF ever time it runs:
log4j.logger.io.confluent.ksql.function.UdfLoader=WARN

# Disable logging of state transitions in KS:
log4j.logger.org.apache.kafka.streams.KafkaStreams=WARN
log4j.logger.org.apache.kafka.streams.processor.internals.StreamThread=WARN
log4j.logger.org.apache.kafka.streams.state.internals.RocksDBTimestampedStore=WARN

# Disable logging of App info
log4j.logger.org.apache.kafka.common.utils.AppInfoParser=WARN
10 changes: 10 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,16 @@
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<log4j.configuration>file:${project.basedir}/../ksql-test-util/src/main/resources/log4j.properties</log4j.configuration>
</systemPropertyVariables>
</configuration>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
Expand Down

0 comments on commit 94904a3

Please sign in to comment.