diff --git a/roles/tispark/tasks/main.yml b/roles/tispark/tasks/main.yml index 206e8e6c6..cdd0eeaa7 100644 --- a/roles/tispark/tasks/main.yml +++ b/roles/tispark/tasks/main.yml @@ -42,3 +42,10 @@ mode: 0755 backup: yes when: "'spark_slaves' in group_names" + +- name: create log4j.properties file + template: + src: log4j.properties.j2 + dest: "{{ deploy_dir }}/spark/conf/log4j.properties" + mode: 0644 + backup: yes diff --git a/roles/tispark/templates/log4j.properties.j2 b/roles/tispark/templates/log4j.properties.j2 new file mode 100644 index 000000000..d9d193670 --- /dev/null +++ b/roles/tispark/templates/log4j.properties.j2 @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# Set the default spark-shell log level to WARN. When running the spark-shell, the +# log level for this class is used to overwrite the root logger's log level, so that +# the user can have different defaults for the shell and regular Spark apps. +log4j.logger.org.apache.spark.repl.Main=WARN + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.spark_project.jetty=WARN +log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO +log4j.logger.org.apache.parquet=ERROR +log4j.logger.parquet=ERROR + +# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support +log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL +log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR + +# tispark disable "WARN ObjectStore:568 - Failed to get database" +log4j.logger.org.apache.hadoop.hive.metastore.ObjectStore=ERROR