[NSE-687]Try to upgrade log4j

.github/workflows/unittests.yml

@@ -96,7 +96,6 @@ jobs:
           mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip
       - name: Run unit tests
         run: |
-          sed -i "199, 202d" pom.xml
           mvn clean install -N
           cd arrow-data-source
           mvn clean install -DskipTests -Dbuild_arrow=OFF
@@ -144,7 +143,6 @@ jobs:
           mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip
       - name: Run unit tests
         run: |
-          sed -i "199, 202d" pom.xml
           mvn clean install -N
           cd arrow-data-source
           mvn clean install -DskipTests -Dbuild_arrow=OFF

arrow-data-source/pom.xml

@@ -145,6 +145,26 @@
       <artifactId>scalatest_${scala.binary.version}</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

native-sql-engine/core/pom.xml

@@ -301,6 +301,26 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

native-sql-engine/core/src/test/resources/log4j2.properties

@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file core/target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.file.ref = File
+
+#Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+appender.console.filter.threshold.type = ThresholdFilter
+appender.console.filter.threshold.level = warn
+
+#File Appender
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+# Set the logger level of File Appender to WARN
+appender.file.filter.threshold.type = ThresholdFilter
+appender.file.filter.threshold.level = info
+
+# Some packages are noisy for no good reason.
+logger.parquet_recordreader.name = org.apache.parquet.hadoop.ParquetRecordReader
+logger.parquet_recordreader.additivity = false
+logger.parquet_recordreader.level = off
+
+logger.parquet_outputcommitter.name = org.apache.parquet.hadoop.ParquetOutputCommitter
+logger.parquet_outputcommitter.additivity = false
+logger.parquet_outputcommitter.level = off
+
+logger.hadoop_lazystruct.name = org.apache.hadoop.hive.serde2.lazy.LazyStruct
+logger.hadoop_lazystruct.additivity = false
+logger.hadoop_lazystruct.level = off
+
+logger.hadoop_retryinghmshandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
+logger.hadoop_retryinghmshandler.additivity = false
+logger.hadoop_retryinghmshandler.level = off
+
+logger.hive_metadata.name = hive.ql.metadata.Hive
+logger.hive_metadata.additivity = false
+logger.hive_metadata.level = off
+
+# Parquet related logging
+logger.parquet1.name = org.apache.parquet.CorruptStatistics
+logger.parquet1.level = error
+
+logger.parquet2.name = parquet.CorruptStatistics
+logger.parquet2.level = error
+
+# The below part is left by log4j 1.x version.
+
+# Some packages are noisy for no good reason.
+log4j.additivity.org.apache.parquet.hadoop.ParquetRecordReader=false
+log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+
+log4j.additivity.org.apache.parquet.hadoop.ParquetOutputCommitter=false
+log4j.logger.org.apache.parquet.hadoop.ParquetOutputCommitter=OFF
+
+log4j.additivity.org.apache.hadoop.hive.serde2.lazy.LazyStruct=false
+log4j.logger.org.apache.hadoop.hive.serde2.lazy.LazyStruct=OFF
+
+log4j.additivity.org.apache.hadoop.hive.metastore.RetryingHMSHandler=false
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=OFF
+
+log4j.additivity.hive.ql.metadata.Hive=false
+log4j.logger.hive.ql.metadata.Hive=OFF
+
+# Parquet related logging
+log4j.logger.org.apache.parquet.CorruptStatistics=ERROR
+log4j.logger.parquet.CorruptStatistics=ERROR

native-sql-engine/core/src/test/scala/com/intel/oap/misc/PartitioningSuite.scala

@@ -19,7 +19,7 @@ package com.intel.oap.misc
 
 import com.intel.oap.tpc.ds.TPCDSTableGen
 import com.intel.oap.tpc.util.TPCRunner
-//import org.apache.log4j.{Level, LogManager}
+import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.functions.{col, expr}
@@ -68,7 +68,7 @@ class PartitioningSuite extends QueryTest with SharedSparkSession {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    //LogManager.getRootLogger.setLevel(Level.WARN)
+    LogManager.getRootLogger.setLevel(Level.WARN)
 
     lPath = Files.createTempFile("", ".parquet").toFile.getAbsolutePath
     spark.range(scale)

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/Orc_TPCDSSuite.scala

@@ -18,7 +18,7 @@
 package com.intel.oap.tpc.ds
 
 import com.intel.oap.tpc.util.TPCRunner
-//import org.apache.log4j.{Level, LogManager}
+import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSparkSession
@@ -60,7 +60,7 @@ class Orc_TPCDSSuite extends QueryTest with SharedSparkSession {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    //LogManager.getRootLogger.setLevel(Level.WARN)
+    LogManager.getRootLogger.setLevel(Level.WARN)
     val tGen = new Orc_TPCDSTableGen(spark, 0.1D, TPCDS_WRITE_PATH)
     tGen.gen()
     tGen.createTables()

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala

@@ -18,7 +18,7 @@
 package com.intel.oap.tpc.ds
 
 import com.intel.oap.tpc.util.TPCRunner
-//import org.apache.log4j.{Level, LogManager}
+import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.functions.{col, exp, expr}
@@ -64,7 +64,7 @@ class TPCDSSuite extends QueryTest with SharedSparkSession {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    //LogManager.getRootLogger.setLevel(Level.WARN)
+    LogManager.getRootLogger.setLevel(Level.WARN)
     val tGen = new TPCDSTableGen(spark, 0.1D, TPCDS_WRITE_PATH)
     tGen.gen()
     tGen.createTables()

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/Orc_TPCHSuite.scala

@@ -24,7 +24,7 @@ import com.intel.oap.tpc.MallocUtils
 import com.intel.oap.tpc.h.TPCHSuite.RAMMonitor
 import com.intel.oap.tpc.util.TPCRunner
 import org.apache.commons.lang.StringUtils
-//import org.apache.log4j.{Level, LogManager}
+import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSparkSession
@@ -63,7 +63,7 @@ class Orc_TPCHSuite extends QueryTest with SharedSparkSession {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    //LogManager.getRootLogger.setLevel(Level.WARN)
+    LogManager.getRootLogger.setLevel(Level.WARN)
     val tGen = new Orc_TPCHTableGen(spark, 0.1D, TPCH_WRITE_PATH)
     tGen.gen()
     tGen.createTables()

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala

@@ -27,7 +27,7 @@ import com.intel.oap.tpc.MallocUtils
 import com.intel.oap.tpc.h.TPCHSuite.RAMMonitor
 import com.intel.oap.tpc.util.TPCRunner
 import org.apache.commons.lang.StringUtils
-//import org.apache.log4j.{Level, LogManager}
+import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSparkSession
@@ -72,7 +72,7 @@ class TPCHSuite extends QueryTest with SharedSparkSession {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    //LogManager.getRootLogger.setLevel(Level.WARN)
+    LogManager.getRootLogger.setLevel(Level.WARN)
     val tGen = new TPCHTableGen(spark, 0.1D, TPCH_WRITE_PATH)
     tGen.gen()
     tGen.createTables()

native-sql-engine/core/src/test/scala/org/apache/spark/sql/CTEHintSuite.scala

@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-//import org.apache.log4j.Level
+import org.apache.log4j.Level
 
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.test.SharedSparkSession
@@ -55,7 +55,6 @@ class CTEHintSuite extends QueryTest with SharedSparkSession {
     assert(joinHints == expectedHints)
   }
 
-  /* Remark log4j1 unit test
   def verifyJoinHintWithWarnings(
       df: => DataFrame,
       expectedHints: Seq[JoinHint],
@@ -73,7 +72,6 @@ class CTEHintSuite extends QueryTest with SharedSparkSession {
       assert(warningMessages.contains(w))
     }
   }
-  */
 
   def msgNoJoinForJoinHint(strategy: String): String =
     s"A join hint (strategy=$strategy) is specified but it is not part of a join relation."
@@ -135,7 +133,6 @@ class CTEHintSuite extends QueryTest with SharedSparkSession {
           Some(HintInfo(strategy = Some(SHUFFLE_HASH))),
           None) :: Nil
       )
-      /* Remark log4j1 unit test
       verifyJoinHintWithWarnings(
         sql(
           """
@@ -154,7 +151,6 @@ class CTEHintSuite extends QueryTest with SharedSparkSession {
           msgNoJoinForJoinHint("shuffle_hash") ::
             msgJoinHintOverridden("broadcast") :: Nil
       )
-      */
       verifyJoinHint(
         sql(
           """

native-sql-engine/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala

@@ -639,7 +639,6 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
 class BasicCharVarcharTestSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
 
-  /* Remark log4j1 unit test
   test("user-specified schema in cast") {
     def assertNoCharType(df: DataFrame): Unit = {
       checkAnswer(df, Row("0"))
@@ -656,7 +655,6 @@ class BasicCharVarcharTestSuite extends QueryTest with SharedSparkSession {
       assertNoCharType(sql("SELECT CAST(id AS CHAR(5)) FROM range(1)"))
     }
   }
-  */
 
   def failWithInvalidCharUsage[T](fn: => T): Unit = {
     val e = intercept[AnalysisException](fn)

native-sql-engine/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala

@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-//import org.apache.log4j.Level
+import org.apache.log4j.Level
 
 import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide, EliminateResolvedHint}
 import org.apache.spark.sql.catalyst.plans.PlanTest
@@ -45,7 +45,6 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
   def msgJoinHintOverridden(strategy: String): String =
     s"Hint (strategy=$strategy) is overridden by another hint and will not take effect."
 
-  /* Remark log4j1 unit test
   def verifyJoinHintWithWarnings(
       df: => DataFrame,
       expectedHints: Seq[JoinHint],
@@ -63,7 +62,6 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
       assert(warningMessages.contains(w))
     }
   }
-  */
 
   def verifyJoinHint(df: DataFrame, expectedHints: Seq[JoinHint]): Unit = {
     val optimized = df.queryExecution.optimizedPlan
@@ -212,7 +210,6 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
     )
   }
 
-  /* Remark log4j1 unit test
   test("hint merge") {
     verifyJoinHintWithWarnings(
       df.hint("broadcast").filter($"id" > 2).hint("broadcast").join(df, "id"),
@@ -251,9 +248,7 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
         msgJoinHintOverridden("shuffle_hash") :: Nil
     )
   }
-  */
 
-  /* Remark log4j1 unit test
   test("hint merge - SQL") {
     withTempView("a", "b", "c") {
       df1.createOrReplaceTempView("a")
@@ -304,7 +299,6 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
       )
     }
   }
-  */
 
   test("nested hint") {
     verifyJoinHint(

native-sql-engine/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala

@@ -387,7 +387,6 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
 
   }
 
-  /* Remark log4j1 unit test
   test("SPARK-33944: warning setting hive.metastore.warehouse.dir using session options") {
     val msg = "Not allowing to set hive.metastore.warehouse.dir in SparkSession's options"
     val logAppender = new LogAppender(msg)
@@ -400,9 +399,7 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     }
     assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(msg)))
   }
-  */
 
-  /* Remark log4j1 unit test
   test("SPARK-33944: no warning setting spark.sql.warehouse.dir using session options") {
     val msg = "Not allowing to set hive.metastore.warehouse.dir in SparkSession's options"
     val logAppender = new LogAppender(msg)
@@ -415,5 +412,4 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     }
     assert(!logAppender.loggingEvents.exists(_.getRenderedMessage.contains(msg)))
   }
-  */
 }