Initial logging module prototype

CONTRIBUTING.md

@@ -435,7 +435,7 @@ do by declaring a static field of the class. For example:
 
 In rare situations you may want to configure your `Logger` slightly
 differently, perhaps specifying a different class or maybe using one of the
-methods on `org.elasticsearch.common.logging.Loggers` instead.
+methods on `org.elasticsearch.common.logging.org.elasticsearch.logging.impl.Loggers` instead.
 
 If the log message includes values from your code then you must use use
 placeholders rather than constructing the string yourself using simple
@@ -461,7 +461,7 @@ expensive messages that will usually be discarded:
 
 Logging is an important behaviour of the system and sometimes deserves its own
 unit tests, especially if there is complex logic for computing what is logged
-and when to log it. You can use a `org.elasticsearch.test.MockLogAppender` to
+and when to log it. You can use a `MockLogAppender` to
 make assertions about the logs that are being emitted.
 
 Logging is a powerful diagnostic technique but it is not the only possibility.

benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java

@@ -7,10 +7,10 @@
  */
 package org.elasticsearch.benchmark.fs;
 
-import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.logging.spi.LoggingBootstrapSupport;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -47,7 +47,7 @@ public void setup() throws IOException {
         String[] paths = new String[] { path.toString() };
         nodePath = new NodeEnvironment.NodePath(path);
 
-        LogConfigurator.setNodeName("test");
+        LoggingBootstrapSupport.provider().setNodeName("test");
         Settings settings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), path)
             .putList(Environment.PATH_DATA_SETTING.getKey(), paths)

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.benchmark.search;
 
-import org.apache.logging.log4j.util.Strings;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -20,6 +19,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;

build-tools-internal/src/integTest/java/org/elasticsearch/gradle/jarhell/ThirdPartyAuditTaskIT.java

@@ -77,7 +77,7 @@ public void testClassNotFoundAndCompileOnlyIgnored() {
         assertOutputContains(
             result.getOutput(),
             "Missing classes:",
-            "  * org.apache.logging.log4j.LogManager",
+            "  * org.elasticsearch.logging.LogManager",
             "> Audit of third party dependencies failed"
         );
         assertOutputMissing(result.getOutput(), "Classes with violations:");

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -117,26 +117,10 @@ java.time.zone.ZoneRules#getStandardOffset(java.time.Instant)
 java.time.zone.ZoneRules#getDaylightSavings(java.time.Instant)
 java.time.zone.ZoneRules#isDaylightSavings(java.time.Instant)
 
-@defaultMessage Use logger methods with non-Object parameter
-org.apache.logging.log4j.Logger#trace(java.lang.Object)
-org.apache.logging.log4j.Logger#trace(java.lang.Object, java.lang.Throwable)
-org.apache.logging.log4j.Logger#debug(java.lang.Object)
-org.apache.logging.log4j.Logger#debug(java.lang.Object, java.lang.Throwable)
-org.apache.logging.log4j.Logger#info(java.lang.Object)
-org.apache.logging.log4j.Logger#info(java.lang.Object, java.lang.Throwable)
-org.apache.logging.log4j.Logger#warn(java.lang.Object)
-org.apache.logging.log4j.Logger#warn(java.lang.Object, java.lang.Throwable)
-org.apache.logging.log4j.Logger#error(java.lang.Object)
-org.apache.logging.log4j.Logger#error(java.lang.Object, java.lang.Throwable)
-org.apache.logging.log4j.Logger#fatal(java.lang.Object)
-org.apache.logging.log4j.Logger#fatal(java.lang.Object, java.lang.Throwable)
-
-@defaultMessage Use getLogger(Class)
-org.apache.logging.log4j.LogManager#getLogger()
 
 # This is permitted in test code, where we have a Checkstyle rule to guard
 # against unsafe uses. This leniency does not extend to server code.
-java.lang.String#formatted(java.lang.Object[]) @ Uses default locale - use String#format(Locale, String, Object...) instead
+java.lang.String#formatted(java.lang.Object[])  @ Uses default locale - use String#format(Locale, String, Object...) instead
 
 @defaultMessage Unbatched cluster state tasks are a source of performance and stability bugs. Implement the update logic in a executor which is reused across tasks instead.
 org.elasticsearch.cluster.ClusterStateTaskExecutor#unbatched()

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -357,8 +357,10 @@ private void commonNodeConfig() {
             } else {
                 if (node.getVersion().onOrAfter("7.16.0")) {
                     node.defaultConfig.put("cluster.deprecation_indexing.enabled", "false");
+                    node.defaultConfig.put("xpack.ml.enabled", "false");
                 }
             }
+            node.defaultConfig.put("xpack.security.enabled", "false");
 
             // Can only configure master nodes if we have node names defined
             if (nodeNames != null) {

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -111,7 +111,9 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
         "path.repo",
         "discovery.seed_providers",
-        "cluster.deprecation_indexing.enabled"
+        "cluster.deprecation_indexing.enabled",
+        "xpack.ml.enabled",
+        "xpack.security.enabled"
 
     );
 
@@ -1019,6 +1021,13 @@ public synchronized void stop(boolean tailLogs) {
         requireNonNull(esProcess, "Can't stop `" + this + "` as it was not started or already stopped.");
         // Test clusters are not reused, don't spend time on a graceful shutdown
         stopHandle(esProcess.toHandle(), true);
+        // try {
+        // for (Thread oThread : threads) {
+        // oThread.join(Duration.ofSeconds(30).toMillis());
+        // }
+        // } catch (InterruptedException e) {
+        // throw new RuntimeException(e);
+        // }
         reaperServiceProvider.get().unregister(toString());
         esProcess = null;
         // Clean up the ports file in case this is started again.

client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java

@@ -7,14 +7,14 @@
  */
 package org.elasticsearch.client.benchmark.ops.bulk;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.benchmark.BenchmarkTask;
 import org.elasticsearch.client.benchmark.metrics.Sample;
 import org.elasticsearch.client.benchmark.metrics.SampleRecorder;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 
 import java.io.BufferedReader;
 import java.io.IOException;

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -10,8 +10,6 @@
 
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
@@ -76,6 +74,8 @@
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.plugins.spi.NamedXContentProvider;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestStatus;

client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt

@@ -10,13 +10,7 @@ org.apache.http.entity.ContentType#create(java.lang.String,java.lang.String)
 org.apache.http.entity.ContentType#create(java.lang.String,java.nio.charset.Charset)
 org.apache.http.entity.ContentType#create(java.lang.String,org.apache.http.NameValuePair[])
 
-@defaultMessage ES's logging infrastructure uses log4j2 which we don't want to force on high level rest client users
-org.elasticsearch.common.logging.DeprecationLogger
-org.elasticsearch.common.logging.LogConfigurator
-org.elasticsearch.common.logging.LoggerMessageFormat
-org.elasticsearch.common.logging.Loggers
-org.elasticsearch.common.logging.NodeNamePatternConverter
-org.elasticsearch.common.logging.PrefixLogger
+
 
 @defaultMessage We can't rely on log4j2 being on the classpath so don't log deprecations!
 org.elasticsearch.common.xcontent.LoggingDeprecationHandler

distribution/src/bin/elasticsearch

@@ -43,6 +43,7 @@ while [ $# -gt 0 ]; do
    fi
 done
 
+echo "temp dir"
 if [ -z "$ES_TMPDIR" ]; then
   ES_TMPDIR=`"$JAVA" -cp "$LAUNCHERS_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
 fi

distribution/src/bin/elasticsearch-env

@@ -5,7 +5,7 @@ set -e -o pipefail
 CDPATH=""
 
 SCRIPT="$0"
-
+echo "env"
 # SCRIPT might be an arbitrarily deep series of symbolic links; loop until we
 # have the concrete path
 while [ -h "$SCRIPT" ] ; do

distribution/tools/ansi-console/src/main/java/org/elasticsearch/io/ansi/AnsiConsoleLoader.java

@@ -7,10 +7,10 @@
  */
 package org.elasticsearch.io.ansi;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.bootstrap.ConsoleLoader;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.logging.Logger;
 import org.fusesource.jansi.Ansi;
 import org.fusesource.jansi.AnsiConsole;
 import org.fusesource.jansi.AnsiPrintStream;
@@ -22,7 +22,7 @@
 import java.nio.charset.Charset;
 import java.util.function.Supplier;
 
-import static org.apache.logging.log4j.LogManager.getLogger;
+import static org.elasticsearch.logging.LogManager.getLogger;
 
 /**
  * Loads the {@link AnsiConsole} and checks whether it meets our requirements for a "Console".

docs/reference/setup/logging-config.asciidoc

@@ -259,7 +259,7 @@ logs streams when parsing.
 appender.rolling.layout.type = ECSJsonLayout
 appender.rolling.layout.dataset = elasticsearch.server
 --------------------------------------------------
-:es-json-layout-java-doc: {elasticsearch-javadoc}/org/elasticsearch/common/logging/ESJsonLayout.html
+:es-json-layout-java-doc: {elasticsearch-javadoc}/org/elasticsearch/common/logging/org.elasticsearch.logging.impl.ESJsonLayout.html
 
 Each line contains a single JSON document with the properties configured in `ECSJsonLayout`.
 See this class {es-json-layout-java-doc}[javadoc] for more details.

libs/build.gradle

@@ -23,7 +23,9 @@ configure(subprojects - project('elasticsearch-log4j')) {
         Project depProject = dep.dependencyProject
         if (depProject != null
           && false == depProject.path.equals(':libs:elasticsearch-x-content')
+          && false == depProject.path.equals(':libs:elasticsearch-logging')
           && false == depProject.path.equals(':libs:elasticsearch-core')
+          && false == depProject.path.equals(':libs:elasticsearch-cli')
           && depProject.path.startsWith(':libs')
           && depProject.name.startsWith('elasticsearch-')) {
           throw new InvalidUserDataException("projects in :libs "

libs/core/src/main/java/module-info.java

@@ -13,5 +13,5 @@
     exports org.elasticsearch.jdk;
     exports org.elasticsearch.core.internal.io;
     exports org.elasticsearch.core.internal.net;
-    exports org.elasticsearch.core.internal.provider to org.elasticsearch.xcontent;
+    exports org.elasticsearch.core.internal.provider to org.elasticsearch.xcontent, org.elasticsearch.logging;
 }

libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java

@@ -235,7 +235,7 @@ private static Map<String, CodeSource> getProviderPrefixes(ClassLoader parent, S
         String providerPrefix = IMPL_PREFIX + providerName;
         URL listingURL = parent.getResource(providerPrefix + JAR_LISTING_FILE);
         if (listingURL == null) {
-            throw new IllegalStateException("missing x-content provider jars list");
+            throw new IllegalStateException("missing x-content provider jars list");// TODO PG x-content??
         }
         try (
             InputStream in = listingURL.openStream();

libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedModulePath.java

@@ -145,6 +145,12 @@ record ScanResult(Set<String> classFiles, Set<String> serviceFiles) {}
      */
     static ModuleDescriptor descriptorForAutomatic(Path path) {
         String moduleName = moduleNameFromManifestOrNull(path);
+        if (moduleName == null && path.endsWith("log4j2-ecs-layout-1.2.0.jar")) {
+            moduleName = "log4j2.ecs.layout";
+        }
+        if (moduleName == null && path.endsWith("ecs-logging-core-1.2.0.jar")) {
+            moduleName = "ecs.logging.core";
+        }
         if (moduleName == null) {
             throw new FindException("automatic module without a manifest name is not supported, for:" + path);
         }

libs/log4j2-es-logging/build.gradle

@@ -0,0 +1,59 @@
+import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersTask
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.publish'
+String log4jVersion = "2.17.1"
+
+dependencies {
+
+  implementation "org.apache.logging.log4j:log4j-api:${log4jVersion}"
+  api project(':libs:elasticsearch-core')
+  api project(':libs:elasticsearch-logging')
+
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.elasticsearch', module: 'elasticsearch-slf4j-es-logging'
+  }
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+}
+
+tasks.named('forbiddenApisMain').configure {
+  // geo does not depend on server
+  // TODO: Need to decide how we want to handle for forbidden signatures with the changes to core
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+//TODO PG fix
+tasks.named("thirdPartyAudit").configure {
+  ignoreMissingClasses(
+      'org.osgi.framework.Bundle',
+      'org.osgi.framework.BundleActivator',
+      'org.osgi.framework.BundleContext',
+      'org.osgi.framework.BundleEvent',
+      'org.osgi.framework.SynchronousBundleListener',
+      'org.osgi.framework.wiring.BundleWire',
+      'org.osgi.framework.wiring.BundleWiring'
+
+  )
+}
+
+