Remove native code info from xpack info api (#43125)

The native code info is a portion of the xpack info api that emits
arbitrary info about feature plugins that are backed by native code.
This is currently only used by machine learning. Additionally, it is the
only non enabled/available information reported by the info api.

This commit moves the native code info reporting to the usage api for
machine learning. The commit info reported is only used for debugging
purposes, and not captured by the current uses of the info api
(monitoring and telemetry) since it requires passing the verbose flag.

Long term, this information would be better suited to node info. Until
node info is extendable by plugins, this new location will allow
cleaning up the xpack info api to be implemented in a generic way for
all xpack features.

client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java

@@ -73,15 +73,12 @@ public void testXPackInfo() throws IOException {
         FeatureSet graph = info.getFeatureSetsInfo().getFeatureSets().get("graph");
         assertTrue(graph.available());
         assertTrue(graph.enabled());
-        assertNull(graph.nativeCodeInfo());
         FeatureSet monitoring = info.getFeatureSetsInfo().getFeatureSets().get("monitoring");
         assertTrue(monitoring.available());
         assertTrue(monitoring.enabled());
-        assertNull(monitoring.nativeCodeInfo());
         FeatureSet ml = info.getFeatureSetsInfo().getFeatureSets().get("ml");
         assertTrue(ml.available());
         assertTrue(ml.enabled());
-        assertEquals(mainResponse.getVersion().getNumber(), ml.nativeCodeInfo().get("version").toString());
     }
 
     public void testXPackInfoEmptyRequest() throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/XPackInfoResponseTests.java

@@ -70,7 +70,7 @@ private LicenseInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackIn
     private FeatureSetsInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackInfoResponse.FeatureSetsInfo featureSetsInfo) {
         return featureSetsInfo != null
             ? new FeatureSetsInfo(featureSetsInfo.getFeatureSets().values().stream()
-            .map(fs -> new FeatureSet(fs.name(), fs.available(), fs.enabled(), fs.nativeCodeInfo()))
+            .map(fs -> new FeatureSet(fs.name(), fs.available(), fs.enabled()))
             .collect(Collectors.toSet()))
             : null;
     }
@@ -169,19 +169,6 @@ private FeatureSet randomFeatureSet() {
         return new FeatureSet(
             randomAlphaOfLength(5),
             randomBoolean(),
-            randomBoolean(),
-            randomNativeCodeInfo());
-    }
-
-    private Map<String, Object> randomNativeCodeInfo() {
-        if (randomBoolean()) {
-            return null;
-        }
-        int size = between(0, 10);
-        Map<String, Object> nativeCodeInfo = new HashMap<>(size);
-        while (nativeCodeInfo.size() < size) {
-            nativeCodeInfo.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
-        }
-        return nativeCodeInfo;
+            randomBoolean());
     }
 }

docs/reference/rest-api/info.asciidoc

@@ -85,11 +85,7 @@ Example response:
       },
       "ml" : {
          "available" : true,
-         "enabled" : true,
-         "native_code_info" : {
-           "version" : "7.0.0-alpha1-SNAPSHOT",
-           "build_hash" : "99a07c016d5a73"
-        }
+         "enabled" : true
       },
       "monitoring" : {
          "available" : true,

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java

@@ -71,11 +71,6 @@ public void testName() {
         assertThat(featureSet.name(), equalTo("ccr"));
     }
 
-    public void testNativeCodeInfo() {
-        CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState);
-        assertNull(featureSet.nativeCodeInfo());
-    }
-
     public void testUsageStats() throws Exception {
         MetaData.Builder metaData = MetaData.builder();
 

x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java

@@ -334,18 +334,18 @@ public static class FeatureSet implements ToXContentObject, Writeable {
             private final String name;
             private final boolean available;
             private final boolean enabled;
-            @Nullable private final Map<String, Object> nativeCodeInfo;
 
-            public FeatureSet(String name, boolean available, boolean enabled,
-                              @Nullable Map<String, Object> nativeCodeInfo) {
+            public FeatureSet(String name, boolean available, boolean enabled) {
                 this.name = name;
                 this.available = available;
                 this.enabled = enabled;
-                this.nativeCodeInfo = nativeCodeInfo;
             }
 
             public FeatureSet(StreamInput in) throws IOException {
-                this(in.readString(), readAvailable(in), in.readBoolean(), in.readMap());
+                this(in.readString(), readAvailable(in), in.readBoolean());
+                if (in.getVersion().before(Version.V_8_0_0)) {
+                    in.readMap(); // backcompat reading native code info, but no longer used here
+                }
             }
 
             // this is separated out so that the removed description can be read from the stream on construction
@@ -365,7 +365,9 @@ public void writeTo(StreamOutput out) throws IOException {
                 }
                 out.writeBoolean(available);
                 out.writeBoolean(enabled);
-                out.writeMap(nativeCodeInfo);
+                if (out.getVersion().before(Version.V_8_0_0)) {
+                    out.writeMap(Collections.emptyMap());
+                }
             }
 
             public String name() {
@@ -380,35 +382,26 @@ public boolean enabled() {
                 return enabled;
             }
 
-            @Nullable
-            public Map<String, Object> nativeCodeInfo() {
-                return nativeCodeInfo;
-            }
-
             @Override
             public boolean equals(Object other) {
                 if (other == null || other.getClass() != getClass()) return false;
                 if (this == other) return true;
                 FeatureSet rhs = (FeatureSet) other;
                 return Objects.equals(name, rhs.name)
                         && available == rhs.available
-                        && enabled == rhs.enabled
-                        && Objects.equals(nativeCodeInfo, rhs.nativeCodeInfo);
+                        && enabled == rhs.enabled;
             }
 
             @Override
             public int hashCode() {
-                return Objects.hash(name, available, enabled, nativeCodeInfo);
+                return Objects.hash(name, available, enabled);
             }
 
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                 builder.startObject();
                 builder.field("available", available);
                 builder.field("enabled", enabled);
-                if (nativeCodeInfo != null) {
-                    builder.field("native_code_info", nativeCodeInfo);
-                }
                 return builder.endObject();
             }
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRFeatureSet.java

@@ -31,7 +31,6 @@
 
 import java.io.IOException;
 import java.time.Instant;
-import java.util.Map;
 import java.util.Objects;
 
 public class CCRFeatureSet implements XPackFeatureSet {
@@ -60,11 +59,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
     public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
 
         private final Settings settings;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/EmptyXPackFeatureSet.java

@@ -5,9 +5,6 @@
  */
 package org.elasticsearch.xpack.core;
 
-import java.util.Collections;
-import java.util.Map;
-
 public class EmptyXPackFeatureSet implements XPackFeatureSet {
     @Override
     public String name() {
@@ -24,8 +21,4 @@ public boolean enabled() {
         return false;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return Collections.emptyMap();
-    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatureSet.java

@@ -12,7 +12,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.Map;
 
 public interface XPackFeatureSet {
 
@@ -22,8 +21,6 @@ public interface XPackFeatureSet {
 
     boolean enabled();
 
-    Map<String, Object> nativeCodeInfo();
-
     abstract class Usage implements ToXContentObject, NamedWriteable {
 
         private static final String AVAILABLE_XFIELD = "available";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java

@@ -58,8 +58,7 @@ protected void doExecute(Task task, XPackInfoRequest request, ActionListener<XPa
         XPackInfoResponse.FeatureSetsInfo featureSetsInfo = null;
         if (request.getCategories().contains(XPackInfoRequest.Category.FEATURES)) {
             Set<FeatureSet> featureSets = this.featureSets.stream().map(fs ->
-                    new FeatureSet(fs.name(), fs.available(), fs.enabled(),
-                            request.isVerbose() ? fs.nativeCodeInfo() : null))
+                    new FeatureSet(fs.name(), fs.available(), fs.enabled()))
                     .collect(Collectors.toSet());
             featureSetsInfo = new XPackInfoResponse.FeatureSetsInfo(featureSets);
         }

x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSet.java

@@ -97,11 +97,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
     public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
 
         private final boolean enabled;

x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphFeatureSet.java

@@ -24,8 +24,6 @@
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 
-import java.util.Map;
-
 public class GraphFeatureSet implements XPackFeatureSet {
 
     private final boolean enabled;
@@ -52,11 +50,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
     public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
 
         private final Settings settings;

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java

@@ -61,11 +61,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
     public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
         private final boolean enabled;
         private final XPackLicenseState licenseState;

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java

@@ -81,11 +81,6 @@ public void testName() {
         assertThat(featureSet.name(), equalTo("ilm"));
     }
 
-    public void testNativeCodeInfo() {
-        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState);
-        assertNull(featureSet.nativeCodeInfo());
-    }
-
     public void testUsageStats() throws Exception {
         Map<String, String> indexPolicies = new HashMap<>();
         List<LifecyclePolicy> policies = new ArrayList<>();

x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashFeatureSet.java

@@ -24,8 +24,6 @@
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
 
-import java.util.Map;
-
 public class LogstashFeatureSet implements XPackFeatureSet {
 
     private final boolean enabled;
@@ -52,11 +50,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-      return null;
-    }
-
     public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
 
         private final Settings settings;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.ml;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.ElasticsearchException;
@@ -18,6 +17,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.Platforms;
@@ -40,18 +40,14 @@
 import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
 import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator;
 import org.elasticsearch.xpack.ml.job.JobManagerHolder;
-import org.elasticsearch.xpack.ml.process.NativeController;
-import org.elasticsearch.xpack.ml.process.NativeControllerHolder;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
 public class MachineLearningFeatureSet implements XPackFeatureSet {
@@ -60,36 +56,18 @@ public class MachineLearningFeatureSet implements XPackFeatureSet {
      * List of platforms for which the native processes are available
      */
     private static final List<String> mlPlatforms =
-            Arrays.asList("darwin-x86_64", "linux-x86_64", "windows-x86_64");
+        Arrays.asList("darwin-x86_64", "linux-x86_64", "windows-x86_64");
 
     private final boolean enabled;
     private final XPackLicenseState licenseState;
-    private final Map<String, Object> nativeCodeInfo;
 
     @Inject
-    public MachineLearningFeatureSet(Environment environment, ClusterService clusterService, XPackLicenseState licenseState) {
-        this.enabled = XPackSettings.MACHINE_LEARNING_ENABLED.get(environment.settings());
+    public MachineLearningFeatureSet(Settings settings, XPackLicenseState licenseState) {
+        this.enabled = XPackSettings.MACHINE_LEARNING_ENABLED.get(settings);
         this.licenseState = licenseState;
-        Map<String, Object> nativeCodeInfo = NativeController.UNKNOWN_NATIVE_CODE_INFO;
-        // Don't try to get the native code version if ML is disabled - it causes too much controversy
-        // if ML has been disabled because of some OS incompatibility.
-        if (enabled) {
-            try {
-                if (isRunningOnMlPlatform(true)) {
-                    NativeController nativeController = NativeControllerHolder.getNativeController(clusterService.getNodeName(),
-                        environment);
-                    if (nativeController != null) {
-                        nativeCodeInfo = nativeController.getNativeCodeInfo();
-                    }
-                }
-            } catch (IOException | TimeoutException e) {
-                LogManager.getLogger(MachineLearningFeatureSet.class).error("Cannot get native code info for Machine Learning", e);
-                throw new ElasticsearchException("Cannot communicate with Machine Learning native code");
-            }
-        }
-        this.nativeCodeInfo = nativeCodeInfo;
     }
 
+    // TODO: remove these methods
     static boolean isRunningOnMlPlatform(boolean fatalIfNot) {
         return isRunningOnMlPlatform(Constants.OS_NAME, Constants.OS_ARCH, fatalIfNot);
     }
@@ -101,7 +79,7 @@ static boolean isRunningOnMlPlatform(String osName, String osArch, boolean fatal
         }
         if (fatalIfNot) {
             throw new ElasticsearchException("X-Pack is not supported and Machine Learning is not available for [" + platformName
-                    + "]; you can use the other X-Pack features (unsupported) by setting xpack.ml.enabled: false in elasticsearch.yml");
+                + "]; you can use the other X-Pack features (unsupported) by setting xpack.ml.enabled: false in elasticsearch.yml");
         }
         return false;
     }
@@ -121,14 +99,8 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return nativeCodeInfo;
-    }
-
     public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
 
-        private final Environment environment;
         private final Client client;
         private final XPackLicenseState licenseState;
         private final JobManagerHolder jobManagerHolder;
@@ -141,7 +113,6 @@ public UsageTransportAction(TransportService transportService, ClusterService cl
                                     XPackLicenseState licenseState, JobManagerHolder jobManagerHolder) {
             super(XPackUsageFeatureAction.MACHINE_LEARNING.name(), transportService, clusterService,
                   threadPool, actionFilters, indexNameExpressionResolver);
-            this.environment = environment;
             this.client = client;
             this.licenseState = licenseState;
             this.jobManagerHolder = jobManagerHolder;