[Profiling] Switch to OTEL cloud.provider, cloud.region, host.type (#…

…106656) * [Profiling] Switch to OTEL cloud.provider, cloud.region, host.type * Remove wrong spaces from auto-format
elastic · Mar 22, 2024 · 0eca8dd · 0eca8dd
1 parent bdce52e
commit 0eca8dd
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 37 deletions.
diff --git a/...e/template-resources/src/main/resources/profiling/component-template/profiling-hosts.json b/...e/template-resources/src/main/resources/profiling/component-template/profiling-hosts.json
@@ -33,11 +33,28 @@
           "type": "date",
           "format": "epoch_second"
         },
-        "host.id": {
-          "type": "keyword"
+        "host": {
+          "properties": {
+            "arch": {
+              "type": "keyword"
+            },
+            "id": {
+              "type": "keyword"
+            },
+            "type": {
+              "type": "keyword"
+            }
+          }
         },
-        "host.arch": {
-          "type": "keyword"
+        "cloud": {
+          "properties": {
+            "provider": {
+              "type": "keyword"
+            },
+            "region": {
+              "type": "keyword"
+            }
+          }
         },
         "profiling": {
           "properties": {

diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson
@@ -1,4 +1,4 @@
 {"create": {"_index": "profiling-hosts","_id":"eLH27YsBj2lLi3tJYlvr"}}
-{"profiling.project.id":100,"host.id":"8457605156473051743","@timestamp":1700504426,"ecs.version":"1.12.0","profiling.agent.build_timestamp":1688111067,"profiling.instance.private_ipv4s":["192.168.1.2"],"ec2.instance_life_cycle":"on-demand","profiling.agent.config.map_scale_factor":0,"ec2.instance_type":"i3.2xlarge","profiling.host.ip":"192.168.1.2","profiling.agent.config.bpf_log_level":0,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.file":"/etc/prodfiler/prodfiler.conf","ec2.local_ipv4":"192.168.1.2","profiling.agent.config.no_kernel_version_check":false,"host.arch":"amd64","profiling.host.tags":["cloud_provider:aws","cloud_environment:qa","cloud_region:eu-west-1"],"profiling.agent.config.probabilistic_threshold":100,"profiling.agent.config.disable_tls":false,"profiling.agent.config.tracers":"all","profiling.agent.start_time":1700090045589,"profiling.agent.config.max_elements_per_interval":800,"ec2.placement.region":"eu-west-1","profiling.agent.config.present_cpu_cores":8,"profiling.host.kernel_version":"9.9.9-0-aws","profiling.agent.config.bpf_log_size":65536,"profiling.agent.config.known_traces_entries":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.verbose":false,"profiling.agent.config.probabilistic_interval":"1m0s","ec2.placement.availability_zone_id":"euw1-az1","ec2.security_groups":"","ec2.local_hostname":"ip-192-168-1-2.eu-west-1.compute.internal","ec2.placement.availability_zone":"eu-west-1c","profiling.agent.config.upload_symbols":false,"profiling.host.sysctl.kernel.bpf_stats_enabled":0,"profiling.host.name":"ip-192-168-1-2","ec2.mac":"00:11:22:33:44:55","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-aws","profiling.agent.config.cache_directory":"/var/cache/optimyze/","profiling.agent.version":"v8.12.0","ec2.hostname":"ip-192-168-1-2.eu-west-1.compute.internal","profiling.agent.config.elastic_mode":false,"ec2.ami_id":"ami-aaaaaaaaaaa","ec2.instance_id":"i-0b999999999999999"}
+{"profiling.project.id":100,"host.id":"8457605156473051743","@timestamp":1700504426,"ecs.version":"1.12.0","profiling.agent.build_timestamp":1688111067,"profiling.instance.private_ipv4s":["192.168.1.2"],"ec2.instance_life_cycle":"on-demand","profiling.agent.config.map_scale_factor":0,"host.type":"i3.2xlarge","profiling.host.ip":"192.168.1.2","profiling.agent.config.bpf_log_level":0,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.file":"/etc/prodfiler/prodfiler.conf","ec2.local_ipv4":"192.168.1.2","profiling.agent.config.no_kernel_version_check":false,"host.arch":"amd64","profiling.host.tags":["cloud_provider:aws","cloud_environment:qa","cloud_region:eu-west-1"],"profiling.agent.config.probabilistic_threshold":100,"profiling.agent.config.disable_tls":false,"profiling.agent.config.tracers":"all","profiling.agent.start_time":1700090045589,"profiling.agent.config.max_elements_per_interval":800,"cloud.provider":"aws","cloud.region":"eu-west-1","profiling.agent.config.present_cpu_cores":8,"profiling.host.kernel_version":"9.9.9-0-aws","profiling.agent.config.bpf_log_size":65536,"profiling.agent.config.known_traces_entries":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.verbose":false,"profiling.agent.config.probabilistic_interval":"1m0s","ec2.placement.availability_zone_id":"euw1-az1","ec2.security_groups":"","ec2.local_hostname":"ip-192-168-1-2.eu-west-1.compute.internal","ec2.placement.availability_zone":"eu-west-1c","profiling.agent.config.upload_symbols":false,"profiling.host.sysctl.kernel.bpf_stats_enabled":0,"profiling.host.name":"ip-192-168-1-2","ec2.mac":"00:11:22:33:44:55","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-aws","profiling.agent.config.cache_directory":"/var/cache/optimyze/","profiling.agent.version":"v8.12.0","ec2.hostname":"ip-192-168-1-2.eu-west-1.compute.internal","profiling.agent.config.elastic_mode":false,"ec2.ami_id":"ami-aaaaaaaaaaa","ec2.instance_id":"i-0b999999999999999"}
 {"create": {"_index": "profiling-hosts", "_id": "u_fHlYwBkmZvQ6tVo1Lr"}}
-{"profiling.project.id":100,"host.id":"7416508186220657211","@timestamp":1703319912,"ecs.version":"1.12.0","profiling.agent.version":"8.11.0","profiling.agent.config.map_scale_factor":0,"profiling.agent.config.probabilistic_threshold":100,"profiling.host.name":"ip-192-186-1-3","profiling.agent.config.no_kernel_version_check":false,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.elastic_mode":false,"azure.compute.vmsize":"Standard_D4s_v3","azure.compute.environment":"AzurePublicCloud","profiling.agent.config.bpf_log_level":0,"profiling.agent.config.known_traces_entries":65536,"profiling.agent.config.ca_address":"example.com:443","profiling.agent.config.tags":"cloud_provider:azure;cloud_environment:qa;cloud_region:eastus2","profiling.host.tags":["cloud_provider:azure","cloud_environment:qa","cloud_region:eastus2"],"profiling.host.kernel_version":"9.9.9-0-azure","profiling.agent.revision":"head-52cc2030","azure.compute.subscriptionid":"1-2-3-4-5","profiling.host.sysctl.kernel.bpf_stats_enabled":0,"host.arch":"amd64","azure.compute.zone":"3","profiling.agent.config.cache_directory":"/var/cache/Elastic/universal-profiling","azure.compute.name":"example-qa-eastus2-001-v1-zone3_6","profiling.agent.config.probabilistic_interval":"1m0s","azure.compute.location":"eastus2","azure.compute.version":"1234.20230510.233254","profiling.instance.private_ipv4s":["192.168.1.3"],"profiling.agent.build_timestamp":1699000836,"profiling.agent.config.file":"/etc/Elastic/universal-profiling/pf-host-agent.conf","profiling.agent.config.bpf_log_size":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.tracers":"all","profiling.agent.config.present_cpu_cores":4,"profiling.agent.start_time":1702306987358,"profiling.agent.config.disable_tls":false,"azure.compute.ostype":"Linux","profiling.host.ip":"192.168.1.3","profiling.agent.config.max_elements_per_interval":400,"profiling.agent.config.upload_symbols":false,"azure.compute.tags":"bootstrap-version:v1;ece-id:001;environment:qa;identifier:v1;initial-config:;managed-by:terraform;monitored-by:core-infrastructure;owner:core-infrastructure;region_type:ess;role:blueprint;secondary_role:;vars-identifier:eastus2-001-v1","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-azure","profiling.agent.config.verbose":false,"azure.compute.vmid":"1-2-3-4-5"}
+{"profiling.project.id":100,"host.id":"7416508186220657211","@timestamp":1703319912,"ecs.version":"1.12.0","profiling.agent.version":"8.11.0","profiling.agent.config.map_scale_factor":0,"profiling.agent.config.probabilistic_threshold":100,"profiling.host.name":"ip-192-186-1-3","profiling.agent.config.no_kernel_version_check":false,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.elastic_mode":false,"host.type":"Standard_D4s_v3","azure.compute.environment":"AzurePublicCloud","profiling.agent.config.bpf_log_level":0,"profiling.agent.config.known_traces_entries":65536,"profiling.agent.config.ca_address":"example.com:443","profiling.agent.config.tags":"cloud_provider:azure;cloud_environment:qa;cloud_region:eastus2","profiling.host.tags":["cloud_provider:azure","cloud_environment:qa","cloud_region:eastus2"],"profiling.host.kernel_version":"9.9.9-0-azure","profiling.agent.revision":"head-52cc2030","azure.compute.subscriptionid":"1-2-3-4-5","profiling.host.sysctl.kernel.bpf_stats_enabled":0,"host.arch":"amd64","azure.compute.zone":"3","profiling.agent.config.cache_directory":"/var/cache/Elastic/universal-profiling","azure.compute.name":"example-qa-eastus2-001-v1-zone3_6","profiling.agent.config.probabilistic_interval":"1m0s","cloud.provider":"azure","cloud.region":"eastus2","azure.compute.version":"1234.20230510.233254","profiling.instance.private_ipv4s":["192.168.1.3"],"profiling.agent.build_timestamp":1699000836,"profiling.agent.config.file":"/etc/Elastic/universal-profiling/pf-host-agent.conf","profiling.agent.config.bpf_log_size":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.tracers":"all","profiling.agent.config.present_cpu_cores":4,"profiling.agent.start_time":1702306987358,"profiling.agent.config.disable_tls":false,"azure.compute.ostype":"Linux","profiling.host.ip":"192.168.1.3","profiling.agent.config.max_elements_per_interval":400,"profiling.agent.config.upload_symbols":false,"azure.compute.tags":"bootstrap-version:v1;ece-id:001;environment:qa;identifier:v1;initial-config:;managed-by:terraform;monitored-by:core-infrastructure;owner:core-infrastructure;region_type:ess;role:blueprint;secondary_role:;vars-identifier:eastus2-001-v1","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-azure","profiling.agent.config.verbose":false,"azure.compute.vmid":"1-2-3-4-5"}
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java
@@ -35,6 +35,45 @@ final class InstanceType implements ToXContentObject {
      * @return the {@link InstanceType}
      */
     public static InstanceType fromHostSource(Map<String, Object> source) {
+        String provider = (String) source.get("cloud.provider");
+        if (provider != null) {
+            String region = (String) source.get("cloud.region");
+            String instanceType = (String) source.get("host.type");
+            return new InstanceType(provider, region, instanceType);
+        }
+
+        // Check and handle pre-8.14.0 host sources for backwards-compatibility.
+        InstanceType instanceType = fromObsoleteHostSource(source);
+        if (instanceType != null) {
+            return instanceType;
+        }
+
+        // Support for configured tags (ECS).
+        // Example of tags:
+        // "profiling.host.tags": [
+        // "cloud_provider:aws",
+        // "cloud_environment:qa",
+        // "cloud_region:eu-west-1",
+        // ],
+        String region = null;
+        List<String> tags = listOf(source.get("profiling.host.tags"));
+        for (String tag : tags) {
+            String[] kv = tag.toLowerCase(Locale.ROOT).split(":", 2);
+            if (kv.length != 2) {
+                continue;
+            }
+            if ("cloud_provider".equals(kv[0])) {
+                provider = kv[1];
+            }
+            if ("cloud_region".equals(kv[0])) {
+                region = kv[1];
+            }
+        }
+
+        return new InstanceType(provider, region, null);
+    }
+
+    private static InstanceType fromObsoleteHostSource(Map<String, Object> source) {
         // Check and handle AWS.
         String region = (String) source.get("ec2.placement.region");
         if (region != null) {
@@ -67,30 +106,7 @@ public static InstanceType fromHostSource(Map<String, Object> source) {
             return new InstanceType("azure", region, instanceType);
         }
 
-        // Support for configured tags (ECS).
-        // Example of tags:
-        // "profiling.host.tags": [
-        // "cloud_provider:aws",
-        // "cloud_environment:qa",
-        // "cloud_region:eu-west-1",
-        // ],
-        String provider = null;
-        region = null;
-        List<String> tags = listOf(source.get("profiling.host.tags"));
-        for (String tag : tags) {
-            String[] kv = tag.toLowerCase(Locale.ROOT).split(":", 2);
-            if (kv.length != 2) {
-                continue;
-            }
-            if ("cloud_provider".equals(kv[0])) {
-                provider = kv[1];
-            }
-            if ("cloud_region".equals(kv[0])) {
-                region = kv[1];
-            }
-        }
-
-        return new InstanceType(provider, region, null);
+        return null;
     }
 
     @SuppressWarnings("unchecked")

diff --git a/...iling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java b/...iling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java
@@ -47,13 +47,14 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry {
     // version 4: Added 'service.name' keyword mapping to profiling-events
     // version 5: Add optional component template '<idx-name>@custom' to all index templates that reference component templates
     // version 6: Added 'host.arch' keyword mapping to profiling-hosts
-    public static final int INDEX_TEMPLATE_VERSION = 6;
+    // version 7: Added 'host.type', 'cloud.provider', 'cloud.region' keyword mappings to profiling-hosts
+    public static final int INDEX_TEMPLATE_VERSION = 7;
 
     // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index
     public static final int PROFILING_EVENTS_VERSION = 2;
     public static final int PROFILING_EXECUTABLES_VERSION = 1;
     public static final int PROFILING_METRICS_VERSION = 1;
-    public static final int PROFILING_HOSTS_VERSION = 1;
+    public static final int PROFILING_HOSTS_VERSION = 2;
     public static final int PROFILING_STACKFRAMES_VERSION = 1;
     public static final int PROFILING_STACKTRACES_VERSION = 1;
     public static final int PROFILING_SYMBOLS_VERSION = 1;

diff --git a/...k/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/HostMetadataTests.java b/...k/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/HostMetadataTests.java
@@ -14,15 +14,41 @@
 
 public class HostMetadataTests extends ESTestCase {
     public void testCreateFromSourceAWS() {
+        final String hostID = "1440256254710195396";
+        final String arch = "amd64";
+        final String provider = "aws";
+        final String region = "eu-west-1";
+        final String instanceType = "md5x.large";
+
+        // tag::noformat
+        HostMetadata host = HostMetadata.fromSource (
+            Map.of (
+                "host.id", hostID,
+                "host.arch", arch,
+                "host.type", instanceType,
+                "cloud.provider", provider,
+                "cloud.region", region
+            )
+        );
+        // end::noformat
+
+        assertEquals(hostID, host.hostID);
+        assertEquals(arch, host.hostArchitecture);
+        assertEquals(provider, host.instanceType.provider);
+        assertEquals(region, host.instanceType.region);
+        assertEquals(instanceType, host.instanceType.name);
+    }
+
+    public void testCreateFromSourceAWSCompat() {
         final String hostID = "1440256254710195396";
         final String arch = "x86_64";
         final String provider = "aws";
         final String region = "eu-west-1";
         final String instanceType = "md5x.large";
 
         // tag::noformat
-        HostMetadata host = HostMetadata.fromSource(
-            Map.of(
+        HostMetadata host = HostMetadata.fromSource (
+            Map.of (
                 "host.id", hostID,
                 "host.arch", arch,
                 "ec2.instance_type", instanceType,
@@ -39,6 +65,32 @@ public void testCreateFromSourceAWS() {
     }
 
     public void testCreateFromSourceGCP() {
+        final String hostID = "1440256254710195396";
+        final String arch = "amd64";
+        final String provider = "gcp";
+        final String[] regions = { "", "", "europe-west1", "europewest", "europe-west1" };
+
+        for (String region : regions) {
+            // tag::noformat
+            HostMetadata host = HostMetadata.fromSource (
+                Map.of (
+                    "host.id", hostID,
+                    "host.arch", arch,
+                    "cloud.provider", provider,
+                    "cloud.region", region
+                )
+            );
+            // end::noformat
+
+            assertEquals(hostID, host.hostID);
+            assertEquals(arch, host.hostArchitecture);
+            assertEquals(provider, host.instanceType.provider);
+            assertEquals(region, host.instanceType.region);
+            assertEquals("", host.instanceType.name);
+        }
+    }
+
+    public void testCreateFromSourceGCPCompat() {
         final String hostID = "1440256254710195396";
         final String arch = "x86_64";
         final String provider = "gcp";
@@ -142,8 +194,8 @@ public void testCreateFromSourceECS() {
             Map.of(
                 "host.id", hostID,
                 "host.arch", arch,
-                "profiling.host.tags", Arrays.asList(
-                    "cloud_provider:"+provider, "cloud_environment:qa", "cloud_region:"+region)
+                "profiling.host.tags", Arrays.asList (
+                    "cloud_provider:" + provider, "cloud_environment:qa", "cloud_region:" + region)
             )
         );
         // end::noformat