[Extensions.AWSXRay] Add ECS and Logs attributes based on Metadata v4…

… endpoint

src/OpenTelemetry.Contrib.Extensions.AWSXRay/CHANGELOG.md

@@ -5,6 +5,8 @@
 * Enhancement - AWSXRayIdGenerator - Generate X-Ray IDs with global Random
   instance instead of recreating with ThreadLocal
   ([#380](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/380))
+* Enhancement - AWSECSResourceDetector - Implement `aws.{ecs.*,log.*}` resource
+  attributes with data from ECS Metadata endpoint v4
 
 ## 1.2.0
 

src/OpenTelemetry.Contrib.Extensions.AWSXRay/OpenTelemetry.Contrib.Extensions.AWSXRay.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFrameworks>net452;netstandard2.0</TargetFrameworks>
+    <TargetFrameworks>net461;netstandard2.0</TargetFrameworks>
     <Description>OpenTelemetry extensions for AWS X-Ray.</Description>
     <MinVerTagPrefix>Extensions.AWSXRay-</MinVerTagPrefix>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="OpenTelemetry" Version="1.1.0" />
+    <PackageReference Include="OpenTelemetry" Version="$(OpenTelemetryCoreLatestVersion)" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />
   </ItemGroup>
 

src/OpenTelemetry.Contrib.Extensions.AWSXRay/Resources/AWSECSResourceDetector.cs

@@ -16,6 +16,10 @@
 
 using System;
 using System.Collections.Generic;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+
+using Newtonsoft.Json.Linq;
 
 namespace OpenTelemetry.Contrib.Extensions.AWSXRay.Resources;
 
@@ -52,6 +56,15 @@ public IEnumerable<KeyValuePair<string, object>> Detect()
             AWSXRayEventSource.Log.ResourceAttributesExtractException(nameof(AWSECSResourceDetector), ex);
         }
 
+        try
+        {
+            resourceAttributes.AddRange(this.ExtractMetadataV4ResourceAttributes());
+        }
+        catch (Exception ex)
+        {
+            AWSXRayEventSource.Log.ResourceAttributesExtractException(nameof(AWSECSResourceDetector), ex);
+        }
+
         return resourceAttributes;
     }
 
@@ -67,6 +80,78 @@ internal List<KeyValuePair<string, object>> ExtractResourceAttributes(string con
         return resourceAttributes;
     }
 
+    internal List<KeyValuePair<string, object>> ExtractMetadataV4ResourceAttributes()
+    {
+        var metadataV4Url = Environment.GetEnvironmentVariable(AWSECSMetadataURLV4Key);
+        if (Environment.GetEnvironmentVariable(metadataV4Url) == null)
+        {
+            return new List<KeyValuePair<string, object>>();
+        }
+
+        var httpClientHandler = new HttpClientHandler();
+        var metadaV4ContainerResponse = ResourceDetectorUtils.SendOutRequest(metadataV4Url, "GET", null, httpClientHandler).Result;
+        var metadaV4TaskResponse = ResourceDetectorUtils.SendOutRequest($"{metadataV4Url}/task", "GET", null, httpClientHandler).Result;
+
+        var containerResponse = JObject.Parse(metadaV4ContainerResponse);
+        var taskResponse = JObject.Parse(metadaV4TaskResponse);
+
+        var containerArn = (string)containerResponse["ContainerARN"];
+        var clusterArn = (string)taskResponse["Cluster"];
+
+        if (!clusterArn.StartsWith("arn:"))
+        {
+            var baseArn = containerArn.Substring(containerArn.LastIndexOf(":"));
+            clusterArn = $"{baseArn}:cluster/{clusterArn}";
+        }
+
+        var launchType = (string)taskResponse["LaunchType"] switch {
+            string type when "ec2".Equals(type.ToLower()) => AWSSemanticConventions.ValueEcsLaunchTypeEc2,
+            string type when "fargate".Equals(type.ToLower()) => AWSSemanticConventions.ValueEcsLaunchTypeFargate,
+            _ => null,
+        };
+
+        if (launchType == null)
+        {
+            throw new ArgumentOutOfRangeException($"Unrecognized launch type '{taskResponse["LaunchType"]}'");
+        }
+
+        var resourceAttributes = new List<KeyValuePair<string, object>>()
+        {
+            new KeyValuePair<string, object>(AWSSemanticConventions.AttributeEcsContainerArn, containerArn),
+            new KeyValuePair<string, object>(AWSSemanticConventions.AttributeEcsClusterArn, clusterArn),
+            new KeyValuePair<string, object>(AWSSemanticConventions.AttributeEcsLaunchtype, launchType),
+            new KeyValuePair<string, object>(AWSSemanticConventions.AttributeEcsTaskArn, (string)taskResponse["TaskARN"]),
+            new KeyValuePair<string, object>(AWSSemanticConventions.AttributeEcsTaskFamily, (string)taskResponse["Family"]),
+            new KeyValuePair<string, object>(AWSSemanticConventions.AttributeEcsTaskRevision, (string)taskResponse["Revision"]),
+        };
+
+        if ("awslogs".Equals(containerResponse["LogDriver"]))
+        {
+            JObject logOptions = (JObject)containerResponse["LogOptions"];
+
+            var regex = new Regex(@"arn:aws:ecs:([^:]+):([^:]+):.*");
+            var match = regex.Match(containerArn);
+
+            if (!match.Success)
+            {
+                throw new ArgumentOutOfRangeException($"Cannot parse region and account from the container ARN '{containerArn}'");
+            }
+
+            var logsRegion = match.Groups[1];
+            var logsAccount = match.Groups[2];
+
+            var logGroupName = (string)logOptions["AwsLogsGroup"];
+            var logStreamName = (string)logOptions["AwsLogsStream"];
+
+            resourceAttributes.Add(new KeyValuePair<string, object>(AWSSemanticConventions.AttributeLogGroupNames, logGroupName));
+            resourceAttributes.Add(new KeyValuePair<string, object>(AWSSemanticConventions.AttributeLogGroupArns, $"arn:aws:logs:{logsRegion}:{logsAccount}:log-group:{logGroupName}:*"));
+            resourceAttributes.Add(new KeyValuePair<string, object>(AWSSemanticConventions.AttributeLogStreamNames, logStreamName));
+            resourceAttributes.Add(new KeyValuePair<string, object>(AWSSemanticConventions.AttributeLogStreamArns, $"arn:aws:logs:{logsRegion}:{logsAccount}:log-group:{logGroupName}:log-stream:{logStreamName}"));
+        }
+
+        return resourceAttributes;
+    }
+
     internal string GetECSContainerId(string path)
     {
         string containerId = null;

src/OpenTelemetry.Contrib.Extensions.AWSXRay/Resources/AWSSemanticConventions.cs

@@ -26,6 +26,15 @@ internal static class AWSSemanticConventions
 
     public const string AttributeContainerID = "container.id";
 
+    public const string AttributeEcsContainerArn = "aws.ecs.container.arn";
+    public const string AttributeEcsClusterArn = "aws.ecs.cluster.arn";
+    public const string AttributeEcsLaunchtype = "aws.ecs.launchtype";
+    public const string ValueEcsLaunchTypeEc2 = "ec2";
+    public const string ValueEcsLaunchTypeFargate = "fargate";
+    public const string AttributeEcsTaskArn = "aws.ecs.task.arn";
+    public const string AttributeEcsTaskFamily = "aws.ecs.task.family";
+    public const string AttributeEcsTaskRevision = "aws.ecs.task.revision";
+
     public const string AttributeFaasExecution = "faas.execution";
     public const string AttributeFaasID = "faas.id";
     public const string AttributeFaasName = "faas.name";
@@ -37,6 +46,11 @@ internal static class AWSSemanticConventions
 
     public const string AttributeK8SClusterName = "k8s.cluster.name";
 
+    public const string AttributeLogGroupNames = "aws.log.group.names";
+    public const string AttributeLogGroupArns = "aws.log.group.arns";
+    public const string AttributeLogStreamNames = "aws.log.stream.names";
+    public const string AttributeLogStreamArns = "aws.log.stream.arns";
+
     public const string AttributeServiceName = "service.name";
     public const string AttributeServiceNamespace = "service.namespace";
     public const string AttributeServiceInstanceID = "service.instance.id";

src/OpenTelemetry.Contrib.Extensions.AWSXRay/Resources/ResourceDetectorUtils.cs

@@ -29,13 +29,16 @@ namespace OpenTelemetry.Contrib.Extensions.AWSXRay.Resources;
 /// </summary>
 public class ResourceDetectorUtils
 {
-    internal static async Task<string> SendOutRequest(string url, string method, KeyValuePair<string, string> header, HttpClientHandler handler = null)
+    internal static async Task<string> SendOutRequest(string url, string method, KeyValuePair<string, string>? header, HttpClientHandler handler = null)
     {
         using (var httpRequestMessage = new HttpRequestMessage())
         {
             httpRequestMessage.RequestUri = new Uri(url);
             httpRequestMessage.Method = new HttpMethod(method);
-            httpRequestMessage.Headers.Add(header.Key, header.Value);
+            if (header.HasValue)
+            {
+                httpRequestMessage.Headers.Add(header.Value.Key, header.Value.Value);
+            }
 
             var httpClient = handler == null ? new HttpClient() : new HttpClient(handler);
             using (var response = await httpClient.SendAsync(httpRequestMessage))

test/OpenTelemetry.Contrib.Extensions.AWSXRay.Tests/Resources/ecs_metadata/metadatav4-response-container-ec2.json

@@ -0,0 +1,44 @@
+{
+    "DockerId": "ea32192c8553fbff06c9340478a2ff089b2bb5646fb718b4ee206641c9086d66",
+    "Name": "curl",
+    "DockerName": "ecs-curltest-24-curl-cca48e8dcadd97805600",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/8f03e41243824aea923aca126495f665",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "24"
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {
+        "CPU": 10,
+        "Memory": 128
+    },
+    "CreatedAt": "2020-10-02T00:15:07.620912337Z",
+    "StartedAt": "2020-10-02T00:15:08.062559351Z",
+    "Type": "NORMAL",
+    "LogDriver": "awslogs",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/metadata",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/8f03e41243824aea923aca126495f665"
+    },
+    "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": [
+                "10.0.2.100"
+            ],
+            "AttachmentIndex": 0,
+            "MACAddress": "0e:9e:32:c7:48:85",
+            "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+            "PrivateDNSName": "ip-10-0-2-100.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "10.0.2.1/24"
+        }
+    ]
+}

test/OpenTelemetry.Contrib.Extensions.AWSXRay.Tests/Resources/ecs_metadata/metadatav4-response-container-fargate.json

@@ -0,0 +1,50 @@
+{
+    "DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
+    "Name": "curl",
+    "DockerName": "curl",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "2"
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {
+        "CPU": 10,
+        "Memory": 128
+    },
+    "CreatedAt": "2020-10-08T20:09:11.44527186Z",
+    "StartedAt": "2020-10-08T20:09:11.44527186Z",
+    "Type": "NORMAL",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": [
+                "192.0.2.3"
+            ],
+            "AttachmentIndex": 0,
+            "MACAddress": "0a:de:f6:10:51:e5",
+            "IPv4SubnetCIDRBlock": "192.0.2.0/24",
+            "DomainNameServers": [
+                "192.0.2.2"
+            ],
+            "DomainNameSearchList": [
+                "us-west-2.compute.internal"
+            ],
+            "PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "192.0.2.0/24"
+        }
+    ],
+    "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/containerlogs",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50"
+    },
+    "LogDriver": "awslogs"
+}

test/OpenTelemetry.Contrib.Extensions.AWSXRay.Tests/Resources/ecs_metadata/metadatav4-response-task-ec2.json

@@ -0,0 +1,94 @@
+{
+    "Cluster": "default",
+    "TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+    "Family": "curltest",
+    "Revision": "26",
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "PullStartedAt": "2020-10-02T00:43:06.202617438Z",
+    "PullStoppedAt": "2020-10-02T00:43:06.31288465Z",
+    "AvailabilityZone": "us-west-2d",
+    "LaunchType": "EC2",
+    "Containers": [
+        {
+            "DockerId": "598cba581fe3f939459eaba1e071d5c93bb2c49b7d1ba7db6bb19deeb70d8e38",
+            "Name": "~internal~ecs~pause",
+            "DockerName": "ecs-curltest-26-internalecspause-e292d586b6f9dade4a00",
+            "Image": "amazon/amazon-ecs-pause:0.1.0",
+            "ImageID": "",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "default",
+                "com.amazonaws.ecs.container-name": "~internal~ecs~pause",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "26"
+            },
+            "DesiredStatus": "RESOURCES_PROVISIONED",
+            "KnownStatus": "RESOURCES_PROVISIONED",
+            "Limits": {
+                "CPU": 0,
+                "Memory": 0
+            },
+            "CreatedAt": "2020-10-02T00:43:05.602352471Z",
+            "StartedAt": "2020-10-02T00:43:06.076707576Z",
+            "Type": "CNI_PAUSE",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": [
+                        "10.0.2.61"
+                    ],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "0e:10:e2:01:bd:91",
+                    "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+                    "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "10.0.2.1/24"
+                }
+            ]
+        },
+        {
+            "DockerId": "ee08638adaaf009d78c248913f629e38299471d45fe7dc944d1039077e3424ca",
+            "Name": "curl",
+            "DockerName": "ecs-curltest-26-curl-a0e7dba5aca6d8cb2e00",
+            "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+            "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "default",
+                "com.amazonaws.ecs.container-name": "curl",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "26"
+            },
+            "DesiredStatus": "RUNNING",
+            "KnownStatus": "RUNNING",
+            "Limits": {
+                "CPU": 10,
+                "Memory": 128
+            },
+            "CreatedAt": "2020-10-02T00:43:06.326590752Z",
+            "StartedAt": "2020-10-02T00:43:06.767535449Z",
+            "Type": "NORMAL",
+            "LogDriver": "awslogs",
+            "LogOptions": {
+                "awslogs-create-group": "true",
+                "awslogs-group": "/ecs/metadata",
+                "awslogs-region": "us-west-2",
+                "awslogs-stream": "ecs/curl/158d1c8083dd49d6b527399fd6414f5c"
+            },
+            "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/abb51bdd-11b4-467f-8f6c-adcfe1fe059d",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": [
+                        "10.0.2.61"
+                    ],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "0e:10:e2:01:bd:91",
+                    "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+                    "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "10.0.2.1/24"
+                }
+            ]
+        }
+    ]
+}