report aws-sdk sns payloads as tags

packages/datadog-plugin-aws-sdk/src/base.js

@@ -4,9 +4,11 @@ const analyticsSampler = require('../../dd-trace/src/analytics_sampler')
 const ClientPlugin = require('../../dd-trace/src/plugins/client')
 const { storage } = require('../../datadog-core')
 const { isTrue } = require('../../dd-trace/src/util')
+const { tagsFromRequest, tagsFromResponse } = require('../../dd-trace/src/payload-tagging')
 
 class BaseAwsSdkPlugin extends ClientPlugin {
   static get id () { return 'aws' }
+  static get isPayloadReporter () { return false }
 
   get serviceIdentifier () {
     const id = this.constructor.id.toLowerCase()
@@ -19,6 +21,12 @@ class BaseAwsSdkPlugin extends ClientPlugin {
     return id
   }
 
+  get cloudTaggingConfig () { return this._tracerConfig.cloudPayloadTagging }
+
+  get payloadTaggingRules () {
+    return this.cloudTaggingConfig.rules['aws']?.[this.constructor.id]
+  }
+
   constructor (...args) {
     super(...args)
 
@@ -50,6 +58,12 @@ class BaseAwsSdkPlugin extends ClientPlugin {
 
       this.requestInject(span, request)
 
+      if (this.constructor.isPayloadReporter && this.cloudTaggingConfig.requestsEnabled) {
+        const maxDepth = this.cloudTaggingConfig.maxDepth
+        const requestTags = tagsFromRequest(this.payloadTaggingRules, request.params, { maxDepth })
+        span.addTags(requestTags)
+      }
+
       const store = storage.getStore()
 
       this.enter(span, store)
@@ -109,13 +123,30 @@ class BaseAwsSdkPlugin extends ClientPlugin {
     const params = response.request.params
     const operation = response.request.operation
     const extraTags = this.generateTags(params, operation, response) || {}
+
     const tags = Object.assign({
       'aws.response.request_id': response.requestId,
       'resource.name': operation,
       'span.kind': 'client'
     }, extraTags)
 
     span.addTags(tags)
+
+    if (this.constructor.isPayloadReporter && this.cloudTaggingConfig.responsesEnabled) {
+      const maxDepth = this.cloudTaggingConfig.maxDepth
+      const responseBody = this.extractResponseBody(response)
+      const responseTags = tagsFromResponse(this.payloadTaggingRules, responseBody, { maxDepth })
+      span.addTags(responseTags)
+    }
+  }
+
+  extractResponseBody (response) {
+    if (response.hasOwnProperty('data')) {
+      return response.data
+    }
+    return Object.fromEntries(
+      Object.entries(response).filter(([key]) => !['request', 'requestId', 'error', '$metadata'].includes(key))
+    )
   }
 
   generateTags () {

packages/datadog-plugin-aws-sdk/src/services/sns.js

@@ -7,6 +7,7 @@ const BaseAwsSdkPlugin = require('../base')
 class Sns extends BaseAwsSdkPlugin {
   static get id () { return 'sns' }
   static get peerServicePrecursors () { return ['topicname'] }
+  static get isPayloadReporter () { return true }
 
   generateTags (params, operation, response) {
     if (!params) return {}
@@ -20,6 +21,7 @@ class Sns extends BaseAwsSdkPlugin {
 
     // Get the topic name from the last part of the ARN
     const topicName = arnParts[arnParts.length - 1]
+
     return {
       'resource.name': `${operation} ${params.TopicArn || response.data.TopicArn}`,
       'aws.sns.topic_arn': TopicArn,

packages/datadog-plugin-aws-sdk/test/sns.spec.js

@@ -80,6 +80,246 @@ describe('Sns', () => {
       })
     }
 
+    describe('with payload tagging', () => {
+      before(() => {
+        parentId = '0'
+        spanId = '0'
+
+        return agent.load('aws-sdk', {}, {
+          cloudPayloadTagging: {
+            request: ['$.MessageAttributes.foo', '$.MessageAttributes.redacted.StringValue.foo'],
+            response: ['$.MessageId', '$.Attributes.DisplayName']
+          }
+        })
+      })
+
+      before(done => {
+        tracer = require('../../dd-trace')
+        tracer.use('aws-sdk')
+
+        createResources('TestQueue', 'TestTopic', done)
+      })
+
+      after(done => {
+        sns.deleteTopic({ TopicArn }, done)
+      })
+
+      after(done => {
+        sqs.deleteQueue({ QueueUrl }, done)
+      })
+
+      after(() => {
+        return agent.close({ ritmReset: false, wipe: true })
+      })
+
+      it('adds request and response payloads as flattened tags', done => {
+        agent.use(traces => {
+          const span = traces[0][0]
+
+          expect(span.resource).to.equal(`publish ${TopicArn}`)
+          expect(span.meta).to.include({
+            'aws.sns.topic_arn': TopicArn,
+            'topicname': 'TestTopic',
+            'aws_service': 'SNS',
+            'region': 'us-east-1',
+            'aws.request.body.TopicArn': TopicArn,
+            'aws.request.body.Message': 'message 1',
+            'aws.request.body.MessageAttributes.baz.DataType': 'String',
+            'aws.request.body.MessageAttributes.baz.StringValue': 'bar',
+            'aws.request.body.MessageAttributes.keyOne.DataType': 'String',
+            'aws.request.body.MessageAttributes.keyOne.StringValue': 'keyOne',
+            'aws.request.body.MessageAttributes.keyTwo.DataType': 'String',
+            'aws.request.body.MessageAttributes.keyTwo.StringValue': 'keyTwo',
+            'aws.response.body.MessageId': 'redacted'
+          })
+        }).then(done, done)
+
+        sns.publish({
+          TopicArn,
+          Message: 'message 1',
+          MessageAttributes: {
+            baz: { DataType: 'String', StringValue: 'bar' },
+            keyOne: { DataType: 'String', StringValue: 'keyOne' },
+            keyTwo: { DataType: 'String', StringValue: 'keyTwo' }
+          }
+        }, e => e && done(e))
+      })
+
+      it('expands and redacts keys identified as expandable', done => {
+        agent.use(traces => {
+          const span = traces[0][0]
+
+          expect(span.resource).to.equal(`publish ${TopicArn}`)
+          expect(span.meta).to.include({
+            'aws.sns.topic_arn': TopicArn,
+            'topicname': 'TestTopic',
+            'aws_service': 'SNS',
+            'region': 'us-east-1',
+            'aws.request.body.TopicArn': TopicArn,
+            'aws.request.body.Message': 'message 1',
+            'aws.request.body.MessageAttributes.redacted.StringValue.foo': 'redacted',
+            'aws.request.body.MessageAttributes.unredacted.StringValue.foo': 'bar',
+            'aws.request.body.MessageAttributes.unredacted.StringValue.baz': 'yup',
+            'aws.response.body.MessageId': 'redacted'
+          })
+        }).then(done, done)
+
+        sns.publish({
+          TopicArn,
+          Message: 'message 1',
+          MessageAttributes: {
+            unredacted: { DataType: 'String', StringValue: '{"foo": "bar", "baz": "yup"}' },
+            redacted: { DataType: 'String', StringValue: '{"foo": "bar"}' }
+          }
+        }, e => e && done(e))
+      })
+
+      describe('user-defined redaction', () => {
+        it('redacts user-defined keys to suppress in request', done => {
+          agent.use(traces => {
+            const span = traces[0][0]
+
+            expect(span.resource).to.equal(`publish ${TopicArn}`)
+            expect(span.meta).to.include({
+              'aws.sns.topic_arn': TopicArn,
+              'topicname': 'TestTopic',
+              'aws_service': 'SNS',
+              'region': 'us-east-1',
+              'aws.request.body.TopicArn': TopicArn,
+              'aws.request.body.Message': 'message 1',
+              'aws.request.body.MessageAttributes.foo': 'redacted',
+              'aws.request.body.MessageAttributes.keyOne.DataType': 'String',
+              'aws.request.body.MessageAttributes.keyOne.StringValue': 'keyOne',
+              'aws.request.body.MessageAttributes.keyTwo.DataType': 'String',
+              'aws.request.body.MessageAttributes.keyTwo.StringValue': 'keyTwo'
+            })
+            expect(span.meta).to.have.property('aws.response.body.MessageId')
+          }).then(done, done)
+
+          sns.publish({
+            TopicArn,
+            Message: 'message 1',
+            MessageAttributes: {
+              foo: { DataType: 'String', StringValue: 'bar' },
+              keyOne: { DataType: 'String', StringValue: 'keyOne' },
+              keyTwo: { DataType: 'String', StringValue: 'keyTwo' }
+            }
+          }, e => e && done(e))
+        })
+
+        // TODO add response tests
+        it('redacts user-defined keys to suppress in response', done => {
+          agent.use(traces => {
+            const span = traces[0][0]
+            expect(span.resource).to.equal(`getTopicAttributes ${TopicArn}`)
+            expect(span.meta).to.include({
+              'aws.sns.topic_arn': TopicArn,
+              'topicname': 'TestTopic',
+              'aws_service': 'SNS',
+              'region': 'us-east-1',
+              'aws.request.body.TopicArn': TopicArn,
+              'aws.response.body.Attributes.DisplayName': 'redacted'
+            })
+          }).then(done, done)
+
+          sns.getTopicAttributes({ TopicArn }, e => e && done(e))
+        })
+      })
+
+      describe('redaction of internally suppressed keys', () => {
+        const supportsSMSNotification = (moduleName, version) => {
+          switch (moduleName) {
+            case 'aws-sdk':
+              // aws-sdk-js phone notifications introduced in c6d1bb1a
+              return semver.intersects(version, '>=2.10.0')
+            case '@aws-sdk/smithy-client':
+              return true
+            default:
+              return false
+          }
+        }
+
+        if (supportsSMSNotification(moduleName, version)) {
+          // TODO test this
+          describe.skip('phone number', () => {
+            before(done => {
+              sns.createSMSSandboxPhoneNumber({ PhoneNumber: '+33628606135' }, err => err && done(err))
+              sns.createSMSSandboxPhoneNumber({ PhoneNumber: '+33628606136' }, err => err && done(err))
+            })
+
+            after(done => {
+              sns.deleteSMSSandboxPhoneNumber({ PhoneNumber: '+33628606135' }, err => err && done(err))
+              sns.deleteSMSSandboxPhoneNumber({ PhoneNumber: '+33628606136' }, err => err && done(err))
+            })
+
+            it('redacts phone numbers in request', done => {
+              agent.use(traces => {
+                const span = traces[0][0]
+
+                expect(span.resource).to.equal(`publish`)
+                expect(span.meta).to.include({
+                  'aws_service': 'SNS',
+                  'region': 'us-east-1',
+                  'aws.request.body.PhoneNumber': 'redacted',
+                  'aws.request.body.Message': 'message 1'
+                })
+              }).then(done, done)
+
+              sns.publish({
+                PhoneNumber: '+33628606135',
+                Message: 'message 1'
+              }, e => e && done(e))
+            })
+
+            it('redacts phone numbers in response', done => {
+              agent.use(traces => {
+                const span = traces[0][0]
+
+                expect(span.resource).to.equal(`publish`)
+                expect(span.meta).to.include({
+                  'aws_service': 'SNS',
+                  'region': 'us-east-1',
+                  'aws.response.body.PhoneNumber': 'redacted'
+                })
+              }).then(done, done)
+
+              sns.listSMSSandboxPhoneNumbers({
+                PhoneNumber: '+33628606135',
+                Message: 'message 1'
+              }, e => e && done(e))
+            })
+          })
+        }
+
+        describe.skip('subscription confirmation tokens', () => {
+          // TODO test this
+          it('redacts tokens in request', done => {
+            agent.use(traces => {
+              const span = traces[0][0]
+
+              expect(span.resource).to.equal(`publish`)
+              expect(span.meta).to.include({
+                'aws_service': 'SNS',
+                'aws.sns.topic_arn': TopicArn,
+                'topicname': 'TestTopic',
+                'region': 'us-east-1',
+                'aws.request.body.Token': 'redacted',
+                'aws.request.body.TopicArn': 'TestTopic'
+              })
+            }).then(done, done)
+
+            sns.confirmSubscription({
+              TopicArn,
+              Token: '1234'
+            }, e => e && done(e))
+          })
+          it('redacts tokens in response', () => {
+
+          })
+        })
+      })
+    })
+
     describe('no configuration', () => {
       before(() => {
         parentId = '0'
@@ -252,7 +492,7 @@ describe('Sns', () => {
       })
 
       after(() => {
-        return agent.close({ ritmReset: false })
+        return agent.close({ ritmReset: false, wipe: true })
       })
 
       afterEach(() => {

packages/dd-trace/src/payload-tagging/config/aws.json

@@ -0,0 +1,30 @@
+{
+  "sns": {
+    "request": [
+      "$.Attributes.KmsMasterKeyId",
+      "$.Attributes.PlatformCredential",
+      "$.Attributes.PlatformPrincipal",
+      "$.Attributes.Token",
+      "$.AWSAccountId",
+      "$.Endpoint",
+      "$.OneTimePassword",
+      "$.phoneNumber",
+      "$.PhoneNumber",
+      "$.Token"
+    ],
+    "response": [
+      "$.Attributes.KmsMasterKeyId",
+      "$.Attributes.Token",
+      "$.Endpoints.*.Token",
+      "$.PhoneNumber",
+      "$.PhoneNumbers",
+      "$.phoneNumbers",
+      "$.PlatformApplication.*.PlatformCredential",
+      "$.PlatformApplication.*.PlatformPrincipal",
+      "$.Subscriptions.*.Endpoint"
+    ],
+    "expand": [
+      "$.MessageAttributes.*.StringValue"
+    ]
+  }
+}

packages/dd-trace/src/payload-tagging/config/index.js

@@ -1,4 +1,5 @@
-const sdks = { }
+const aws = require('./aws.json')
+const sdks = { aws }
 
 function getSDKRules (sdk, requestInput, responseInput) {
   return Object.fromEntries(