feat: sanitize field names

.tav.yml

@@ -348,3 +348,8 @@ finalhandler:
 memcached:
   versions: '>=2.2.0'
   commands: node test/instrumentation/modules/memcached.js
+
+body-parser:
+  versions: '>=1.19.0'
+  commands:
+    - node test/sanitize-field-names/express.js

CHANGELOG.asciidoc

@@ -25,7 +25,6 @@ Notes:
         ==== x.x.x - YYYY/MM/DD
 ////
 
-
 [[release-notes-3.x]]
 === Node.js Agent version 3.x
 
@@ -65,6 +64,11 @@ slightly to commonalize:
 
 * feat: Add `log_level` central config support. {pull}1908[#1908] +
   Spec: https://github.com/elastic/apm/blob/master/specs/agents/logging.md
+* feat: implemented sanitize_feature_names specification +
+  Allows users to configure a list of wildcard patterns to _remove_ items
+  from the agent's HTTP header and `application/x-www-form-urlencoded` payloads.
+  ** https://github.com/elastic/apm/blob/master/specs/agents/sanitization.md[spec]
+  ** https://github.com/elastic/apm-agent-nodejs/blob/master/docs/configuration.asciidoc#sanitize-field-names[docs]
 
 [float]
 ===== Bug fixes

docs/configuration.asciidoc

@@ -706,6 +706,20 @@ The timeout is applied once the agent has sent the entire request body to the AP
 If the response from the server takes longer than allowed by this timeout,
 the HTTP request is terminated and the TCP socket closed.
 
+[[sanitize-field-names]]
+==== `sanitizeFieldNames`
+* *Type:* Array
+* *Default:* `['password', 'passwd', 'pwd', 'secret', '*key', '*token*', '*session*', '*credit*', '*card*', 'authorization', 'set-cookie', 'pw', 'pass', 'connect.sid']`
+* *Env:* `ELASTIC_SANITIZE_FIELD_NAMES`
+
+Remove sensitive data sent to Elastic APM.
+
+The `sanitizeFieldNames` configuration value allows you to configure a list of wildcard patterns of field names which should be redacted from agent payloads. Wildcard matches are
+case-insensitive by default. You may make wildcard searches case-sensitive by
+using the `(?-i)` prefix. These patterns apply to the request and response HTTP headers, as well as any form field captured during an `application/x-www-form-urlencoded` data request.
+
+The `sanitizeFieldNames` will redact any matched _field names_.  If you wish to filter or _redact_ other data the <<filter-http-headers,`filterHttpHeaders`>> configuration field or the <<apm-add-filter,API filtering functions>> may be a better choice.
+
 [[filter-http-headers]]
 ==== `filterHttpHeaders`
 
@@ -723,7 +737,7 @@ the header value will be set to `[REDACTED]`
 
 Currently, the following HTTP headers are anonymized by default:
 
-* `Authorization` - The full value of this header is redacted
+* `Authorization` - The full value of this header is redacted.  In versions of the agent greater than v3.9, the authorization header is redacted by the default configuration of <<sanitize-field-names>>.
 * `Cookie` - The cookies inside the `Cookie` header are analyzed and their values redacted if they appear sensitive (like a session cookie).
   See the https://github.com/watson/is-secret[is-secret] module for details about which patterns are considered sensitive.
 

index.d.ts

@@ -217,6 +217,7 @@ interface AgentConfigOptions {
   metricsInterval?: string; // Also support `number`, but as we're removing this functionality soon, there's no need to advertise it
   payloadLogFile?: string;
   centralConfig?: boolean;
+  sanitizeFieldNames?: Array<string>;
   secretToken?: string;
   serverCaCertFile?: string;
   serverTimeout?: string; // Also support `number`, but as we're removing this functionality soon, there's no need to advertise it

lib/config.js

@@ -66,6 +66,10 @@ var DEFAULTS = {
   logUncaughtExceptions: false, // TODO: Change to `true` in the v4.0.0
   metricsInterval: '30s',
   metricsLimit: 1000,
+  sanitizeFieldNames: ['password', 'passwd', 'pwd', 'secret', '*key', '*token*',
+    '*session*', '*credit*', '*card*', 'authorization', 'set-cookie',
+    'pw', 'pass', 'connect.sid'
+  ],
   serviceNodeName: undefined,
   serverTimeout: '30s',
   sourceLinesErrorAppFrames: 5,
@@ -117,6 +121,7 @@ var ENV_TABLE = {
   metricsInterval: 'ELASTIC_APM_METRICS_INTERVAL',
   metricsLimit: 'ELASTIC_APM_METRICS_LIMIT',
   payloadLogFile: 'ELASTIC_APM_PAYLOAD_LOG_FILE',
+  sanitizeFieldNames: 'ELASTIC_SANITIZE_FIELD_NAMES',
   serverCaCertFile: 'ELASTIC_APM_SERVER_CA_CERT_FILE',
   secretToken: 'ELASTIC_APM_SECRET_TOKEN',
   serverTimeout: 'ELASTIC_APM_SERVER_TIMEOUT',
@@ -142,7 +147,8 @@ var CENTRAL_CONFIG = {
   transaction_sample_rate: 'transactionSampleRate',
   transaction_max_spans: 'transactionMaxSpans',
   capture_body: 'captureBody',
-  transaction_ignore_urls: 'transactionIgnoreUrls'
+  transaction_ignore_urls: 'transactionIgnoreUrls',
+  sanitize_field_names: 'sanitizeFieldNames'
 }
 
 var VALIDATORS = {
@@ -195,6 +201,7 @@ var MINUS_ONE_EQUAL_INFINITY = [
 
 var ARRAY_OPTS = [
   'disableInstrumentations',
+  'sanitizeFieldNames',
   'transactionIgnoreUrls'
 ]
 
@@ -214,6 +221,7 @@ class Config {
     this.ignoreUserAgentStr = []
     this.ignoreUserAgentRegExp = []
     this.transactionIgnoreUrlRegExp = []
+    this.sanitizeFieldNamesRegExp = []
     // If we didn't find a config file on process boot, but a path to one is
     // provided as a config option, let's instead try to load that
     if (confFile === null && opts && opts.configFile) {
@@ -408,9 +416,20 @@ function normalize (opts) {
   normalizeTime(opts)
   normalizeBools(opts)
   normalizeIgnoreOptions(opts)
+  normalizeSanitizeFieldNames(opts)
   truncateOptions(opts)
 }
 
+function normalizeSanitizeFieldNames (opts) {
+  if (opts.sanitizeFieldNames) {
+    const wildcard = new WildcardMatcher()
+    for (const ptn of opts.sanitizeFieldNames) {
+      const re = wildcard.compile(ptn)
+      opts.sanitizeFieldNamesRegExp.push(re)
+    }
+  }
+}
+
 function normalizeIgnoreOptions (opts) {
   if (opts.transactionIgnoreUrls) {
     // We can't guarantee that opts will be a Config so set a

lib/constants.js

@@ -0,0 +1,7 @@
+/**
+ * Central location for shared constants
+ */
+
+module.exports = {
+  REDACTED: '[REDACTED]'
+}

lib/filters/http-headers.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const REDACTED = '[REDACTED]'
+const REDACTED = require('../constants').REDACTED
 
 const cookie = require('cookie')
 const redact = require('../redact-secrets')(REDACTED)
@@ -12,18 +12,28 @@ function httpHeaders (obj) {
   const requestHeaders = obj.context && obj.context.request && obj.context.request.headers
   const responseHeaders = obj.context && obj.context.response && obj.context.response.headers
 
-  if (requestHeaders) filterSensitiveHeaders(requestHeaders)
-  if (responseHeaders) filterSensitiveHeaders(responseHeaders)
+  if (requestHeaders) filterCookieHeaders(requestHeaders)
+  if (responseHeaders) filterCookieHeaders(responseHeaders)
 
   return obj
 }
 
-function filterSensitiveHeaders (headers) {
+/**
+ * Filters cookie _values_ in http headers
+ *
+ * The filterCookieHeaders method filters individual
+ * cookie values
+ */
+function filterCookieHeaders (headers) {
   for (const key in headers) {
+    // undefined headers will be dropped when serialized as
+    // json, and don't need to be redacted.  If a cookie
+    // header is already redacted there's no need to parse its
+    // values.
+    if (headers[key] === undefined || REDACTED === headers[key]) {
+      continue
+    }
     switch (key.toLowerCase()) {
-      case 'authorization':
-        headers[key] = REDACTED
-        break
       case 'cookie':
         if (typeof headers[key] === 'string') {
           const cookies = cookie.parse(headers[key])
@@ -34,16 +44,17 @@ function filterSensitiveHeaders (headers) {
         }
         break
       case 'set-cookie':
-        if (typeof headers[key] !== 'undefined') {
-          try {
-            const setCookies = new SetCookie(headers[key])
-            redact.forEach(setCookies)
-            headers[key] = stringify(setCookies)
-          } catch (err) {
-            // Ignore error
-            headers[key] = '[malformed set-cookie header]'
-          }
+        // if the sanitize_field_names module has redacted this there's
+        // no need to attempt the sanitizaion of individual cookies
+        try {
+          const setCookies = new SetCookie(headers[key])
+          redact.forEach(setCookies)
+          headers[key] = stringify(setCookies)
+        } catch (err) {
+          // Ignore error
+          headers[key] = '[malformed set-cookie header]'
         }
+
         break
     }
   }

lib/filters/sanitize-field-names.js

@@ -0,0 +1,77 @@
+'use strict'
+const querystring = require('querystring')
+
+const HEADER_FORM_URLENCODED = 'application/x-www-form-urlencoded'
+const REDACTED = require('../constants').REDACTED
+
+/**
+ * Handles req.body as object or string
+ *
+ * Express provides multiple body parser middlewares with x-www-form-urlencoded
+ * handling.  See http://expressjs.com/en/resources/middleware/body-parser.html
+ */
+function redactKeysFromPostedFormVariables (body, requestHeaders, regexes) {
+  // only redact from application/x-www-form-urlencoded
+  if (HEADER_FORM_URLENCODED !== requestHeaders['content-type']) {
+    return body
+  }
+
+  // if body is a plain object, use redactKeysFromObject
+  if (body !== null && !Buffer.isBuffer(body) && typeof body === 'object') {
+    return redactKeysFromObject(body, regexes)
+  }
+
+  // if body is a string, use querystring to create object,
+  // pass to redactKeysFromObject, and reserialize as string
+  if (typeof body === 'string') {
+    const objBody = querystring.parse(body)
+    redactKeysFromObject(objBody, regexes)
+    return querystring.stringify(objBody)
+  }
+
+  return body
+}
+
+function redactKeyFromObject (obj, regex) {
+  for (const [key] of Object.entries(obj)) {
+    if (regex.test(key)) {
+      obj[key] = REDACTED
+    }
+  }
+  return obj
+}
+
+function redactKeysFromObject (obj, regexes) {
+  if (!obj || !Array.isArray(regexes)) {
+    return obj
+  }
+  for (const [, regex] of regexes.entries()) {
+    redactKeyFromObject(obj, regex)
+  }
+  return obj
+}
+
+function createFilter (conf) {
+  return sanitizeFieldNames
+
+  function sanitizeFieldNames (obj) {
+    const requestHeaders = obj.context && obj.context.request && obj.context.request.headers
+    const responseHeaders = obj.context && obj.context.response && obj.context.response.headers
+    const body = obj.context && obj.context.request && obj.context.request.body
+
+    redactKeysFromObject(requestHeaders, conf.sanitizeFieldNamesRegExp)
+    redactKeysFromObject(responseHeaders, conf.sanitizeFieldNamesRegExp)
+
+    if (body) {
+      obj.context.request.body = redactKeysFromPostedFormVariables(body, requestHeaders, conf.sanitizeFieldNamesRegExp)
+    }
+
+    return obj
+  }
+}
+
+module.exports = {
+  createFilter,
+  redactKeysFromObject,
+  redactKeysFromPostedFormVariables
+}

lib/parsers.js

@@ -16,6 +16,11 @@ var stackman = require('./stackman')
 
 const _MAX_HTTP_BODY_CHARS = 2048
 
+const {
+  redactKeysFromObject,
+  redactKeysFromPostedFormVariables
+} = require('./filters/sanitize-field-names')
+
 var mysqlErrorMsg = /(ER_[A-Z_]+): /
 
 function parseMessage (msg) {
@@ -145,7 +150,10 @@ function getContextFromRequest (req, conf, type) {
   }
 
   if (conf.captureHeaders) {
-    context.headers = Object.assign({}, req.headers)
+    context.headers = redactKeysFromObject(
+      Object.assign({}, req.headers),
+      conf.sanitizeFieldNamesRegExp
+    )
   }
 
   var contentLength = parseInt(req.headers['content-length'], 10)
@@ -160,6 +168,8 @@ function getContextFromRequest (req, conf, type) {
     } else if (Buffer.isBuffer(body)) {
       context.body = '<Buffer>'
     } else {
+      body = redactKeysFromPostedFormVariables(body, req.headers, conf.sanitizeFieldNamesRegExp)
+
       if (typeof body !== 'string') {
         body = tryJsonStringify(body) || stringify(body)
       }
@@ -186,6 +196,7 @@ function getContextFromResponse (res, conf, isError) {
 
   if (conf.captureHeaders) {
     context.headers = res.headers || httpHeaders(res, true)
+    context.headers = redactKeysFromObject(context.headers, conf.sanitizeFieldNamesRegExp)
   }
 
   if (isError) {

lib/redact-secrets/is-secret.js

@@ -22,6 +22,9 @@
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
+
+const agent = require('../..')
+
 var KEYS = [
   // generic
   /passw(or)?d/i,
@@ -39,14 +42,13 @@ var KEYS = [
 var VALUES = [
   /^\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}$/ // credit card number
 ]
-
 exports.key = key
 exports.value = value
 
 function key (str) {
-  return KEYS.some(function (regex) {
-    return regex.test(str)
-  })
+  // grab current value of agent._conf.sanitizeFieldNames
+  // and include those as keys
+  return KEYS.some((regex) => regex.test(str)) || agent._conf.sanitizeFieldNamesRegExp.some((regex) => regex.test(str))
 }
 
 function value (str) {

package.json

@@ -118,8 +118,10 @@
     "@babel/core": "^7.8.4",
     "@babel/preset-env": "^7.8.4",
     "@elastic/elasticsearch": "^7.10.0",
+    "body-parser": "^1.19.0",
     "@hapi/hapi": "^18.4.1",
     "@koa/router": "^9.0.1",
+    "koa-bodyparser": "^3.2.0",
     "@types/node": "^13.7.4",
     "ajv": "^6.12.6",
     "apollo-server-express": "^2.10.1",
@@ -138,6 +140,7 @@
     "express-graphql": "^0.9.0",
     "express-queue": "^0.0.12",
     "fastify": "^2.12.0",
+    "fastify-formbody": "^3.2",
     "finalhandler": "^1.1.2",
     "generic-pool": "^3.7.1",
     "get-port": "^5.1.1",