feat(utils): make data redacters settings case insensitive (#48)

* chore(deps): update dependencies
* feat(utils): make data redacter settings case-insensitive
* chore(tests): add tests for utils

---------

Co-authored-by: DataLens Team <[email protected]>

package.json

@@ -34,7 +34,6 @@
     "@types/jaeger-client": "^3.18.4",
     "@types/jest": "^29.2.3",
     "@types/node": "^18.11.9",
-    "@types/pino": "^7.0.5",
     "axios": "^1.4.0",
     "eslint": "^8.28.0",
     "husky": "^8.0.2",

src/lib/consts.ts

@@ -2,3 +2,4 @@ export const REQUEST_ID_HEADER = 'x-request-id';
 export const REQUEST_ID_PARAM_NAME = 'requestId';
 export const USER_ID_PARAM_NAME = 'userId';
 export const TRACE_KEY = 'uber-trace-id';
+export const REDACTED_STRING = '[REDACTED]';

src/lib/utils/redact-sensitive-headers.ts

@@ -15,12 +15,16 @@ export default function prepareSensitiveHeadersRedacter(
             return inputHeaders;
         }
 
+        const headersWithSensitiveUrlsLowered = headersWithSensitiveUrls.map((name) =>
+            name.toLowerCase(),
+        );
+
         const redactSensitiveKeys = prepareSensitiveKeysRedacter(sensitiveHeaders);
 
         const result = redactSensitiveKeys(inputHeaders) as IncomingHttpHeaders;
 
         Object.keys(result).forEach((headerName) => {
-            if (headersWithSensitiveUrls.includes(headerName.toLowerCase())) {
+            if (headersWithSensitiveUrlsLowered.includes(headerName.toLowerCase())) {
                 result[headerName] = redactSensitiveQueryParams(result[headerName] as string);
             }
         });

src/lib/utils/redact-sensitive-keys.ts

@@ -1,12 +1,14 @@
 import {Dict} from '../../types';
+import {REDACTED_STRING} from '../consts';
 
 export type SensitiveKeysRedacter = (inputObject: Dict) => Dict;
 
 export function prepareSensitiveKeysRedacter(keysToRemove: string[] = []) {
+    const loweredKeysToRemove = keysToRemove.map((key) => key.toLowerCase());
     const redactSensitiveKeys: SensitiveKeysRedacter = (inputObject: Dict) => {
         return Object.keys(inputObject).reduce((result, key) => {
-            if (keysToRemove.includes(key.toLowerCase())) {
-                result[key] = '[REDACTED]';
+            if (loweredKeysToRemove.includes(key.toLowerCase())) {
+                result[key] = REDACTED_STRING;
             } else {
                 result[key] = inputObject[key];
             }

src/tests/batch.test.ts → src/tests/utils/batch.test.ts

@@ -1,6 +1,4 @@
-'use strict';
-
-import {DEFAULT_TICK_INTERVAL, prepareBatchedQueue} from '../lib/utils/batch';
+import {DEFAULT_TICK_INTERVAL, prepareBatchedQueue} from '../../lib/utils/batch';
 
 jest.useFakeTimers({legacyFakeTimers: true});
 

src/tests/utils/is-true-env.test.ts

@@ -0,0 +1,13 @@
+import {isTrueEnvValue} from '../../lib/utils/is-true-env';
+
+it('successfully checks value for truthfulness', () => {
+    expect(isTrueEnvValue('true')).toEqual(true);
+    expect(isTrueEnvValue('1')).toEqual(true);
+});
+
+it('successfully checks value for untruthfulness', () => {
+    expect(isTrueEnvValue('false')).toEqual(false);
+    expect(isTrueEnvValue('0')).toEqual(false);
+    expect(isTrueEnvValue('')).toEqual(false);
+    expect(isTrueEnvValue(undefined)).toEqual(false);
+});

src/tests/utils/redact-sensitive-headers.test.ts

@@ -0,0 +1,43 @@
+import {REDACTED_STRING} from '../../lib/consts';
+import prepareSensitiveHeadersRedacter from '../../lib/utils/redact-sensitive-headers';
+import prepareSensitiveQueryParamsRedacter from '../../lib/utils/redact-sensitive-query-params';
+import {NodeKit} from '../../nodekit';
+
+it('correctly removes sensitive data from headers', () => {
+    const inputHeaders = {
+        Cookie: 'some-cookie-value',
+        SomeHeader: 'non-secret-header',
+        Referer: 'https://example.com/?someSecretParameter=secretValue',
+    };
+
+    const queryParamsRedacter = prepareSensitiveQueryParamsRedacter(['someSecretParameter']);
+    const headersRedacter = prepareSensitiveHeadersRedacter(
+        ['cookie'],
+        ['referer'],
+        queryParamsRedacter,
+    );
+
+    const redactedHeaders = headersRedacter(inputHeaders);
+
+    expect(redactedHeaders['Cookie']).toEqual(REDACTED_STRING);
+
+    const redactedRefererParams = new URL(redactedHeaders['Referer'] as string).searchParams;
+    expect(redactedRefererParams.get('someSecretParameter')).toEqual(REDACTED_STRING);
+});
+
+it('correctly removes sensitive data from headers using default config', () => {
+    const inputHeaders = {
+        Cookie: 'some-cookie-value',
+        SomeHeader: 'non-secret-header',
+        Referer: 'https://example.com/?token=secretValue',
+    };
+
+    const nk = new NodeKit({config: {appSensitiveQueryParams: ['token']}});
+
+    const redactedHeaders = nk.utils.redactSensitiveHeaders(inputHeaders);
+
+    expect(redactedHeaders['Cookie']).toEqual(REDACTED_STRING);
+
+    const redactedRefererParams = new URL(redactedHeaders['Referer'] as string).searchParams;
+    expect(redactedRefererParams.get('token')).toEqual(REDACTED_STRING);
+});

src/tests/utils/redact-sensitive-keys.test.ts

@@ -0,0 +1,97 @@
+import {REDACTED_STRING} from '../../lib/consts';
+import {prepareSensitiveKeysRedacter} from '../../lib/utils/redact-sensitive-keys';
+import {NodeKit} from '../../nodekit';
+import {Dict} from '../../types';
+
+function getTestData() {
+    return {
+        someString: 'lorem',
+        anotherString: 'ipsum',
+        someNumber: 100,
+        someObject: {
+            someValueInObject: 'hello',
+        },
+        verySensitiveValue: 42,
+        verySensitiveObject: {
+            someDataInSensitiveObject: 200,
+        },
+        someNonSensitiveObject: {
+            verySensitiveValue: 300,
+        },
+        VERYSENSITIVEVALUE: 42,
+        verysensitivevalue: 42,
+    };
+}
+
+function getTestConfiguration() {
+    return ['verysensitivevalue', 'verysensitiveobject'];
+}
+
+it('removes sensitive data from the input object', () => {
+    const redactSensitiveKeys = prepareSensitiveKeysRedacter(getTestConfiguration());
+    const redactedData = redactSensitiveKeys(getTestData());
+
+    expect(redactedData.verySensitiveValue).toEqual(REDACTED_STRING);
+    expect(redactedData.verySensitiveObject).toEqual(REDACTED_STRING);
+});
+
+it('removes sensitive keys regardless of their case', () => {
+    const redactSensitiveKeys = prepareSensitiveKeysRedacter(getTestConfiguration());
+    const redactedData = redactSensitiveKeys(getTestData());
+
+    expect(redactedData.VERYSENSITIVEVALUE).toEqual(REDACTED_STRING);
+    expect(redactedData.verysensitivevalue).toEqual(REDACTED_STRING);
+});
+
+it('removes sensitive keys regardless of case in configuration', () => {
+    const redactSensitiveKeys = prepareSensitiveKeysRedacter(
+        getTestConfiguration().map((s) => s.toUpperCase()),
+    );
+    const redactedData = redactSensitiveKeys(getTestData());
+
+    expect(redactedData.VERYSENSITIVEVALUE).toEqual(REDACTED_STRING);
+    expect(redactedData.verysensitivevalue).toEqual(REDACTED_STRING);
+});
+
+it('does not affect data inside objects', () => {
+    const redactSensitiveKeys = prepareSensitiveKeysRedacter(getTestConfiguration());
+    const redactedData = redactSensitiveKeys(getTestData());
+
+    expect((redactedData.someNonSensitiveObject as Dict).verySensitiveValue).toEqual(300);
+});
+
+it('contains default sensitive values', () => {
+    const nk = new NodeKit();
+
+    const inputData = {
+        nonSensitiveData: 42,
+        authorization: 'some-auth-token',
+        cookie: 'some-cookie',
+    };
+    const redactedData = nk.utils.redactSensitiveKeys(inputData);
+
+    expect(redactedData.nonSensitiveData).toEqual(42);
+    expect(redactedData.authorization).toEqual(REDACTED_STRING);
+    expect(redactedData.cookie).toEqual(REDACTED_STRING);
+});
+
+it('conbines default sensitive values with additional from configuration', () => {
+    const nk = new NodeKit({
+        config: {
+            appSensitiveKeys: ['appLevelSensitiveKey'],
+        },
+    });
+
+    const inputData = {
+        nonSensitiveData: 42,
+        authorization: 'some-auth-token',
+        cookie: 'some-cookie',
+        appLevelSensitiveKey: 'some-data',
+    };
+    const redactedData = nk.utils.redactSensitiveKeys(inputData);
+
+    expect(redactedData.nonSensitiveData).toEqual(42);
+    expect(redactedData.authorization).toEqual(REDACTED_STRING);
+    expect(redactedData.cookie).toEqual(REDACTED_STRING);
+    expect(redactedData.appLevelSensitiveKey).toEqual(REDACTED_STRING);
+});

src/tests/utils/redact-sensitive-query-params.test.ts

@@ -0,0 +1,16 @@
+import {REDACTED_STRING} from '../../lib/consts';
+import prepareSensitiveQueryParamsRedacter from '../../lib/utils/redact-sensitive-query-params';
+
+it('removes value of sensitive query parameters', () => {
+    const redactSensitiveQueryParams = prepareSensitiveQueryParamsRedacter(['someSensitiveKey']);
+    const inputUrl =
+        'https://example.com/some/path?foo=42&someSensitiveKey=sensitiveData&someOtherData=hello';
+
+    const redactedUrl = redactSensitiveQueryParams(inputUrl);
+    expect(redactedUrl.includes('sensitiveData')).toBe(false);
+
+    const redactedParams = new URL(redactedUrl).searchParams;
+    expect(redactedParams.get('foo')).toEqual('42');
+    expect(redactedParams.get('someOtherData')).toEqual('hello');
+    expect(redactedParams.get('someSensitiveKey')).toEqual(REDACTED_STRING);
+});