fix(parser): DynamoDBStream schema & envelope (#3482)

Co-authored-by: Alexander Schueren <[email protected]>

packages/commons/package.json

@@ -49,6 +49,10 @@
       "import": "./lib/esm/LRUCache.js",
       "require": "./lib/cjs/LRUCache.js"
     },
+    "./utils/unmarshallDynamoDB": {
+      "import": "./lib/esm/unmarshallDynamoDB.js",
+      "require": "./lib/cjs/unmarshallDynamoDB.js"
+    },
     "./types": {
       "import": "./lib/esm/types/index.js",
       "require": "./lib/cjs/types/index.js"
@@ -68,6 +72,10 @@
         "lib/cjs/LRUCache.d.ts",
         "lib/esm/LRUCache.d.ts"
       ],
+      "utils/unmarshallDynamoDB": [
+        "lib/cjs/unmarshallDynamoDB.d.ts",
+        "lib/esm/unmarshallDynamoDB.d.ts"
+      ],
       "types": [
         "lib/cjs/types/index.d.ts",
         "lib/esm/types/index.d.ts"

packages/commons/src/unmarshallDynamoDB.ts

@@ -0,0 +1,83 @@
+import type { AttributeValue } from '@aws-sdk/client-dynamodb';
+
+class UnmarshallDynamoDBAttributeError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'UnmarshallDynamoDBAttributeError';
+  }
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: we need to use any here to support the different types of DynamoDB attributes
+const typeHandlers: Record<string, (value: any) => unknown> = {
+  NULL: () => null,
+  S: (value) => value,
+  B: (value) => value,
+  BS: (value) => new Set(value),
+  SS: (value) => new Set(value),
+  BOOL: (value) => Boolean(value),
+  N: (value) => convertNumber(value),
+  NS: (value) => new Set((value as Array<string>).map(convertNumber)),
+  L: (value) => (value as Array<AttributeValue>).map(convertAttributeValue),
+  M: (value) =>
+    Object.entries(value).reduce(
+      (acc, [key, value]) => {
+        acc[key] = convertAttributeValue(value as AttributeValue);
+        return acc;
+      },
+      {} as Record<string, unknown>
+    ),
+};
+
+const convertAttributeValue = (
+  data: AttributeValue | Record<string, AttributeValue>
+): unknown => {
+  const [type, value] = Object.entries(data)[0];
+
+  if (value !== undefined) {
+    const handler = typeHandlers[type];
+    if (!handler) {
+      throw new UnmarshallDynamoDBAttributeError(
+        `Unsupported type passed: ${type}`
+      );
+    }
+
+    return handler(value);
+  }
+
+  throw new UnmarshallDynamoDBAttributeError(
+    `Value is undefined for type: ${type}`
+  );
+};
+
+const convertNumber = (numString: string) => {
+  const num = Number(numString);
+  const infinityValues = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY];
+  const isLargeFiniteNumber =
+    (num > Number.MAX_SAFE_INTEGER || num < Number.MIN_SAFE_INTEGER) &&
+    !infinityValues.includes(num);
+  if (isLargeFiniteNumber) {
+    try {
+      return BigInt(numString);
+    } catch (error) {
+      throw new UnmarshallDynamoDBAttributeError(
+        `${numString} can't be converted to BigInt`
+      );
+    }
+  }
+  return num;
+};
+
+/**
+ * Unmarshalls a DynamoDB AttributeValue to a JavaScript object.
+ *
+ * The implementation is loosely based on the official AWS SDK v3 unmarshall function but
+ * without support the customization options and with assumed support for BigInt.
+ *
+ * @param data - The DynamoDB AttributeValue to unmarshall
+ */
+const unmarshallDynamoDB = (
+  data: AttributeValue | Record<string, AttributeValue>
+  // @ts-expect-error - We intentionally wrap the data into a Map to allow for nested structures
+) => convertAttributeValue({ M: data });
+
+export { unmarshallDynamoDB, UnmarshallDynamoDBAttributeError };

packages/commons/tests/unit/unmarshallDynamoDB.test.ts

@@ -0,0 +1,246 @@
+import { describe, expect, it } from 'vitest';
+import {
+  UnmarshallDynamoDBAttributeError,
+  unmarshallDynamoDB,
+} from '../../src/unmarshallDynamoDB.js';
+
+describe('Function: unmarshallDynamoDB', () => {
+  it('unmarshalls a DynamoDB string attribute', () => {
+    // Prepare
+    const value = { Message: { S: 'test string' } };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Message: 'test string' });
+  });
+
+  it('unmarshalls a DynamoDB number attribute', () => {
+    // Prepare
+    const value = { Id: { N: '123' } };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Id: 123 });
+  });
+
+  it('unmarshalls a DynamoDB boolean attribute', () => {
+    // Prepare
+    const value = { Message: { BOOL: true } };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Message: true });
+  });
+
+  it('unmarshalls a DynamoDB null attribute', () => {
+    // Prepare
+    const value = { Message: { NULL: true } };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Message: null });
+  });
+
+  it('unmarshalls a DynamoDB list attribute', () => {
+    // Prepare
+    const value = {
+      Messages: {
+        L: [{ S: 'string' }, { N: '123' }, { BOOL: true }, { NULL: true }],
+      },
+    };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Messages: ['string', 123, true, null] });
+  });
+
+  it('unmarshalls a DynamoDB map attribute', () => {
+    // Prepare
+    const value = {
+      Settings: {
+        M: {
+          string: { S: 'test' },
+          number: { N: '123' },
+          boolean: { BOOL: true },
+          null: { NULL: true },
+        },
+      },
+    };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({
+      Settings: {
+        string: 'test',
+        number: 123,
+        boolean: true,
+        null: null,
+      },
+    });
+  });
+
+  it('unmarshalls a DynamoDB string set attribute', () => {
+    // Prepare
+    const value = { Messages: { SS: ['a', 'b', 'c'] } };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Messages: new Set(['a', 'b', 'c']) });
+  });
+
+  it('unmarshalls a DynamoDB number set attribute', () => {
+    // Prepare
+    const value = { Ids: { NS: ['1', '2', '3'] } };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Ids: new Set([1, 2, 3]) });
+  });
+
+  it('unmarshalls nested DynamoDB structures', () => {
+    // Prepare
+    const value = {
+      Messages: {
+        M: {
+          nested: {
+            M: {
+              list: {
+                L: [
+                  { S: 'string' },
+                  {
+                    M: {
+                      key: { S: 'value' },
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        },
+      },
+    };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({
+      Messages: {
+        nested: {
+          list: ['string', { key: 'value' }],
+        },
+      },
+    });
+  });
+
+  it('unmarshalls a DynamoDB binary attribute', () => {
+    // Prepare
+    const value = {
+      Data: {
+        B: new Uint8Array(
+          Buffer.from('dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk', 'base64')
+        ),
+      },
+    };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Data: expect.any(Uint8Array) });
+  });
+
+  it('unmarshalls a DynamoDB binary set attribute', () => {
+    // Prepare
+    const value = {
+      Data: {
+        BS: [
+          new Uint8Array(
+            Buffer.from('dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk', 'base64')
+          ),
+          new Uint8Array(
+            Buffer.from('dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk', 'base64')
+          ),
+        ],
+      },
+    };
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Data: expect.any(Set) });
+  });
+
+  it('throws if an unsupported type is passed', () => {
+    // Prepare
+    const value = { Message: { NNN: '123' } };
+
+    // Act & Assess
+    // @ts-expect-error - Intentionally invalid value
+    expect(() => unmarshallDynamoDB(value)).toThrow(
+      UnmarshallDynamoDBAttributeError
+    );
+  });
+
+  it('unmarshalls a DynamoDB large number attribute to BigInt', () => {
+    // Prepare
+    const value = { Balance: { N: '9007199254740992' } }; // Number.MAX_SAFE_INTEGER + 1
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Balance: BigInt('9007199254740992') });
+  });
+
+  it('unmarshalls a DynamoDB negative large number attribute to BigInt', () => {
+    // Prepare
+    const value = { Balance: { N: '-9007199254740992' } }; // Number.MIN_SAFE_INTEGER - 1
+
+    // Act
+    const result = unmarshallDynamoDB(value);
+
+    // Assess
+    expect(result).toStrictEqual({ Balance: BigInt('-9007199254740992') });
+  });
+
+  it('throws when trying to convert an invalid number string to BigInt', () => {
+    // Prepare
+    const value = { Balance: { N: '9007199254740992.5' } }; // Invalid BigInt string (decimals not allowed)
+
+    // Act & Assess
+    expect(() => unmarshallDynamoDB(value)).toThrow(
+      new UnmarshallDynamoDBAttributeError(
+        "9007199254740992.5 can't be converted to BigInt"
+      )
+    );
+  });
+
+  it('throws when no data is found', () => {
+    // Prepare
+    const value = undefined;
+
+    // Act & Assess
+    // @ts-expect-error - Intentionally invalid value
+    expect(() => unmarshallDynamoDB(value)).toThrow(
+      UnmarshallDynamoDBAttributeError
+    );
+  });
+});