feat: smithy rpcv2 cbor protocol

.changeset/itchy-zebras-yawn.md

@@ -0,0 +1,5 @@
+---
+"@smithy/core": minor
+---
+
+cbor (de)serializer for JS

packages/core/cbor.js

@@ -0,0 +1,6 @@
+
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/cbor/index.js");

packages/core/package.json

@@ -11,7 +11,7 @@
     "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo || exit 0",
     "lint": "npx eslint -c ../../.eslintrc.js \"src/**/*.ts\" && node ./scripts/lint",
     "format": "prettier --config ../../prettier.config.js --ignore-path ../.prettierignore --write \"**/*.{ts,md,json}\"",
-    "test": "yarn g:jest"
+    "test": "yarn g:jest --maxWorkers 1"
   },
   "main": "./dist-cjs/index.js",
   "module": "./dist-es/index.js",
@@ -27,6 +27,12 @@
       "node": "./package.json",
       "import": "./package.json",
       "require": "./package.json"
+    },
+    "./cbor": {
+      "node": "./dist-cjs/submodules/cbor/index.js",
+      "import": "./dist-es/submodules/cbor/index.js",
+      "require": "./dist-cjs/submodules/cbor/index.js",
+      "types": "./dist-types/submodules/cbor/index.d.ts"
     }
   },
   "author": {
@@ -43,6 +49,7 @@
     "@smithy/smithy-client": "workspace:^",
     "@smithy/types": "workspace:^",
     "@smithy/util-middleware": "workspace:^",
+    "@smithy/util-utf8": "workspace:^",
     "tslib": "^2.6.2"
   },
   "engines": {
@@ -56,7 +63,8 @@
     }
   },
   "files": [
-    "dist-*/**"
+    "dist-*/**",
+    "./cbor.js"
   ],
   "homepage": "https://github.com/awslabs/smithy-typescript/tree/main/packages/core",
   "repository": {
@@ -68,6 +76,7 @@
     "@types/node": "^16.18.96",
     "concurrently": "7.0.0",
     "downlevel-dts": "0.10.1",
+    "json-bigint": "^1.0.0",
     "rimraf": "3.0.2",
     "typedoc": "0.23.23"
   },

packages/core/src/submodules/cbor/cbor.spec.ts

@@ -0,0 +1,237 @@
+import * as fs from "fs";
+import JSONbig from "json-bigint";
+import * as path from "path";
+
+import { cbor } from "./cbor";
+
+// syntax is ESM but the test target is CJS.
+const here = __dirname;
+
+const errorTests = JSONbig({ useNativeBigInt: true, alwaysParseAsBig: false }).parse(
+  fs.readFileSync(path.join(here, "test-data", "decode-error-tests.json"))
+);
+const successTests = JSONbig({ useNativeBigInt: true, alwaysParseAsBig: false }).parse(
+  fs.readFileSync(path.join(here, "test-data", "success-tests.json"))
+);
+
+describe("cbor", () => {
+  const examples = [
+    {
+      name: "false",
+      data: false,
+      // special major 7 = 0b111 plus false(20) = 0b10100
+      cbor: new Uint8Array([0b111_10100]),
+    },
+    {
+      name: "true",
+      data: true,
+      // increment from false
+      cbor: new Uint8Array([0b111_10101]),
+    },
+    {
+      name: "null",
+      data: null,
+      // increment from true
+      cbor: new Uint8Array([0b111_10110]),
+    },
+    {
+      name: "an unsigned zero integer",
+      data: 0,
+      // unsigned int major (0) plus 00's.
+      cbor: new Uint8Array([0b000_00000]),
+    },
+    {
+      name: "negative 1",
+      data: -1,
+      // negative major (1) plus 00's, since -1 is the first negative number.
+      cbor: new Uint8Array([0b001_00000]),
+    },
+    {
+      name: "Number.MIN_SAFE_INTEGER",
+      data: -9007199254740991,
+      cbor: new Uint8Array([0b001_11011, 0, 31, 255, 255, 255, 255, 255, 254]),
+    },
+    {
+      name: "Number.MAX_SAFE_INTEGER",
+      data: 9007199254740991,
+      cbor: new Uint8Array([0b000_11011, 0, 31, 255, 255, 255, 255, 255, 255]),
+    },
+    {
+      name: "int64 min",
+      data: BigInt("-18446744073709551616"),
+      cbor: new Uint8Array([0b001_11011, 255, 255, 255, 255, 255, 255, 255, 255]),
+    },
+    {
+      name: "int64 max",
+      data: BigInt("18446744073709551615"),
+      cbor: new Uint8Array([0b000_11011, 255, 255, 255, 255, 255, 255, 255, 255]),
+    },
+    {
+      name: "negative float",
+      data: -3015135.135135135,
+      cbor: new Uint8Array([0b111_11011, +193, +71, +0, +239, +145, +76, +27, +173]),
+    },
+    {
+      name: "positive float",
+      data: 3015135.135135135,
+      cbor: new Uint8Array([0b111_11011, +65, +71, +0, +239, +145, +76, +27, +173]),
+    },
+    {
+      name: "an empty string",
+      data: "",
+      // string major plus 00's
+      cbor: new Uint8Array([0b011_00000]),
+    },
+    {
+      name: "a short string",
+      data: "hello, world",
+      cbor: new Uint8Array([108, 104, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100]),
+    },
+    {
+      name: "simple object",
+      data: {
+        message: "hello, world",
+      },
+      cbor: new Uint8Array([
+        161, 103, 109, 101, 115, 115, 97, 103, 101, 108, 104, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100,
+      ]),
+    },
+
+    {
+      name: "complex object",
+      data: {
+        number: 135019305913059,
+        message: "hello, world",
+        list: [0, false, { a: "b" }],
+        map: {
+          a: "a",
+          b: "b",
+          items: [0, -1, true, false, null, "", "test", ["nested item A", "nested item B"]],
+        },
+      },
+      cbor: new Uint8Array([
+        164, 102, 110, 117, 109, 98, 101, 114, 27, 0, 0, 122, 204, 161, 196, 74, 227, 103, 109, 101, 115, 115, 97, 103,
+        101, 108, 104, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100, 100, 108, 105, 115, 116, 131, 0, 244, 161,
+        97, 97, 97, 98, 99, 109, 97, 112, 163, 97, 97, 97, 97, 97, 98, 97, 98, 101, 105, 116, 101, 109, 115, 136, 0, 32,
+        245, 244, 246, 96, 100, 116, 101, 115, 116, 130, 109, 110, 101, 115, 116, 101, 100, 32, 105, 116, 101, 109, 32,
+        65, 109, 110, 101, 115, 116, 101, 100, 32, 105, 116, 101, 109, 32, 66,
+      ]),
+    },
+  ];
+
+  const toBytes = (hex: string) => {
+    const bytes = [];
+    hex.replace(/../g, (substr: string): string => {
+      bytes.push(parseInt(substr, 16));
+      return substr;
+    });
+    return new Uint8Array(bytes);
+  };
+
+  describe("locally curated scenarios", () => {
+    for (const { name, data, cbor: cbor_representation } of examples) {
+      it(`should encode for ${name}`, async () => {
+        const serialized = cbor.serialize(data);
+        expect(serialized).toEqual(cbor_representation);
+      });
+
+      it(`should decode for ${name}`, async () => {
+        const deserialized = cbor.deserialize(cbor_representation);
+        expect(deserialized).toEqual(data);
+      });
+    }
+  });
+
+  describe("externally curated scenarios", () => {
+    for (const { description, input, error } of errorTests) {
+      it(description, () => {
+        expect(error).toBe(true);
+        const bytes = toBytes(input);
+        expect(() => {
+          cbor.deserialize(bytes);
+        }).toThrow();
+      });
+    }
+
+    function binaryToFloat32(b: number) {
+      const dv = new DataView(new ArrayBuffer(4));
+      dv.setInt32(0, Number(b));
+      return dv.getFloat32(0);
+    }
+
+    function binaryToFloat64(b: number) {
+      const binaryArray = b.toString(2).split("").map(Number);
+      const pad = Array(64).fill(0);
+      const binary64 = new Uint8Array(pad.concat(binaryArray).slice(-64));
+
+      const sign = binary64[0];
+      const exponent = Number("0b" + Array.from(binary64.subarray(1, 12)).join(""));
+      const fraction = binary64.subarray(12);
+
+      const scalar = (-1) ** sign;
+      let sum = 1;
+      for (let i = 1; i <= 52; ++i) {
+        const position = i - 1;
+        const bit = fraction[position];
+        sum += 2 ** -i * bit;
+      }
+      const exponentScalar = Math.pow(2, exponent - 1023);
+      return scalar * sum * exponentScalar;
+    }
+
+    function translateTestData(data: any) {
+      const [type, value] = Object.entries(data)[0] as [string, any];
+      switch (type) {
+        case "null":
+          return null;
+        case "uint":
+        case "negint":
+        case "bool":
+        case "string":
+          return value;
+        case "float32":
+          return binaryToFloat32(value);
+        case "float64":
+          return binaryToFloat64(value);
+        case "bytestring":
+          return new Uint8Array(value.map(Number));
+        case "list":
+          return value.map(translateTestData);
+        case "map":
+          const output = {} as Record<string, any>;
+          for (const [k, v] of Object.entries(value)) {
+            output[k] = translateTestData(v);
+          }
+          return output;
+        case "tag":
+          const { id, value: tagValue } = value;
+          return {
+            tag: id,
+            value: translateTestData(tagValue),
+          };
+        default:
+          throw new Error(`Unrecognized test scenario <expect> type ${type}.`);
+      }
+    }
+
+    for (const { description, input, expect: _expect } of successTests) {
+      const bytes = toBytes(input);
+      const jsObject = translateTestData(_expect);
+
+      it(`serialization for ${description}`, () => {
+        const serialized = cbor.serialize(jsObject);
+        const redeserialized = cbor.deserialize(serialized);
+        /**
+         * We cannot assert that serialized == bytes,
+         * because there are multiple serializations
+         * that deserialize to the same object.
+         */
+        expect(redeserialized).toEqual(jsObject);
+      });
+      it(`deserialization for ${description}`, () => {
+        const deserialized = cbor.deserialize(bytes);
+        expect(deserialized).toEqual(jsObject);
+      });
+    }
+  });
+});