chore(scripts): add compression routines for endpoints rulesets (#4300)

* chore(scripts): add compression routines for endpoints rulesets

Co-authored-by: Trivikram Kamat <[email protected]>

scripts/endpoints-ruleset/compress.js

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const path = require("path");
+
+const RemoveWhitespace = require("./compression-algorithms/RemoveWhitespace");
+const PatternDetection = require("./compression-algorithms/PatternDetection");
+
+/**
+ * Run compression on ruleset objects for SDK clients.
+ */
+const main = () => {
+  const root = path.join(__dirname, "..", "..");
+  const clientsFolder = path.join(root, "clients");
+  const modelsFolder = path.join(root, "codegen", "sdk-codegen", "aws-models");
+  const modelsList = fs.readdirSync(modelsFolder);
+  const tempFolder = path.join(__dirname, "temp");
+
+  // clean temp folder.
+  for (const tempFile of fs.readdirSync(tempFolder)) {
+    if (tempFile !== ".gitignore") {
+      fs.rmSync(path.join(tempFolder, tempFile));
+    }
+  }
+
+  /**
+   * The first algorithm which passes self-verification will be used.
+   */
+  const compressionAlgorithms = [(data) => new PatternDetection(data), (data) => new RemoveWhitespace(data)];
+
+  for (const serviceName of modelsList) {
+    const client = serviceName.replace(".json", "");
+    const rulesetFolder = path.join(clientsFolder, "client-" + client, "src", "endpoint");
+    const rulesetTs = path.join(rulesetFolder, "ruleset.ts");
+    const serviceJson = path.join(modelsFolder, serviceName);
+
+    const service = require(serviceJson);
+    const rulesetObject = Object.entries(service.shapes).find(([k, v]) => v.type === "service")[1].traits[
+      "smithy.rules#endpointRuleSet"
+    ];
+
+    const data = rulesetObject;
+    let selectedAlgorithm = null;
+    for (const factory of compressionAlgorithms) {
+      const algo = factory(data);
+      try {
+        algo.verifyImplementation();
+        selectedAlgorithm = factory(data);
+        break;
+      } catch (e) {
+        const sample = factory(data).toCodeString("module.exports = $;");
+        fs.writeFileSync(path.join(path.join(__dirname, "temp", client + "-failed.ts")), sample, "utf-8");
+        console.warn(`WARN: Algorithm ${algo.constructor.name} failed for ${client}.`);
+      }
+    }
+
+    if (!selectedAlgorithm) {
+      throw new Error(`No viable algorithm for ${client}`);
+    }
+
+    const modifiedSource = `// @ts-nocheck
+// generated code, do not edit
+import { RuleSetObject } from "@aws-sdk/util-endpoints";
+
+/* This file is compressed. Log this object
+   or see "smithy.rules#endpointRuleSet"
+   in codegen/sdk-codegen/aws-models/${client}.json */
+
+${selectedAlgorithm.toCodeString("export const ruleSet: RuleSetObject = $;")}
+`;
+
+    fs.writeFileSync(rulesetTs, modifiedSource, "utf-8");
+
+    if (client === "s3") {
+      fs.writeFileSync(
+        path.join(tempFolder, "s3.js"),
+        new PatternDetection(data).toCodeString("module.exports = $;"),
+        "utf-8"
+      );
+      fs.writeFileSync(path.join(tempFolder, "s3.json"), JSON.stringify(data, null, 2), "utf-8");
+    }
+
+    console.log(client, `OK - ${selectedAlgorithm.constructor.name}`);
+  }
+
+  return 0;
+};
+
+main();

scripts/endpoints-ruleset/compression-algorithms/CompressionAlgorithm.js

@@ -0,0 +1,102 @@
+const fs = require("fs");
+const path = require("path");
+const assert = require("assert");
+const uuid = require("uuid");
+
+/**
+ * Outputs a JSON object as JavaScript code with some form of compression.
+ */
+module.exports = class CompressionAlgorithm {
+  /**
+   * @param data - to be compressed.
+   * @param _export - e.g. "module.exports = $"
+   */
+  constructor(data, _export) {
+    /**
+     * The export string, using $ as a placeholder.
+     * cjs: module.exports = $;
+     * esm: export default $;
+     * esm: export const data = $;
+     */
+    this._export = _export;
+    /**
+     * Reference to original unmodified data object.
+     */
+    this.data = data;
+  }
+
+  /**
+   * Verify the implementation is correct.
+   * @throws if implementation serialization does not match original data object.
+   */
+  verifyImplementation() {
+    const tempFile = path.join(__dirname, "..", "temp", `__${uuid.v4()}tmp.js`); // defeat require-cache.
+    let original = 0;
+    let compressed = 1;
+    try {
+      fs.writeFileSync(tempFile, this.toCodeString("module.exports = $"), "utf-8");
+      original = this.toJsonString(this.data);
+      compressed = this.toJsonString(require(tempFile));
+    } catch (e) {}
+    fs.rmSync(tempFile);
+    if (original !== compressed) {
+      fs.writeFileSync(path.join(__dirname, "..", "temp", "__original.json"), original);
+      fs.writeFileSync(path.join(__dirname, "..", "temp", "__compressed.json"), compressed);
+    }
+    assert(original === compressed, `Compression implementation is not correct for ${this.constructor.name}.`);
+  }
+
+  /**
+   * @returns deterministic JSON representation.
+   */
+  toJsonString(data = this.data, indent = "") {
+    let buffer = "";
+    switch (typeof data) {
+      case "undefined":
+        break;
+      case "object":
+        if (Array.isArray(data)) {
+          buffer += `\n` + indent + `[\n`;
+          for (let i = 0; i < data.length; ++i) {
+            const element = data[i];
+            buffer += indent + this.toJsonString(element, indent + "  ");
+            if (i !== data.length - 1) {
+              buffer += indent + `,\n`;
+            }
+          }
+          buffer += `\n` + indent + `]\n`;
+          break;
+        }
+        const keys = Object.keys(data).sort();
+        buffer += "\n" + indent + `{\n`;
+        for (let i = 0; i < keys.length; ++i) {
+          const key = keys[i];
+          buffer += indent + `"${key}": `;
+          buffer += this.toJsonString(data[key], indent + "  ");
+          if (i !== keys.length - 1) {
+            buffer += indent + `,\n`;
+          }
+        }
+        buffer += `\n` + indent + `}\n`;
+        break;
+      case "boolean":
+        buffer += indent + data;
+        break;
+      case "number":
+        buffer += indent + data;
+        break;
+      case "string":
+        buffer += indent + `"${data.replaceAll(/"/g, `\\"`)}"`;
+        break;
+      default:
+    }
+    return buffer;
+  }
+
+  /**
+   * @returns {string} source code string of compressed data.
+   */
+  toCodeString(_export = this._export) {
+    throw new Error("not implemented in abstract base.");
+  }
+};