Pluggable serializers

lib/jetpack.js

@@ -19,6 +19,7 @@ const symlink = require("./symlink");
 const streams = require("./streams");
 const tmpDir = require("./tmp_dir");
 const write = require("./write");
+const serializers = require("./utils/serializers");
 
 // The Jetpack Context object.
 // It provides the public API, and resolves all paths regarding to
@@ -62,6 +63,9 @@ const jetpackContext = (cwdPath) => {
   const api = {
     cwd,
     path: getPath,
+    setSerializer: serializers.add,
+    listSerializers: serializers.list,
+    deleteSerializer: serializers.remove,
 
     append: (path, data, options) => {
       append.validateInput("append", path, data, options);

lib/read.js

@@ -2,18 +2,24 @@
 
 const fs = require("./utils/fs");
 const validate = require("./utils/validate");
+const { parseMaybe, JsonSerializer } = require("./utils/serializers");
 
-const supportedReturnAs = ["utf8", "buffer", "json", "jsonWithDates"];
+const supportedReturnAs = ["utf8", "buffer", "json", "jsonWithDates", "auto"];
 
 const validateInput = (methodName, path, returnAs) => {
   const methodSignature = `${methodName}(path, returnAs)`;
   validate.argument(methodSignature, "path", path, ["string"]);
   validate.argument(methodSignature, "returnAs", returnAs, [
     "string",
+    "function",
     "undefined",
   ]);
 
-  if (returnAs && supportedReturnAs.indexOf(returnAs) === -1) {
+  if (
+    returnAs &&
+    typeof returnAs === "string" &&
+    supportedReturnAs.indexOf(returnAs) === -1
+  ) {
     throw new Error(
       `Argument "returnAs" passed to ${methodSignature} must have one of values: ${supportedReturnAs.join(
         ", "
@@ -22,23 +28,8 @@ const validateInput = (methodName, path, returnAs) => {
   }
 };
 
-// Matches strings generated by Date.toJSON()
-// which is called to serialize date to JSON.
-const jsonDateParser = (key, value) => {
-  const reISO =
-    /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:\.\d*))(?:Z|(\+|-)([\d|:]*))?$/;
-  if (typeof value === "string") {
-    if (reISO.exec(value)) {
-      return new Date(value);
-    }
-  }
-  return value;
-};
-
-const makeNicerJsonParsingError = (path, err) => {
-  const nicerError = new Error(
-    `JSON parsing failed while reading ${path} [${err}]`
-  );
+const makeNicerParsingError = (path, err) => {
+  const nicerError = new Error(`Parsing failed while reading ${path} [${err}]`);
   nicerError.originalError = err;
   return nicerError;
 };
@@ -69,12 +60,16 @@ const readSync = (path, returnAs) => {
 
   try {
     if (retAs === "json") {
-      data = JSON.parse(data);
+      return JsonSerializer.parse(data);
     } else if (retAs === "jsonWithDates") {
-      data = JSON.parse(data, jsonDateParser);
+      return JsonSerializer.parse(data, { dates: true });
+    } else if (retAs === "auto") {
+      return parseMaybe(path, data);
+    } else if (typeof retAs === "object") {
+      return parseMaybe(path, data, retAs.parse);
     }
   } catch (err) {
-    throw makeNicerJsonParsingError(path, err);
+    throw makeNicerParsingError(path, err);
   }
 
   return data;
@@ -97,14 +92,16 @@ const readAsync = (path, returnAs) => {
         // Make final parsing of the data before returning.
         try {
           if (retAs === "json") {
-            resolve(JSON.parse(data));
+            resolve(JsonSerializer.parse(data, { dates: false }));
           } else if (retAs === "jsonWithDates") {
-            resolve(JSON.parse(data, jsonDateParser));
+            resolve(JsonSerializer.parse(data, { dates: true }));
+          } else if (retAs === "auto") {
+            resolve(parseMaybe(path, data));
           } else {
             resolve(data);
           }
         } catch (err) {
-          reject(makeNicerJsonParsingError(path, err));
+          reject(makeNicerParsingError(path, err));
         }
       })
       .catch((err) => {

lib/utils/serializers.js

@@ -0,0 +1,91 @@
+"use strict";
+
+const pathUtil = require("path");
+
+const JsonSerializer = {
+  validate: (data) => {
+    return typeof data === "object" && !Buffer.isBuffer(data) && data !== null;
+  },
+
+  parse: (data, options) => {
+    if (options?.dates) {
+      return JSON.parse(data, jsonDateParser);
+    } else {
+      return JSON.parse(data);
+    }
+  },
+
+  stringify: (data, options) => {
+    let indent = options?.jsonIndent;
+
+    if (typeof indent !== "number") {
+      indent = 2;
+    }
+
+    return JSON.stringify(data, null, indent);
+  },
+};
+
+// Matches strings generated by Date.toJSON()
+// which is called to serialize date to JSON.
+const jsonDateParser = (key, value) => {
+  const reISO =
+    /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}(?:\.\d*))(?:Z|(\+|-)([\d|:]*))?$/;
+  if (typeof value === "string") {
+    if (reISO.exec(value)) {
+      return new Date(value);
+    }
+  }
+  return value;
+};
+
+const serializers = new Map().set(".json", JsonSerializer);
+
+const add = (extension, serializer) => {
+  const ext =
+    (extension.startsWith(".") ? "" : ".") + extension.toLocaleLowerCase();
+  serializers.set(ext, serializer);
+};
+
+const remove = (extension) => {
+  const ext = pathUtil.extname(extension).toLocaleLowerCase();
+  serializers.delete(ext);
+};
+
+const list = () => {
+  return Object.fromEntries(serializers.entries());
+};
+
+const find = (path) => {
+  const ext = pathUtil.extname(path).toLocaleLowerCase();
+  return serializers.get(ext);
+};
+
+const stringifyMaybe = (path, data, options) => {
+  const serializer = options?.customSerializer ?? find(path) ?? JsonSerializer;
+  if (serializer) {
+    if (serializer.validate && !serializer.validate(data)) {
+      return data;
+    } else {
+      return serializer.stringify(data, options);
+    }
+  }
+  return data;
+};
+
+const parseMaybe = (path, data, customSerializer) => {
+  const serializer = customSerializer ?? find(path);
+  if (serializer) {
+    return serializer.parse(data);
+  }
+};
+
+module.exports = {
+  add,
+  list,
+  remove,
+  find,
+  stringifyMaybe,
+  parseMaybe,
+  JsonSerializer,
+};

lib/write.js

@@ -4,6 +4,7 @@ const pathUtil = require("path");
 const fs = require("./utils/fs");
 const validate = require("./utils/validate");
 const dir = require("./dir");
+const { stringifyMaybe } = require("./utils/serializers");
 
 const validateInput = (methodName, path, data, options) => {
   const methodSignature = `${methodName}(path, data, [options])`;
@@ -18,25 +19,13 @@ const validateInput = (methodName, path, data, options) => {
     mode: ["string", "number"],
     atomic: ["boolean"],
     jsonIndent: ["number"],
+    serialize: ["object", "boolean"],
   });
 };
 
 // Temporary file extensions used for atomic file overwriting.
 const newExt = ".__new__";
 
-const serializeToJsonMaybe = (data, jsonIndent) => {
-  let indent = jsonIndent;
-  if (typeof indent !== "number") {
-    indent = 2;
-  }
-
-  if (typeof data === "object" && !Buffer.isBuffer(data) && data !== null) {
-    return JSON.stringify(data, null, indent);
-  }
-
-  return data;
-};
-
 // ---------------------------------------------------------
 // SYNC
 // ---------------------------------------------------------
@@ -66,7 +55,7 @@ const writeAtomicSync = (path, data, options) => {
 
 const writeSync = (path, data, options) => {
   const opts = options || {};
-  const processedData = serializeToJsonMaybe(data, opts.jsonIndent);
+  const processedData = stringifyMaybe(path, data, opts);
 
   let writeStrategy = writeFileSync;
   if (opts.atomic) {
@@ -118,7 +107,7 @@ const writeAtomicAsync = (path, data, options) => {
 
 const writeAsync = (path, data, options) => {
   const opts = options || {};
-  const processedData = serializeToJsonMaybe(data, opts.jsonIndent);
+  const processedData = stringifyMaybe(path, data, opts);
 
   let writeStrategy = writeFileAsync;
   if (opts.atomic) {

spec/read.spec.ts

@@ -97,7 +97,7 @@ describe("read", () => {
     };
 
     const expectations = (err: any) => {
-      expect(err.message).to.have.string("JSON parsing failed while reading");
+      expect(err.message).to.have.string("Parsing failed while reading");
     };
 
     it("sync", () => {
@@ -257,12 +257,12 @@ describe("read", () => {
           expect(() => {
             test.method("abc", true);
           }).to.throw(
-            `Argument "returnAs" passed to ${test.methodName}(path, returnAs) must be a string or an undefined. Received boolean`
+            `Argument "returnAs" passed to ${test.methodName}(path, returnAs) must be a string or a function or an undefined. Received boolean`
           );
           expect(() => {
             test.method("abc", "foo");
           }).to.throw(
-            `Argument "returnAs" passed to ${test.methodName}(path, returnAs) must have one of values: utf8, buffer, json, jsonWithDates`
+            `Argument "returnAs" passed to ${test.methodName}(path, returnAs) must have one of values: utf8, buffer, json, jsonWithDates, auto`
           );
         });
       });

spec/utils/serializers.spec.ts

@@ -0,0 +1,55 @@
+import { expect } from "chai";
+import * as fse from "fs-extra";
+import * as jetpack from "../..";
+import { Serializer } from "../../types";
+import helper from "../helper";
+
+describe("serializer management", () => {
+  const dummySerializer: Serializer = {
+    stringify: (input: any) => "[serializer-test]",
+    parse: (input: string) => input.length,
+  };
+
+  it("set", () => {
+    jetpack.setSerializer("serializer-test", dummySerializer);
+  });
+
+  it("list", () => {
+    const extensions = jetpack.listSerializers();
+  });
+
+  it("delete", () => {
+    jetpack.deleteSerializer("serializer-test");
+  });
+});
+
+describe("ndjson stringifies and parses", () => {
+  beforeEach(helper.setCleanTestCwd);
+  afterEach(helper.switchBackToCorrectCwd);
+
+  const obj = [{ utf8: "ąćłźż" }, { utf8: "☃" }];
+
+  const NdJson: Serializer<any[], any[]> = {
+    validate: (input: unknown) => Array.isArray(input),
+    parse: function (data: string) {
+      const lines = data.split("\n");
+      return lines.map((line) => JSON.parse(line));
+    },
+    stringify: function (data: any[]): string {
+      return data.map((item) => JSON.stringify(item, undefined, 0)).join("\n");
+    },
+  };
+
+  jetpack.setSerializer(".ndjson", NdJson);
+
+  it("sync", (done) => {
+    jetpack.write("file.ndjson", obj);
+    const raw = jetpack.read("file.ndjson");
+    const output = jetpack.read("file.ndjson", "auto");
+
+    expect(raw).to.equal('{"utf8":"ąćłźż"}\n{"utf8":"☃"}');
+    expect(output).to.deep.equal(obj);
+
+    done();
+  });
+});