Use @node-rs/crc32 v1.6.0

impls/sse4_crc32c.js

@@ -1,5 +1,5 @@
-const bindings = require('@node-rs/crc32');
+const bindings = require('sse4_crc32');
 
 module.exports = {
-  calculate: bindings.crc32c,
+  calculate: bindings.calculate,
 };

package.json

@@ -10,12 +10,13 @@
   },
   "main": "./loader",
   "optionalDependencies": {
-    "@node-rs/crc32": "^0.1.7"
+    "sse4_crc32": "^7.0.0"
   },
   "scripts": {
-    "test": "nyc --reporter=html --reporter=text mocha",
+    "test:unit": "nyc --reporter=html --reporter=text mocha test/unit/*.test.js",
+    "test:mem": "node test/mem/mem_test.test.js",
     "coverage": "nyc report --reporter=text-lcov | coveralls",
-    "benchmark": "node benchmark"
+    "benchmark": "benchmark test/perf/benchmark.test.js"
   },
   "keywords": [
     "crc",
@@ -26,13 +27,13 @@
     "sse4.2"
   ],
   "devDependencies": {
-    "benchtable": "^0.1.0",
+    "@dapplion/benchmark": "^0.2.4",
+    "@node-rs/crc32": "^1.5.1",
     "buffer-crc32": "^0.2.13",
+    "chai": "^4.3.7",
     "coveralls": "^3.0.6",
-    "mocha": "^6.2.1",
+    "mocha": "^10.0.0",
     "nyc": "^14.1.1",
-    "random-string": "^0.2.0",
-    "should": "^13.2.3",
-    "sse4_crc32": "^6.0.1"
+    "random-string": "^0.2.0"
   }
 }

test/mem/mem_test.test.js

@@ -0,0 +1,46 @@
+const Sse4Crc32 = require("sse4_crc32");
+// Enable this to show memory test issue
+// const nodeRs = require("@node-rs/crc32");
+// const nodeRsCrc32c = nodeRs.crc32c;
+
+const bytes = Buffer.alloc(1000);
+
+
+/**
+ * No memory issue with sse4_crc32 but @node-rs/crc32
+ * Refer to https://github.com/napi-rs/node-rs/issues/655
+ */
+async function memTest() {
+  const count = 1_000_000_000;
+  for (let i = 0; i < count; i++) {
+    // Enable this to show memory test issue with @node-rs/crc32
+    // nodeRsCrc32c(bytes);
+    Sse4Crc32.calculate(bytes);
+    if (i % 100_000 === 0) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      const {heapTotal, rss} = process.memoryUsage();
+      console.log(
+        "Memory usage",
+        Math.floor((i * 100) / count) + "%",
+        "heapTotal",
+        toMem(heapTotal),
+        "rss",
+        toMem(rss)
+      );
+    }
+  }
+}
+
+function toMem(n) {
+  const bytes = Math.abs(n);
+  const sign = n > 0 ? "+" : "-";
+  if (bytes < 1e6) return sign + Math.floor(bytes / 10) / 100 + " KB";
+
+  if (bytes < 1e9) return sign + Math.floor(bytes / 1e4) / 100 + " MB";
+
+  return sign + Math.floor(bytes / 1e7) / 100 + " GB";
+}
+
+console.log("Start node-rs crc32c mem test");
+
+memTest().then(() => console.log("Done"));

test/perf/benchmark.test.js

@@ -0,0 +1,43 @@
+const {itBench, setBenchOpts} = require("@dapplion/benchmark");
+const crypto = require("node:crypto");
+const nodeRsCrc32c = require('@node-rs/crc32').crc32c;
+const sse42Crc = require('sse4_crc32').sse42_crc;
+const tableCrc = require('sse4_crc32').table_crc;
+const jsCrc32c = require('../../impls/js_crc32c').calculate;
+const jsCrc32 = require('buffer-crc32').unsigned;
+
+const tests = [{
+  name: '@node-rs/crc32c',
+  calculate: nodeRsCrc32c,
+},
+{
+  name: 'sse4_crc32c_hw',
+  calculate: sse42Crc,
+}, {
+  name: 'sse4_crc32c_sw',
+  calculate: tableCrc,
+}, {
+  name: 'js_crc32c',
+  calculate: jsCrc32c,
+}, {
+  name: 'js_crc32',
+  calculate: jsCrc32,
+}];
+
+describe("crc32 implementations", function () {
+  setBenchOpts({
+    minMs: 10 * 1000,
+  })
+
+  const counts = [500, 1000, 10_000, 100_000, 1_000_000];
+  for (const c of counts) {
+    const bytes = crypto.randomBytes(c);
+
+    for (const {name, calculate} of tests) {
+      itBench({
+        id: `${name} ${c} bytes`,
+        fn: () => calculate(bytes),
+      })
+    }
+  }
+});

test/crc32c.js → test/unit/crc32c.test.js

@@ -1,15 +1,16 @@
+const {expect} = require('chai');
 const sets = require('./sets.json');
 
 sets.buffer.cases.forEach(function(cs) {
   cs.input = Buffer.from(cs.input);
 });
 
 describe('crc32c.js', function() {
-  describe('calculate()', testCalculate(require('../impls/js_crc32c')));
+  describe('calculate()', testCalculate(require('../../impls/js_crc32c')));
 });
 
 describe('fast-crc32c', function() {
-  describe('calculate()', testCalculate(require('../')));
+  describe('calculate()', testCalculate(require('../../loader')));
 });
 
 function testCalculate(crc32) {
@@ -18,13 +19,13 @@ function testCalculate(crc32) {
       const set = sets[type];
       set.cases.forEach(function(cs) {
         it(`should digest "${cs.input}" correctly`, function() {
-          crc32.calculate(cs.input).should.eql(cs.want);
+          expect(crc32.calculate(cs.input)).to.be.equal(cs.want);
         });
       });
       it(`should digest all ${type} correctly`, function() {
-        set.cases.reduce(function(prev, cs) {
+        expect(set.cases.reduce(function(prev, cs) {
           return crc32.calculate(cs.input, prev);
-        }, 0).should.eql(set.want);
+        }, 0)).to.be.equal(set.want);
       })
     }
   };