Skip to content

Commit

Permalink
feat: implement compress and uncompress
Browse files Browse the repository at this point in the history
  • Loading branch information
Brooooooklyn committed Jun 10, 2021
1 parent adc3418 commit df2ccd2
Show file tree
Hide file tree
Showing 36 changed files with 881 additions and 181 deletions.
12 changes: 6 additions & 6 deletions .github/workflows/CI.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ jobs:
shell: bash

- name: Test bindings
run: docker run --rm -v $(pwd):/canvas -w /canvas node:${{ matrix.node }}-slim yarn test
run: docker run --rm -v $(pwd):/snappy -w /snappy node:${{ matrix.node }}-slim yarn test

test-linux-x64-musl-binding:
name: Test bindings on x86_64-unknown-linux-musl - node@${{ matrix.node }}
Expand Down Expand Up @@ -322,7 +322,7 @@ jobs:
shell: bash

- name: Test bindings
run: docker run --rm -v $(pwd):/canvas -w /canvas node:${{ matrix.node }}-alpine yarn test
run: docker run --rm -v $(pwd):/snappy -w /snappy node:${{ matrix.node }}-alpine yarn test

test-linux-aarch64-gnu-binding:
name: Test bindings on aarch64-unknown-linux-gnu - node@${{ matrix.node }}
Expand Down Expand Up @@ -357,8 +357,8 @@ jobs:
distro: ubuntu20.04

dockerRunArgs: |
--volume "${PWD}:/canvas"
-w /canvas
--volume "${PWD}:/snappy"
-w /snappy
# Not required, but speeds up builds by storing container images in
# a GitHub package registry.
Expand Down Expand Up @@ -408,8 +408,8 @@ jobs:
distro: ubuntu20.04

dockerRunArgs: |
--volume "${PWD}:/canvas"
-w /canvas
--volume "${PWD}:/snappy"
-w /snappy
# Not required, but speeds up builds by storing container images in
# a GitHub package registry.
Expand Down
8 changes: 0 additions & 8 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,9 @@ version = "0.1.0"
crate-type = ["cdylib"]

[dependencies]
encoding_rs = "0.8"
napi = {version = "1", features = ["napi5"]}
napi-derive = "1"
snap = "1"
tokio = {version = "1", features = ["full"]}

[target.'cfg(all(unix, not(target_env = "musl"), not(target_arch = "aarch64"), not(target_arch = "arm")))'.dependencies]
jemallocator = {version = "0.3", features = ["disable_initial_exec_tls"]}

[target.'cfg(all(windows, target_arch = "x86_64"))'.dependencies]
mimalloc = {version = "0.1"}

[build-dependencies]
napi-build = "1"
Expand Down
54 changes: 51 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# `@napi-rs/snappy`

![https://github.com/Brooooooklyn/snappy/actions](https://github.com/Brooooooklyn/snappy/workflows/CI/badge.svg)
![](https://img.shields.io/npm/dm/@napi-rs/snappy.svg?sanitize=true)

> 🚀 Help me to become a full-time open-source developer by [sponsoring me on Github](https://github.com/sponsors/Brooooooklyn)
Fastest Snappy compression library in Node.js, powered by [napi-rs](https://napi.rs) and [rust-snappy](https://github.com/BurntSushi/rust-snappy).

## Install this test package
## Install this package

```
yarn add @napi-rs/snappy
Expand All @@ -31,8 +32,55 @@ yarn add @napi-rs/snappy

## API

> WIP
```ts
export function compressSync(input: Buffer | string | ArrayBuffer | Uint8Array): Buffer
export function compress(input: Buffer | string | ArrayBuffer | Uint8Array): Promise<Buffer>
export function uncompressSync(compressed: Buffer): Buffer
export function uncompress(compressed: Buffer): Promise<Buffer>
```

## Performance

> WIP
### Hardware

```
Model Name: MacBook Pro
Model Identifier: MacBookPro15,1
Processor Name: Intel Core i7
Processor Speed: 2.6 GHz
Number of Processors: 1
Total Number of Cores: 6
L2 Cache (per Core): 256 KB
L3 Cache: 12 MB
Hyper-Threading Technology: Enabled
Memory: 16 GB
```
### Result
```
Running "Compress data" suite...
Progress: 100%

@napi-rs/snappy:
279 ops/s, ±7.86% | fastest

snappy node:
235 ops/s, ±1.85% | slowest, 15.77% slower

Finished 2 cases!
Fastest: @napi-rs/snappy
Slowest: snappy node
Running "Uncompress data" suite...
Progress: 100%

@napi-rs/snappy:
379 ops/s, ±2.09% | fastest

snappy node:
347 ops/s, ±1.93% | slowest, 8.44% slower

Finished 2 cases!
Fastest: @napi-rs/snappy
Slowest: snappy node
```
24 changes: 19 additions & 5 deletions __test__/index.spec.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,23 @@
import { Duplex } from 'stream'

import test from 'ava'

import { SnappyStream } from '../index'
import { compressSync, compress, uncompressSync, uncompress } from '../index'

test('should be able to compress Buffer', (t) => {
const fixture = 'hello world'
t.deepEqual(compressSync(fixture), Buffer.from([11, 40, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100]))
})

test('compress should be async version of compressSync', async (t) => {
const fixture = 'hello world 😂 🎧 🚀'
t.deepEqual(await compress(fixture), compressSync(fixture))
})

test('should be able to decompress sync', (t) => {
const fixture = 'hello world 😂 🎧 🚀'
t.deepEqual(uncompressSync(compressSync(fixture)), Buffer.from(fixture))
})

test('SnappyStream should be instance of Duplex', (t) => {
t.true(new SnappyStream() instanceof Duplex)
test('should be able to decompress', async (t) => {
const fixture = 'hello world 😂 🎧 🚀'
t.deepEqual(await uncompress(await compress(fixture)), Buffer.from(fixture))
})
65 changes: 56 additions & 9 deletions benchmark/bench.ts
Original file line number Diff line number Diff line change
@@ -1,21 +1,68 @@
import { readFileSync } from 'fs'
import { cpus } from 'os'
import { join } from 'path'

import b from 'benny'
import { compress as compressCpp, uncompress as uncompressCpp } from 'snappy'

import { sync } from '../index'
import { compress, uncompress, compressSync } from '../index'

function add(a: number) {
return a + 100
}
const FIXTURE = readFileSync(join(__dirname, '..', 'yarn.lock'))
const COMPRESSED_FIXTURE = Buffer.from(compressSync(FIXTURE))

const THREADS = cpus().length

async function run() {
await b.suite(
'Add 100',
'Compress data',

b.add('@napi-rs/snappy', async () => {
await Promise.all(Array.from({ length: THREADS }).map(() => compress(FIXTURE)))
}),

b.add('snappy node', async () => {
await Promise.all(
Array.from({ length: THREADS }).map(
() =>
new Promise((resolve, reject) => {
compressCpp(FIXTURE, (err, buffer) => {
if (err) {
reject(err)
} else {
resolve(buffer)
}
})
}),
),
)
}),

b.cycle(),
b.complete(),
)

await b.suite(
'Uncompress data',

b.add('Native a + 100', () => {
sync(10)
b.add('@napi-rs/snappy', async () => {
await Promise.all(Array.from({ length: THREADS }).map(() => uncompress(COMPRESSED_FIXTURE)))
}),

b.add('JavaScript a + 100', () => {
add(10)
b.add('snappy node', async () => {
await Promise.all(
Array.from({ length: THREADS }).map(
() =>
new Promise((resolve, reject) => {
uncompressCpp(COMPRESSED_FIXTURE, { asBuffer: true }, (err, buffer) => {
if (err) {
reject(err)
} else {
resolve(buffer)
}
})
}),
),
)
}),

b.cycle(),
Expand Down
4 changes: 4 additions & 0 deletions index.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
export function compressSync(input: Buffer | string | ArrayBuffer | Uint8Array): Buffer
export function compress(input: Buffer | string | ArrayBuffer | Uint8Array): Promise<Buffer>
export function uncompressSync(compressed: Buffer): Buffer
export function uncompress(compressed: Buffer): Promise<Buffer>
19 changes: 19 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
const { loadBinding } = require('@node-rs/helper')

const {
compressSync: _compressSync,
compress: _compress,
uncompress,
uncompressSync,
} = loadBinding(__dirname, 'snappy', '@napi-rs/snappy')

module.exports = {
compressSync: function compressSync(input) {
return _compressSync(Buffer.from(input))
},
compress: function compress(input) {
return _compress(Buffer.from(input))
},
uncompress,
uncompressSync,
}
15 changes: 0 additions & 15 deletions index.ts

This file was deleted.

4 changes: 2 additions & 2 deletions npm/android-arm64/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# `@napi-rs/package-template-android-arm64`
# `@napi-rs/snappy-android-arm64`

This is the **aarch64-linux-android** binary for `@napi-rs/package-template`
This is the **aarch64-linux-android** binary for `@napi-rs/snappy`
24 changes: 18 additions & 6 deletions npm/android-arm64/package.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
{
"name": "@napi-rs/package-template-android-arm64",
"name": "@napi-rs/snappy-android-arm64",
"version": "1.0.0",
"os": ["android"],
"cpu": ["arm64"],
"main": "package-template.android-arm64.node",
"files": ["package-template.android-arm64.node"],
"description": "Template project for writing node package with napi-rs",
"keywords": ["napi-rs", "NAPI", "N-API", "Rust", "node-addon", "node-addon-api"],
"main": "snappy.android-arm64.node",
"files": ["snappy.android-arm64.node"],
"description": "Fastest Snappy compression library in Node.js",
"keywords": [
"snappy",
"snap",
"compression",
"compress",
"napi-rs",
"NAPI",
"N-API",
"Rust",
"Node-API",
"node-addon",
"node-addon-api"
],
"license": "MIT",
"engines": {
"node": ">= 10"
Expand All @@ -15,5 +27,5 @@
"registry": "https://registry.npmjs.org/",
"access": "public"
},
"repository": "git@github.com:napi-rs/package-template.git"
"repository": "https://github.com/Brooooooklyn/snappy"
}
4 changes: 2 additions & 2 deletions npm/darwin-arm64/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# `@napi-rs/package-template-darwin-arm64`
# `@napi-rs/snappy-darwin-arm64`

This is the **aarch64-apple-darwin** binary for `@napi-rs/package-template`
This is the **aarch64-apple-darwin** binary for `@napi-rs/snappy`
24 changes: 18 additions & 6 deletions npm/darwin-arm64/package.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
{
"name": "@napi-rs/package-template-darwin-arm64",
"name": "@napi-rs/snappy-darwin-arm64",
"version": "1.0.0",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "package-template.darwin-arm64.node",
"files": ["package-template.darwin-arm64.node"],
"description": "Template project for writing node package with napi-rs",
"keywords": ["napi-rs", "NAPI", "N-API", "Rust", "node-addon", "node-addon-api"],
"main": "snappy.darwin-arm64.node",
"files": ["snappy.darwin-arm64.node"],
"description": "Fastest Snappy compression library in Node.js",
"keywords": [
"snappy",
"snap",
"compression",
"compress",
"napi-rs",
"NAPI",
"N-API",
"Rust",
"Node-API",
"node-addon",
"node-addon-api"
],
"license": "MIT",
"engines": {
"node": ">= 10"
Expand All @@ -15,5 +27,5 @@
"registry": "https://registry.npmjs.org/",
"access": "public"
},
"repository": "git@github.com:napi-rs/package-template.git"
"repository": "https://github.com/Brooooooklyn/snappy"
}
4 changes: 2 additions & 2 deletions npm/darwin-x64/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# `@napi-rs/package-template-darwin-x64`
# `@napi-rs/snappy-darwin-x64`

This is the **x86_64-apple-darwin** binary for `@napi-rs/package-template`
This is the **x86_64-apple-darwin** binary for `@napi-rs/snappy`
Loading

0 comments on commit df2ccd2

Please sign in to comment.