pack sqljs wasm (#365)

* pack sqljs wasm

* update script

* lint

* bump

.eslintignore

@@ -4,4 +4,5 @@ executor/
 vendor/
 lib/
 dist/
+packages/core/scripts/
 chopsticks.js

executor/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@acala-network/chopsticks-executor",
 	"description": "Chopsticks executor",
-	"version": "0.8.0-2",
+	"version": "0.8.0-3",
 	"license": "Apache-2.0",
 	"repository": {
 		"type": "git",

executor/scripts/pack-wasm.js

@@ -19,8 +19,9 @@ const BYTES = '${base64}';
 import { base64Decode, unzlibSync } from '@polkadot/wasm-util';
 const WASM_BYTES = unzlibSync(base64Decode(BYTES, new Uint8Array(LEN_IN)), new Uint8Array(LEN_OUT));
 
-import { initSync } from "./chopsticks_executor.js";
-initSync(WASM_BYTES);
+import wasmInit from "./chopsticks_executor.js";
+const blob = new Blob([WASM_BYTES], { type: "application/wasm" });
+wasmInit(URL.createObjectURL(blob));
 
 export * from "./chopsticks_executor.js";
 `);

package.json

@@ -34,22 +34,22 @@
 		"*.{js,ts,css,md}": "prettier --write"
 	},
 	"devDependencies": {
-		"@swc/core": "^1.3.68",
-		"@types/node": "^20.4.5",
+		"@swc/core": "^1.3.76",
+		"@types/node": "^20.4.9",
 		"@types/prettier": "^3.0.0",
-		"@typescript-eslint/eslint-plugin": "^6.0.0",
-		"@typescript-eslint/parser": "^6.0.0",
-		"eslint": "^8.44.0",
-		"eslint-config-prettier": "^8.8.0",
-		"eslint-import-resolver-typescript": "^3.5.5",
+		"@typescript-eslint/eslint-plugin": "^6.3.0",
+		"@typescript-eslint/parser": "^6.3.0",
+		"eslint": "^8.46.0",
+		"eslint-config-prettier": "^9.0.0",
+		"eslint-import-resolver-typescript": "^3.6.0",
 		"eslint-plugin-import": "^2.28.0",
 		"eslint-plugin-sort-imports-es6-autofix": "^0.6.0",
 		"husky": "^8.0.3",
-		"prettier": "^3.0.0",
+		"prettier": "^3.0.1",
 		"typescript": "^5.1.6",
 		"unplugin-swc": "^1.3.2",
 		"vite-tsconfig-paths": "^4.2.0",
-		"vitest": "^0.33.0",
+		"vitest": "^0.34.1",
 		"wasm-pack": "^0.12.1"
 	}
 }

packages/chopsticks/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@acala-network/chopsticks",
-	"version": "0.8.0-2",
+	"version": "0.8.0-3",
 	"author": "Bryan Chen <[email protected]>",
 	"license": "Apache-2.0",
 	"bin": "./chopsticks.js",
@@ -32,8 +32,8 @@
 	"devDependencies": {
 		"@types/global-agent": "^2.1.1",
 		"@types/js-yaml": "^4.0.5",
-		"@types/lodash": "^4.14.195",
-		"@types/node": "^20.4.5",
+		"@types/lodash": "^4.14.197",
+		"@types/node": "^20.4.9",
 		"@types/ws": "^8.5.5",
 		"@types/yargs": "^17.0.24",
 		"ts-node": "^10.9.1",

packages/core/package.json

@@ -1,11 +1,12 @@
 {
 	"name": "@acala-network/chopsticks-core",
-	"version": "0.8.0-2",
+	"version": "0.8.0-3",
 	"author": "Bryan Chen <[email protected]>",
 	"license": "Apache-2.0",
 	"scripts": {
 		"clean": "rm -rf lib tsconfig.tsbuildinfo",
-		"build": "tsc -p ./tsconfig.json"
+		"pack-wasm": "scripts/pack-wasm.js",
+		"build": "yarn pack-wasm; tsc -p ./tsconfig.json"
 	},
 	"dependencies": {
 		"@acala-network/chopsticks-executor": "workspace:*",
@@ -14,16 +15,17 @@
 		"eventemitter3": "^5.0.1",
 		"localforage": "^1.10.0",
 		"lodash": "^4.17.21",
-		"pino": "^8.14.1",
-		"pino-pretty": "^10.0.1",
+		"pino": "^8.15.0",
+		"pino-pretty": "^10.2.0",
 		"sql.js": "^1.8.0",
 		"sqlite3": "^5.1.6",
 		"typeorm": "^0.3.17",
 		"zod": "^3.21.4"
 	},
 	"devDependencies": {
-		"@types/lodash": "^4.14.195",
-		"@types/sql.js": "^1",
+		"@types/lodash": "^4.14.197",
+		"@types/sql.js": "^1.4.4",
+		"fflate": "^0.8.0",
 		"typescript": "^5.1.6"
 	},
 	"files": [

packages/core/scripts/pack-wasm.js

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+
+const fs = require('node:fs')
+const path = require('node:path')
+const { zlibSync } = require('fflate/node')
+const { formatNumber } = require('@polkadot/util')
+
+const data = fs.readFileSync(path.resolve(__dirname, `../../../node_modules/sql.js/dist/sql-wasm.wasm`))
+const compressed = Buffer.from(zlibSync(data, { level: 9 }))
+const base64 = compressed.toString('base64')
+
+console.log(
+  `*** Compressed WASM: in=${formatNumber(data.length)}, out=${formatNumber(compressed.length)}, opt=${(
+    (100 * compressed.length) /
+    data.length
+  ).toFixed(2)}%, base64=${formatNumber(base64.length)}`,
+)
+
+fs.writeFileSync(
+  path.resolve(__dirname, `../src/db/sql-wasm.ts`),
+  `// Auto-generated file, do not edit by hand
+const LEN_IN = ${compressed.length}
+const LEN_OUT = ${data.length}
+const BYTES =
+  '${base64}'
+
+import { base64Decode, unzlibSync } from '@polkadot/wasm-util'
+export const SQL_WASM_BYTES = unzlibSync(base64Decode(BYTES, new Uint8Array(LEN_IN)), new Uint8Array(LEN_OUT))
+`,
+)

packages/core/src/db/browser.ts

@@ -3,23 +3,23 @@ import { createInstance } from 'localforage'
 import initSqlJs from 'sql.js'
 
 import * as entities from './entities'
+import { SQL_WASM_BYTES } from './sql-wasm'
 
-export const openDb = async (sqlWasmUrl: URL): Promise<DataSource> => {
+export const openDb = async (location: string): Promise<DataSource> => {
   if (!globalThis.localforage) {
     globalThis.localforage = createInstance({ name: 'chopsticks' })
   }
-  const wasmBinary = await fetch(sqlWasmUrl).then((response) => response.arrayBuffer())
   const source = new DataSource({
     type: 'sqljs',
-    location: 'cache',
+    location,
     entities: Object.values(entities),
     synchronize: true,
     autoSave: true,
     logging: false,
     useLocalForage: true,
     driver: initSqlJs,
     sqlJsConfig: {
-      wasmBinary,
+      wasmBinary: SQL_WASM_BYTES,
     },
   })
 

packages/core/src/db/index.ts

@@ -1,7 +1,7 @@
-export const openDb = async (dbPath: string | URL) => {
-  if (dbPath instanceof URL) {
-    return (await import('./browser')).openDb(dbPath)
-  } else {
+export const openDb = async (dbPath: string) => {
+  if (typeof window === 'undefined') {
     return (await import('./node')).openDb(dbPath)
+  } else {
+    return (await import('./browser')).openDb(dbPath)
   }
 }

packages/core/src/setup.ts

@@ -26,7 +26,7 @@ type Options = {
   block?: string | number | null
   genesis?: string | Genesis
   buildBlockMode?: BuildBlockMode
-  db?: string | URL
+  db?: string
   mockSignatureHost?: boolean
   allowUnresolvedImports?: boolean
   runtimeLogLevel?: number
@@ -64,11 +64,6 @@ export const setup = async (options: Options) => {
 
   let db: DataSource | undefined
   if (options.db) {
-    if (typeof window !== 'undefined') {
-      if (!(options.db instanceof URL)) {
-        throw new Error('options.db must be an URL to sql-wasm.wasm when running in browser')
-      }
-    }
     db = await openDb(options.db)
   }
 

packages/e2e/package.json

@@ -16,6 +16,6 @@
 	},
 	"devDependencies": {
 		"typescript": "^5.1.6",
-		"vitest": "^0.33.0"
+		"vitest": "^0.34.1"
 	}
 }

packages/testing/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@acala-network/chopsticks-testing",
-	"version": "0.8.0-2",
+	"version": "0.8.0-3",
 	"author": "Bryan Chen <[email protected]>",
 	"license": "Apache-2.0",
 	"scripts": {

packages/web-test/package.json

@@ -12,9 +12,9 @@
 		"@acala-network/chopsticks-core": "workspace:*"
 	},
 	"devDependencies": {
-		"@playwright/test": "^1.36.2",
+		"@playwright/test": "^1.37.0",
 		"crypto-browserify": "^3.12.0",
-		"path-browserify": "^1.0.0",
+		"path-browserify": "^1.0.1",
 		"stream-browserify": "^3.0.0",
 		"typescript": "^5.1.6"
 	}

packages/web-test/src/index.ts

@@ -12,7 +12,7 @@ import { setStorage, setup } from '@acala-network/chopsticks-core'
     endpoint: 'wss://acala-rpc-0.aca-api.network',
     block: 4_000_000,
     mockSignatureHost: true,
-    db: new URL('../../../node_modules/sql.js/dist/sql-wasm.wasm', import.meta.url),
+    db: 'cache',
   })
   globalThis.chain = chain