feat(mpq): mpq reader that can extract files from patch_d2.mpq

.gitignore

@@ -4,4 +4,5 @@ tsconfig.tsbuildinfo
 dist
 .cache
 cdk.context.json
-test-data
+test-data
+*.ndjson

README.md

@@ -7,5 +7,6 @@ Tools to work with diablo2
 - [bintools](./packages/bintools) - Binary parsers to read the `.bin` files
 - [huffman](./packages/huffman) - Decompressor for network data
 - [packets](./packages/packets) - Diablo2 network protocol
-- [map](./packages/map) - Diablo2 map generation api (Docker)
+- [mpq](./packages/mpq) - MPQ reader / extractor
+- [map](./packages/map) - Diablo2 map generation api (Docker based)
 - [sniffer](./packages/sniffer) - Diablo2 network sniffer

packages/bintools/package.json

@@ -14,7 +14,7 @@
   },
   "dependencies": {
     "@diablo2/data": "^0.1.0",
-    "binparse": "^1.0.0",
+    "binparse": "^1.0.1",
     "ulid": "^2.3.0"
   },
   "publishConfig": {

packages/mpq/README.md

@@ -0,0 +1,21 @@
+## @diablo2/mpq
+
+MPQ Reader for diablo2
+
+Supports decoding PKWARE compression
+
+
+```typescript
+
+import {Mpq} from '@diablo2/mpq'
+
+// Read from file
+const mpq = Mpq.load('patch_d2.mpq');
+const lang = await mpq.extract('data\\local\\LNG\\ENG\\patchstring.tbl');
+
+
+// Read from buffer
+const mpqBuf = await fs.promises.read('patch_d2.mpq');
+const mpq = Mpq.load(mpqBuf);
+const lang = await mpq.extract('data\\local\\LNG\\ENG\\patchstring.tbl');
+```

packages/mpq/example/index.ts

@@ -0,0 +1,40 @@
+import { Mpq } from '../src';
+import { promises as fs } from 'fs';
+import { createHash } from 'crypto';
+import { toHex } from 'binparse';
+
+const MpqFileName = './patch_d2.mpq';
+async function main(): Promise<void> {
+  // const bytes = await decompressSector(sector, 0, sector.length);
+  // console.log(bytes);
+
+  console.time('ReadMpq');
+  const fileData = await fs.readFile(MpqFileName);
+  const mpqFileHash = createHash('sha256').update(fileData).digest('base64');
+  const mpq = Mpq.load(fileData);
+  console.timeEnd('ReadMpq');
+
+  console.log('LoadedMPQ', MpqFileName, { hash: mpqFileHash });
+
+  const files: string[] = ['patchstring.tbl', 'expansionstring.tbl', 'string.tbl'];
+  for (const fileName of files) {
+    console.time(`extracting:${fileName}`);
+    const dat = await mpq.extract(`data\\local\\LNG\\ENG\\${fileName}`);
+    console.timeEnd(`extracting:${fileName}`);
+    if (dat == null) continue;
+    const hash = createHash('sha1').update(dat).digest('hex');
+
+    console.log('Extracted', { fileName, hash, size: dat.length });
+
+    const readFile = readFileSync(`/home/blacha/workspace/tmp/mpq-ex/ext/data/local/LNG/ENG/${fileName}`);
+    let ok = true;
+    for (let i = 0; i < readFile.length; i++) {
+      if (readFile[i] == dat[i]) continue;
+      console.log('MissMatch', i, { expected: toHex(readFile[i]), got: toHex(dat[i]) });
+      ok = false;
+      break;
+    }
+    console.log({ ok });
+  }
+}
+main();

packages/mpq/package.json

@@ -0,0 +1,25 @@
+{
+    "name": "@diablo2/mpq",
+    "version": "0.1.0",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/blacha/diablo2.git",
+        "directory": "packages/mpq"
+    },
+    "license": "MIT",
+    "main": "./build/index.js",
+    "types": "./build/index.d.ts",
+    "scripts": {
+        "test": "ospec --globs 'build/**/*.test.js'"
+    },
+    "dependencies": {
+        "binparse": "^1.0.1",
+        "implode-decoder": "^1.0.1"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "files": [
+        "build/"
+    ]
+}

packages/mpq/src/__test__/mpq.encryption.data.ts

@@ -0,0 +1,18 @@
+export const MpqDecryptionTests = [
+  {
+    name: 'tableA',
+    input:
+      'W2NIPTTaCMqNVTb447FH6X8dP7akh8VkNgR/4uZkCz0K49H4mvy6jJaeI/m0VLqVkfwNEpdYeN067d3GJqtFnb+nA/tl1p1SAKRgi05uIQDSi6hgWUdo7zULcrExselXmYYoGmQ+8eh60G9BvMIaaHO6e2JfkBepVmQSseSJCtwY07W5yswXaolQGoKNh4c9QHyBswYYTa3Q70mR7nsztSGhqxA94kIdzPCnqsqhUQk+vt9tr2qEo+4MHOkqUM6CwvCh47qXeeFj7QVLxYRvpAvaaSk8ELZNykeB6YphVyLc7Z3JbkD5di00KitJ15MGwtEX4MVirGywVJI87NKIyYBAq69GiUqc+VKkCh9h2mk=',
+    output:
+      '0AQAAGUCAABlAgAAAAIAgUUHAAAfAgAAHwIAAAACAIF0CQAALgEAAEMCAAAAAgCBsgoAACQGAAA1DQAAAAIAgeYQAAA+BgAA6AwAAAACAIE0FwAAhU0AAGGQAAAAAgCB2WQAAPMWAAA4JwAAAAIAgdx7AAA4AAAAOAAAAAACAIEkfAAAKAAAADQAAAAAAgCBXHwAAHABAACuAQAAAAIAgQAAAAAAAAAAAAAAAAACAIDcfQAArxIAAMx+AAAAAgCBm5AAACgCAACxAwAAAAIAgdOSAACbAwAA2xQAAAACAIF+lgAADAAAAAwAAAAAAgCBmpYAALUAAABNAQAAAAIAgV+XAABcAQAAXAEAAAACAIE=',
+    key: 0xec83b3a3,
+  },
+  {
+    name: 'tableB',
+    input:
+      'Ygo1UN89fvcs2XvCVmEGtlBGhEnnwKMpeQ6UcHWi76wcTH+DZ/SxaBM6JLDJIIXu/HaW4y+f/bwPlRtG2B0X3l8pYlcMsc0JqzhYuVSJhbr63okln89wI/tRK2imNW+n4Y29K68wRkrPH/pFlgDH2cIPYSHB8L9vr50LLGsYWdcsveBF3ntvLcP6u9PwrmERD/NrsziQEPfZeX/hK87lPNOrgP2N1I0Ukwb57sud8/dUi/URBg8efOvyPPgYrAHGw6va6idpntcXvYqA5EZrZlvfiMSIJssbp10EC9BNtkFTjGcJqilIxQaqeusfj6mUqjuDZ8h/mTUQQpx672qtogoQidGQXyLjkikAfqAg32eLrJmkHVdGY+I9bsTvVVwY7nm7mffyUKd4NMyZqtzJ/m5BhNG/KAI5gX/5r3I0A9nbHvMObRPOGDKE7So/ZsROHeCkOMkLpb0RJE+JI428h+DRDnmtaQ9hbAXR+DXuF+l2l3IUaJ88zo2PLY763AUaWeKV0MJoeLmjY+LwNfGlOBWcW0euO8F30jZ2PeXVFhZGT2svOFFMSF17M0rKQslWKQLGpZKCrDRzt5nSBb9HUcqinHLXV1eM1rgWjd8OFAZgzfLQkpLDGvfnkmukQhakc7Llfsswobbr6Ns8N+3hrAqzV2Fjcty+ZTWAWdqmYLM=',
+    output:
+      'rsUJ1lnRqMoAAAAAAQAAAGTPaUyL40FgAAAAAAQAAAAJQvjByg1YQQAAAAAOAAAAS6XCqpUrdvQAAAAABgAAAP////////////////////9wt+XJtvYYOwAAAAAJAAAAewg8NII2jicAAAAACAAAAKAeKztX8C63AAAAAAwAAAD//////////////////////////////////////////5zCg9OSLkDvAAAAAAAAAAD/////////////////////C8Eu6YwgFEIAAAAAAgAAAP/////////////////////dmUR2sGEbawAAAAAFAAAAyzeE0+zq3wcAAAAAEAAAAOHXdC4QHieLAAAAAAoAAAD//////////////////////////////////////////57LGf5dcEBiAAAAAAsAAAD/////////////////////3It+Wlw/Jf8AAAAAAwAAAP///////////////////////////////////////////////////////////////xB5Zf2nmJtOAAAAAA8AAAD/////////////////////////////////////////////////////////////////////////////////////z7CoHSj/zqIAAAAADQAAAIkilTGj+l9qAAAAAAcAAAA=',
+    key: 3283040112,
+  },
+];