mpq compress v2

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "diabloweb",
-  "version": "1.0.37",
+  "version": "1.0.39",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "diabloweb",
-  "version": "1.0.37",
+  "version": "1.0.39",
   "private": true,
   "dependencies": {
     "@babel/core": "7.4.3",

src/mpqcmp/MpqCmp.jscc

@@ -1160,7 +1160,7 @@ try {
 }
 
 var TOTAL_STACK = Module['TOTAL_STACK'] || 5242880;
-var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || 536870912;
+var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || 134217728;
 if (TOTAL_MEMORY < TOTAL_STACK) err('TOTAL_MEMORY should be larger than TOTAL_STACK, was ' + TOTAL_MEMORY + '! (TOTAL_STACK=' + TOTAL_STACK + ')');
 
 // Initialize the runtime's memory
@@ -1697,7 +1697,7 @@ function _put_file_size(size){ self.DApi.put_file_size(size); }
 
 STATIC_BASE = GLOBAL_BASE;
 
-STATICTOP = STATIC_BASE + 113200;
+STATICTOP = STATIC_BASE + 28880;
 /* global initializers */  __ATINIT__.push();
 
 
@@ -1706,7 +1706,7 @@ STATICTOP = STATIC_BASE + 113200;
 
 
 
-var STATIC_BUMP = 113200;
+var STATIC_BUMP = 28880;
 Module["STATIC_BASE"] = STATIC_BASE;
 Module["STATIC_BUMP"] = STATIC_BUMP;
 
@@ -2112,7 +2112,8 @@ var asm =Module["asm"]// EMSCRIPTEN_END_ASM
 (Module.asmGlobalArg, Module.asmLibraryArg, buffer);
 
 Module["asm"] = asm;
-var _DApi_MpqCmp = Module["_DApi_MpqCmp"] = function() {  return Module["asm"]["_DApi_MpqCmp"].apply(null, arguments) };
+var _DApi_Alloc = Module["_DApi_Alloc"] = function() {  return Module["asm"]["_DApi_Alloc"].apply(null, arguments) };
+var _DApi_Compress = Module["_DApi_Compress"] = function() {  return Module["asm"]["_DApi_Compress"].apply(null, arguments) };
 var ___cxa_can_catch = Module["___cxa_can_catch"] = function() {  return Module["asm"]["___cxa_can_catch"].apply(null, arguments) };
 var ___cxa_is_pointer_type = Module["___cxa_is_pointer_type"] = function() {  return Module["asm"]["___cxa_is_pointer_type"].apply(null, arguments) };
 var ___em_js__do_error = Module["___em_js__do_error"] = function() {  return Module["asm"]["___em_js__do_error"].apply(null, arguments) };

src/mpqcmp/compress.js

@@ -1,27 +1,221 @@
 import Worker from './mpqcmp.worker.js';
+import MpqBinary from './MpqCmp.wasm';
+import ListFile from './ListFile.txt';
+import axios from 'axios';
 
-export default function compress(mpq, progress) {
+import { decrypt, encrypt, hash, path_name } from '../api/savefile';
-  progress("Loading...");
+
+const MpqSize = 156977;
+const ListSize = 75542;
+
+const readFile = (file, progress) => new Promise((resolve, reject) => {
+  const reader = new FileReader();
+  reader.onload = () => {
+    if (progress) {
+      progress({loaded: file.size});
+    }
+    resolve(reader.result);
+  };
+  reader.onerror = () => reject(reader.error);
+  reader.onabort = () => reject();
+  if (progress) {
+    reader.addEventListener("progress", progress);
+  }
+  reader.readAsArrayBuffer(file);
+});
+
+async function loadFile(url, progress, responseType='arraybuffer') {
+  const binary = await axios.request({
+    url,
+    responseType,
+    onDownloadProgress: progress,
+  });
+  return binary.data;
+}
+
+function runWorker(data, transfer, progress) {
   return new Promise((resolve, reject) => {
     try {
       const worker = new Worker();
       worker.addEventListener("message", ({data}) => {
         switch (data.action) {
         case "result":
-          resolve(data.result);
+          resolve({buffer: data.buffer, blocks: data.blocks});
           break;
         case "error":
           reject({message: data.error, stack: data.stack});
           break;
         case "progress":
-          progress(data.text, data.loaded, data.total);
+          progress(data.value);
           break;
         default:
         }
       });
-      worker.postMessage({action: "run", mpq});
+      worker.postMessage({action: "run", ...data}, transfer);
     } catch (e) {
       reject(e);
     }
   });
 }
+
+export default async function compress(mpq, progress) {
+  progress("Loading...");
+  const files = [];
+  function updateProgress() {
+    progress("Loading...", files.reduce((sum, {loaded, weight}) => sum + loaded * weight, 0),
+      files.reduce((sum, {total, weight}) => sum + total * weight, 0));
+  }
+  const loader = file => e => { file.loaded = e.loaded; updateProgress(); };
+
+  const fHeader = {loaded: 0, weight: 1, total: mpq.size};
+  fHeader.ready = readFile(mpq.slice(0, 32), loader(fHeader));
+  files.push(fHeader);
+
+  const fBinary = {loaded: 0, weight: 5, total: MpqSize};
+  fBinary.ready = loadFile(MpqBinary, loader(fBinary));
+  files.push(fBinary);
+
+  const fList = {loaded: 0, weight: 5, total: ListSize};
+  fList.ready = loadFile(ListFile, loader(fList), 'text');
+  files.push(fList);
+
+  const header = new Uint32Array(await fHeader.ready);
+  const header16 = new Uint16Array(header.buffer);
+
+  if (header[0] !== 0x1A51504D) {
+    throw Error('invalid MPQ file');
+  }
+
+  const blockSize = 1 << (9 + header16[7]);
+  const hashTablePos = header[4];
+  const blockTablePos = header[5];
+  const hashTableSize = header[6];
+  const blockTableSize = header[7];
+  if (hashTablePos + hashTableSize * 16 > mpq.size || blockTablePos + blockTableSize * 16 > mpq.size) {
+    throw Error('invalid MPQ file');
+  }
+
+  const fHashTable = {loaded: 0, weight: 1, total: hashTableSize * 16};
+  const fBlockTable = {loaded: 0, weight: 1, total: blockTableSize * 16};
+  fHeader.total -= fHashTable.total + fBlockTable.total;
+  fHashTable.ready = readFile(mpq.slice(hashTablePos, hashTablePos + fHashTable.total), loader(fHashTable));
+  fBlockTable.ready = readFile(mpq.slice(blockTablePos, blockTablePos + fBlockTable.total), loader(fBlockTable));
+  files.push(fHashTable, fBlockTable);
+
+  const hashTable = new Uint32Array(await fHashTable.ready);
+  const blockTable = new Uint32Array(await fBlockTable.ready);
+  decrypt(hashTable, hash("(hash table)", 3));
+  decrypt(blockTable, hash("(block table)", 3));
+
+  const list = (await fList.ready).split("\n").map(name => name.trim()).filter(name => name.length);
+  const listMap = {};
+  const hashStr = (h1, h2) => h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0');
+  for (let name of list) {
+    listMap[hashStr(hash(name, 1), hash(name, 2))] = name;
+  }
+
+  const NUM_TASKS = 4;
+  const tasks = [];
+  for (let i = 0; i < NUM_TASKS; ++i) {
+    tasks.push({
+      entries: [],
+      min: mpq.size,
+      max: 0,
+      progress: 0,
+    });
+  }
+  
+  for (let i = 0; i < hashTable.length / 4; ++i) {
+    const index = hashTable[i * 4 + 3];
+    if (index === 0xFFFFFFFF || index === 0xFFFFFFFE) {
+      continue;
+    }
+    const name = listMap[hashStr(hashTable[i * 4], hashTable[i * 4 + 1])];
+    if (!name) {
+      hashTable[i * 4 + 3] = 0xFFFFFFFE;
+      continue;
+    }
+
+    const filePos = blockTable[index * 4];
+    const cSize = blockTable[index * 4 + 1];
+
+    const task = tasks[Math.floor(filePos * NUM_TASKS / mpq.size)];
+    task.entries.push(i);
+    task.min = Math.min(task.min, filePos);
+    task.max = Math.max(task.max, filePos + cSize);
+  }
+
+  const numFiles = tasks.reduce((sum, task) => sum + task.entries.length, 0);
+
+  fHeader.total = 32;
+  for (let task of tasks) {
+    if (task.min < task.max) {
+      const fLoad = {loaded: 0, weight: 1, total: task.max - task.min};
+      task.ready = readFile(mpq.slice(task.min, task.max), loader(fLoad)).then(data => task.data = data);
+      files.push(fLoad);
+    }
+  }
+
+  await Promise.all(tasks.map(t => t.ready).filter(Boolean));
+  const binary = await fBinary.ready;
+
+  progress("Processing...");
+
+  for (let task of tasks) {
+    if (task.data) {
+      const input = new Uint32Array(task.entries.length * 6);
+      task.entries.forEach((i, pos) => {
+        const index = hashTable[i * 4 + 3];
+        const name = listMap[hashStr(hashTable[i * 4], hashTable[i * 4 + 1])];
+        input[pos * 6] = blockTable[index * 4];
+        input[pos * 6 + 1] = blockTable[index * 4 + 1];
+        input[pos * 6 + 2] = blockTable[index * 4 + 2];
+        input[pos * 6 + 3] = blockTable[index * 4 + 3];
+        input[pos * 6 + 4] = hash(path_name(name), 3);
+        input[pos * 6 + 5] = name.match(/\.wav$/i) ? 1 : 0;
+      });
+      task.run = runWorker({binary, mpq: task.data, input, offset: task.min, blockSize}, [task.data, input.buffer], value => {
+        task.progress = value;
+        const sum = tasks.reduce((sum, task) => sum + task.progress, 0);
+        progress("Processing...", sum, numFiles);
+      }).then(res => task.result = res);
+    }
+  }
+
+  await Promise.all(tasks.map(t => t.run).filter(Boolean));
+
+  let outputPos = 32 + fHashTable.total + fBlockTable.total;
+  const outputSize = tasks.reduce((sum, {result}) => sum + (result ? result.buffer.byteLength : 0), outputPos);
+  const output = [header.buffer, hashTable.buffer, blockTable.buffer];
+
+  blockTable.fill(0);
+  let blockPos = 0;
+  for (let task of tasks) {
+    if (task.result) {
+      const {buffer, blocks} = task.result;
+      for (let pos = 0; pos < task.entries.length; ++pos) {
+        const i = task.entries[pos];
+        hashTable[i * 4 + 3] = blockPos + pos;
+        blocks[pos * 4] += outputPos;
+      }
+      blockTable.set(blocks, blockPos * 4);
+      blockPos += task.entries.length;
+      output.push(buffer);
+      outputPos += buffer.byteLength;
+    }
+  }
+
+  header[1] = 32;
+  header[2] = outputSize;
+  header16[6] = 1;
+  header16[7] = 7;
+  header[4] = 32;
+  header[5] = 32 + hashTable.length * 4;
+  header[6] = hashTable.length / 4;
+  header[7] = blockTable.length / 4;
+
+  encrypt(hashTable, hash("(hash table)", 3));
+  encrypt(blockTable, hash("(block table)", 3));
+
+  return new Blob(output, {type: 'binary/octet-stream'});
+}

src/mpqcmp/index.js

@@ -14,8 +14,8 @@ export default class CompressMpq extends React.Component {
   onProgress(progress) {
     this.setState({progress});
   }
-  onDone = result => {
+  onDone = blob => {
-    const blob = new Blob([result], {type: 'binary/octet-stream'});
+    //const blob = new Blob([result], {type: 'binary/octet-stream'});
     const url = URL.createObjectURL(blob);
     this.setState({url});
 

src/mpqcmp/mpqcmp.worker.js

@@ -1,17 +1,14 @@
-import MpqBinary from './MpqCmp.wasm';
 import MpqModule from './MpqCmp.jscc';
-import axios from 'axios';
-
-const MpqSize = 356747;
 
 /* eslint-disable-next-line no-restricted-globals */
 const worker = self;
 
 let input_file = null;
+let input_offset = 0;
 let output_file = null;
 let last_progress = 0;
-function progress(text, loaded, total) {
+function progress(value) {
-  worker.postMessage({action: "progress", text, loaded, total});
+  worker.postMessage({action: "progress", value});
 }
 
 const DApi = {
@@ -20,10 +17,9 @@ const DApi = {
   },
 
   get_file_contents(array, offset) {
-    array.set(input_file.subarray(offset, offset + array.byteLength));
+    array.set(input_file.subarray(offset - input_offset, offset - input_offset + array.byteLength));
   },
   put_file_size(size) {
-    debugger;
     output_file = new Uint8Array(size);
   },
   put_file_contents(array, offset) {
@@ -32,7 +28,7 @@ const DApi = {
 
   progress(done, total) {
     if (done === total || performance.now() > last_progress + 100) {
-      progress("Processing...", done, total);
+      progress(done);
       last_progress = performance.now();
     }
   },
@@ -40,68 +36,26 @@ const DApi = {
 
 worker.DApi = DApi;
 
-let wasm = null;
+async function run({binary, mpq, input, offset, blockSize}) {
-
+  const wasm = await MpqModule({wasmBinary: binary}).ready;
-const readFile = (file, progress) => new Promise((resolve, reject) => {
-  const reader = new FileReader();
-  reader.onload = () => {
-    if (progress) {
-      progress({loaded: file.size});
-    }
-    resolve(reader.result);
-  };
-  reader.onerror = () => reject(reader.error);
-  reader.onabort = () => reject();
-  if (progress) {
-    reader.addEventListener("progress", progress);
-  }
-  reader.readAsArrayBuffer(file);
-});
-
-async function initWasm(progress) {
-  const binary = await axios.request({
-    url: MpqBinary,
-    responseType: 'arraybuffer',
-    onDownloadProgress: progress,
-  });
-  const result = await MpqModule({
-    wasmBinary: binary.data,
-  }).ready;
-  progress({loaded: MpqSize});
-  return result;
-}
-
-async function run(mpq) {
-  progress("Loading...");
-  let mpqLoaded = 0, mpqTotal = (mpq ? mpq.size : 0), wasmLoaded = 0, wasmTotal = MpqSize;
-  const wasmWeight = 5;
-  function updateProgress() {
-    progress("Loading...", mpqLoaded + wasmLoaded * wasmWeight, mpqTotal + wasmTotal * wasmWeight);
-  }
-  const loadWasm = initWasm(e => {
-    wasmLoaded = Math.min(e.loaded, wasmTotal);
-    updateProgress();
-  });
-  let loadMpq = mpq ? readFile(mpq, e => {
-    mpqLoaded = e.loaded;
-    updateProgress();
-  }) : Promise.resolve(null);
-  [wasm, mpq] = await Promise.all([loadWasm, loadMpq]);
 
   input_file = new Uint8Array(mpq);
+  input_offset = offset;
 
-  progress("Processing...");
+  const count = input.length / 6;
+  const ptr = wasm._DApi_Alloc(input.byteLength);
+  wasm.HEAPU32.set(input, ptr >> 2);
 
-  wasm._DApi_MpqCmp(input_file.length);
+  const dst = wasm._DApi_Compress(offset + input_file.length, blockSize, count, ptr) >> 2;
 
-  return output_file.buffer;
+  return [output_file.buffer, wasm.HEAPU32.slice(dst , dst + count * 4)];
 }
 
 worker.addEventListener("message", ({data}) => {
   switch (data.action) {
   case "run":
-    run(data.mpq).then(
+    run(data).then(
-      result => worker.postMessage({action: "result", result}, [result]),
+      ([buffer, blocks]) => worker.postMessage({action: "result", buffer, blocks}, [buffer, blocks.buffer]),
       err => worker.postMessage({action: "error", error: err.toString(), stack: err.stack}));
     break;
   default: