diff --git a/package-lock.json b/package-lock.json index 3f7e9f0..384309a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "diabloweb", - "version": "1.0.37", + "version": "1.0.39", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e9ae3f8..17257b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "diabloweb", - "version": "1.0.37", + "version": "1.0.39", "private": true, "dependencies": { "@babel/core": "7.4.3", diff --git a/src/mpqcmp/MpqCmp.jscc b/src/mpqcmp/MpqCmp.jscc index b48a5e7..205b0f1 100644 --- a/src/mpqcmp/MpqCmp.jscc +++ b/src/mpqcmp/MpqCmp.jscc @@ -1160,7 +1160,7 @@ try { } var TOTAL_STACK = Module['TOTAL_STACK'] || 5242880; -var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || 536870912; +var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || 134217728; if (TOTAL_MEMORY < TOTAL_STACK) err('TOTAL_MEMORY should be larger than TOTAL_STACK, was ' + TOTAL_MEMORY + '! (TOTAL_STACK=' + TOTAL_STACK + ')'); // Initialize the runtime's memory @@ -1697,7 +1697,7 @@ function _put_file_size(size){ self.DApi.put_file_size(size); } STATIC_BASE = GLOBAL_BASE; -STATICTOP = STATIC_BASE + 113200; +STATICTOP = STATIC_BASE + 28880; /* global initializers */ __ATINIT__.push(); @@ -1706,7 +1706,7 @@ STATICTOP = STATIC_BASE + 113200; -var STATIC_BUMP = 113200; +var STATIC_BUMP = 28880; Module["STATIC_BASE"] = STATIC_BASE; Module["STATIC_BUMP"] = STATIC_BUMP; @@ -2112,7 +2112,8 @@ var asm =Module["asm"]// EMSCRIPTEN_END_ASM (Module.asmGlobalArg, Module.asmLibraryArg, buffer); Module["asm"] = asm; -var _DApi_MpqCmp = Module["_DApi_MpqCmp"] = function() { return Module["asm"]["_DApi_MpqCmp"].apply(null, arguments) }; +var _DApi_Alloc = Module["_DApi_Alloc"] = function() { return Module["asm"]["_DApi_Alloc"].apply(null, arguments) }; +var _DApi_Compress = Module["_DApi_Compress"] = function() { return Module["asm"]["_DApi_Compress"].apply(null, arguments) }; var ___cxa_can_catch = Module["___cxa_can_catch"] = function() { return Module["asm"]["___cxa_can_catch"].apply(null, arguments) }; var ___cxa_is_pointer_type = Module["___cxa_is_pointer_type"] = function() { return Module["asm"]["___cxa_is_pointer_type"].apply(null, arguments) }; var ___em_js__do_error = Module["___em_js__do_error"] = function() { return Module["asm"]["___em_js__do_error"].apply(null, arguments) }; diff --git a/src/mpqcmp/MpqCmp.wasm b/src/mpqcmp/MpqCmp.wasm index a1b8386..728bf37 100644 Binary files a/src/mpqcmp/MpqCmp.wasm and b/src/mpqcmp/MpqCmp.wasm differ diff --git a/src/mpqcmp/compress.js b/src/mpqcmp/compress.js index 9f602a6..f1c6fa9 100644 --- a/src/mpqcmp/compress.js +++ b/src/mpqcmp/compress.js @@ -1,27 +1,221 @@ import Worker from './mpqcmp.worker.js'; +import MpqBinary from './MpqCmp.wasm'; +import ListFile from './ListFile.txt'; +import axios from 'axios'; -export default function compress(mpq, progress) { - progress("Loading..."); +import { decrypt, encrypt, hash, path_name } from '../api/savefile'; + +const MpqSize = 156977; +const ListSize = 75542; + +const readFile = (file, progress) => new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + if (progress) { + progress({loaded: file.size}); + } + resolve(reader.result); + }; + reader.onerror = () => reject(reader.error); + reader.onabort = () => reject(); + if (progress) { + reader.addEventListener("progress", progress); + } + reader.readAsArrayBuffer(file); +}); + +async function loadFile(url, progress, responseType='arraybuffer') { + const binary = await axios.request({ + url, + responseType, + onDownloadProgress: progress, + }); + return binary.data; +} + +function runWorker(data, transfer, progress) { return new Promise((resolve, reject) => { try { const worker = new Worker(); worker.addEventListener("message", ({data}) => { switch (data.action) { case "result": - resolve(data.result); + resolve({buffer: data.buffer, blocks: data.blocks}); break; case "error": reject({message: data.error, stack: data.stack}); break; case "progress": - progress(data.text, data.loaded, data.total); + progress(data.value); break; default: } }); - worker.postMessage({action: "run", mpq}); + worker.postMessage({action: "run", ...data}, transfer); } catch (e) { reject(e); } }); } + +export default async function compress(mpq, progress) { + progress("Loading..."); + const files = []; + function updateProgress() { + progress("Loading...", files.reduce((sum, {loaded, weight}) => sum + loaded * weight, 0), + files.reduce((sum, {total, weight}) => sum + total * weight, 0)); + } + const loader = file => e => { file.loaded = e.loaded; updateProgress(); }; + + const fHeader = {loaded: 0, weight: 1, total: mpq.size}; + fHeader.ready = readFile(mpq.slice(0, 32), loader(fHeader)); + files.push(fHeader); + + const fBinary = {loaded: 0, weight: 5, total: MpqSize}; + fBinary.ready = loadFile(MpqBinary, loader(fBinary)); + files.push(fBinary); + + const fList = {loaded: 0, weight: 5, total: ListSize}; + fList.ready = loadFile(ListFile, loader(fList), 'text'); + files.push(fList); + + const header = new Uint32Array(await fHeader.ready); + const header16 = new Uint16Array(header.buffer); + + if (header[0] !== 0x1A51504D) { + throw Error('invalid MPQ file'); + } + + const blockSize = 1 << (9 + header16[7]); + const hashTablePos = header[4]; + const blockTablePos = header[5]; + const hashTableSize = header[6]; + const blockTableSize = header[7]; + if (hashTablePos + hashTableSize * 16 > mpq.size || blockTablePos + blockTableSize * 16 > mpq.size) { + throw Error('invalid MPQ file'); + } + + const fHashTable = {loaded: 0, weight: 1, total: hashTableSize * 16}; + const fBlockTable = {loaded: 0, weight: 1, total: blockTableSize * 16}; + fHeader.total -= fHashTable.total + fBlockTable.total; + fHashTable.ready = readFile(mpq.slice(hashTablePos, hashTablePos + fHashTable.total), loader(fHashTable)); + fBlockTable.ready = readFile(mpq.slice(blockTablePos, blockTablePos + fBlockTable.total), loader(fBlockTable)); + files.push(fHashTable, fBlockTable); + + const hashTable = new Uint32Array(await fHashTable.ready); + const blockTable = new Uint32Array(await fBlockTable.ready); + decrypt(hashTable, hash("(hash table)", 3)); + decrypt(blockTable, hash("(block table)", 3)); + + const list = (await fList.ready).split("\n").map(name => name.trim()).filter(name => name.length); + const listMap = {}; + const hashStr = (h1, h2) => h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0'); + for (let name of list) { + listMap[hashStr(hash(name, 1), hash(name, 2))] = name; + } + + const NUM_TASKS = 4; + const tasks = []; + for (let i = 0; i < NUM_TASKS; ++i) { + tasks.push({ + entries: [], + min: mpq.size, + max: 0, + progress: 0, + }); + } + + for (let i = 0; i < hashTable.length / 4; ++i) { + const index = hashTable[i * 4 + 3]; + if (index === 0xFFFFFFFF || index === 0xFFFFFFFE) { + continue; + } + const name = listMap[hashStr(hashTable[i * 4], hashTable[i * 4 + 1])]; + if (!name) { + hashTable[i * 4 + 3] = 0xFFFFFFFE; + continue; + } + + const filePos = blockTable[index * 4]; + const cSize = blockTable[index * 4 + 1]; + + const task = tasks[Math.floor(filePos * NUM_TASKS / mpq.size)]; + task.entries.push(i); + task.min = Math.min(task.min, filePos); + task.max = Math.max(task.max, filePos + cSize); + } + + const numFiles = tasks.reduce((sum, task) => sum + task.entries.length, 0); + + fHeader.total = 32; + for (let task of tasks) { + if (task.min < task.max) { + const fLoad = {loaded: 0, weight: 1, total: task.max - task.min}; + task.ready = readFile(mpq.slice(task.min, task.max), loader(fLoad)).then(data => task.data = data); + files.push(fLoad); + } + } + + await Promise.all(tasks.map(t => t.ready).filter(Boolean)); + const binary = await fBinary.ready; + + progress("Processing..."); + + for (let task of tasks) { + if (task.data) { + const input = new Uint32Array(task.entries.length * 6); + task.entries.forEach((i, pos) => { + const index = hashTable[i * 4 + 3]; + const name = listMap[hashStr(hashTable[i * 4], hashTable[i * 4 + 1])]; + input[pos * 6] = blockTable[index * 4]; + input[pos * 6 + 1] = blockTable[index * 4 + 1]; + input[pos * 6 + 2] = blockTable[index * 4 + 2]; + input[pos * 6 + 3] = blockTable[index * 4 + 3]; + input[pos * 6 + 4] = hash(path_name(name), 3); + input[pos * 6 + 5] = name.match(/\.wav$/i) ? 1 : 0; + }); + task.run = runWorker({binary, mpq: task.data, input, offset: task.min, blockSize}, [task.data, input.buffer], value => { + task.progress = value; + const sum = tasks.reduce((sum, task) => sum + task.progress, 0); + progress("Processing...", sum, numFiles); + }).then(res => task.result = res); + } + } + + await Promise.all(tasks.map(t => t.run).filter(Boolean)); + + let outputPos = 32 + fHashTable.total + fBlockTable.total; + const outputSize = tasks.reduce((sum, {result}) => sum + (result ? result.buffer.byteLength : 0), outputPos); + const output = [header.buffer, hashTable.buffer, blockTable.buffer]; + + blockTable.fill(0); + let blockPos = 0; + for (let task of tasks) { + if (task.result) { + const {buffer, blocks} = task.result; + for (let pos = 0; pos < task.entries.length; ++pos) { + const i = task.entries[pos]; + hashTable[i * 4 + 3] = blockPos + pos; + blocks[pos * 4] += outputPos; + } + blockTable.set(blocks, blockPos * 4); + blockPos += task.entries.length; + output.push(buffer); + outputPos += buffer.byteLength; + } + } + + header[1] = 32; + header[2] = outputSize; + header16[6] = 1; + header16[7] = 7; + header[4] = 32; + header[5] = 32 + hashTable.length * 4; + header[6] = hashTable.length / 4; + header[7] = blockTable.length / 4; + + encrypt(hashTable, hash("(hash table)", 3)); + encrypt(blockTable, hash("(block table)", 3)); + + return new Blob(output, {type: 'binary/octet-stream'}); +} diff --git a/src/mpqcmp/index.js b/src/mpqcmp/index.js index bce78d8..a58fdf7 100644 --- a/src/mpqcmp/index.js +++ b/src/mpqcmp/index.js @@ -14,8 +14,8 @@ export default class CompressMpq extends React.Component { onProgress(progress) { this.setState({progress}); } - onDone = result => { - const blob = new Blob([result], {type: 'binary/octet-stream'}); + onDone = blob => { + //const blob = new Blob([result], {type: 'binary/octet-stream'}); const url = URL.createObjectURL(blob); this.setState({url}); diff --git a/src/mpqcmp/mpqcmp.worker.js b/src/mpqcmp/mpqcmp.worker.js index 6852f74..0d6b0b5 100644 --- a/src/mpqcmp/mpqcmp.worker.js +++ b/src/mpqcmp/mpqcmp.worker.js @@ -1,17 +1,14 @@ -import MpqBinary from './MpqCmp.wasm'; import MpqModule from './MpqCmp.jscc'; -import axios from 'axios'; - -const MpqSize = 356747; /* eslint-disable-next-line no-restricted-globals */ const worker = self; let input_file = null; +let input_offset = 0; let output_file = null; let last_progress = 0; -function progress(text, loaded, total) { - worker.postMessage({action: "progress", text, loaded, total}); +function progress(value) { + worker.postMessage({action: "progress", value}); } const DApi = { @@ -20,10 +17,9 @@ const DApi = { }, get_file_contents(array, offset) { - array.set(input_file.subarray(offset, offset + array.byteLength)); + array.set(input_file.subarray(offset - input_offset, offset - input_offset + array.byteLength)); }, put_file_size(size) { - debugger; output_file = new Uint8Array(size); }, put_file_contents(array, offset) { @@ -32,7 +28,7 @@ const DApi = { progress(done, total) { if (done === total || performance.now() > last_progress + 100) { - progress("Processing...", done, total); + progress(done); last_progress = performance.now(); } }, @@ -40,70 +36,28 @@ const DApi = { worker.DApi = DApi; -let wasm = null; - -const readFile = (file, progress) => new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - if (progress) { - progress({loaded: file.size}); - } - resolve(reader.result); - }; - reader.onerror = () => reject(reader.error); - reader.onabort = () => reject(); - if (progress) { - reader.addEventListener("progress", progress); - } - reader.readAsArrayBuffer(file); -}); - -async function initWasm(progress) { - const binary = await axios.request({ - url: MpqBinary, - responseType: 'arraybuffer', - onDownloadProgress: progress, - }); - const result = await MpqModule({ - wasmBinary: binary.data, - }).ready; - progress({loaded: MpqSize}); - return result; -} - -async function run(mpq) { - progress("Loading..."); - let mpqLoaded = 0, mpqTotal = (mpq ? mpq.size : 0), wasmLoaded = 0, wasmTotal = MpqSize; - const wasmWeight = 5; - function updateProgress() { - progress("Loading...", mpqLoaded + wasmLoaded * wasmWeight, mpqTotal + wasmTotal * wasmWeight); - } - const loadWasm = initWasm(e => { - wasmLoaded = Math.min(e.loaded, wasmTotal); - updateProgress(); - }); - let loadMpq = mpq ? readFile(mpq, e => { - mpqLoaded = e.loaded; - updateProgress(); - }) : Promise.resolve(null); - [wasm, mpq] = await Promise.all([loadWasm, loadMpq]); +async function run({binary, mpq, input, offset, blockSize}) { + const wasm = await MpqModule({wasmBinary: binary}).ready; input_file = new Uint8Array(mpq); + input_offset = offset; - progress("Processing..."); + const count = input.length / 6; + const ptr = wasm._DApi_Alloc(input.byteLength); + wasm.HEAPU32.set(input, ptr >> 2); - wasm._DApi_MpqCmp(input_file.length); + const dst = wasm._DApi_Compress(offset + input_file.length, blockSize, count, ptr) >> 2; - return output_file.buffer; + return [output_file.buffer, wasm.HEAPU32.slice(dst , dst + count * 4)]; } worker.addEventListener("message", ({data}) => { switch (data.action) { case "run": - run(data.mpq).then( - result => worker.postMessage({action: "result", result}, [result]), + run(data).then( + ([buffer, blocks]) => worker.postMessage({action: "result", buffer, blocks}, [buffer, blocks.buffer]), err => worker.postMessage({action: "error", error: err.toString(), stack: err.stack})); break; default: } -}); \ No newline at end of file +});