Add XPRESS (MS-XCA) decompression operations
This commit is contained in:
MP GOWTHAM
parent
4290ea7539
commit
699bd73c0d
@ -433,7 +433,9 @@
|
||||
"LZMA Compress",
|
||||
"LZ4 Decompress",
|
||||
"LZ4 Compress",
|
||||
"LZNT1 Decompress"
|
||||
"LZNT1 Decompress",
|
||||
"XPRESS Decompress",
|
||||
"XPRESS LZ77+Huffman Decompress"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
263
src/core/lib/XPRESS.mjs
Normal file
263
src/core/lib/XPRESS.mjs
Normal file
@ -0,0 +1,263 @@
|
||||
/**
|
||||
* XPRESS (MS-XCA) decompression.
|
||||
*
|
||||
* @author MP Gowtham [mpgowtham@users.noreply.github.com]
|
||||
* @copyright Crown Copyright 2026
|
||||
* @license Apache-2.0
|
||||
*
|
||||
* Implements the two XPRESS variants from:
|
||||
* https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/
|
||||
* (2.1 XPRESS Algorithm Details, 2.2 LZ77+Huffman Algorithm Details)
|
||||
*
|
||||
* Cross-validated against the reference decoder in The Sleuth Kit
|
||||
* (tsk/fs/xpress.c) and go-ntfs (parser/xpress.go).
|
||||
*/
|
||||
|
||||
import OperationError from "../errors/OperationError.mjs";
|
||||
|
||||
/** Maximum output per call (Windows limits XPRESS blocks to 1 MiB). */
|
||||
const MAX_DECOMPRESSED = 1000000;
|
||||
|
||||
/**
|
||||
* Decompress an XPRESS plain-LZ77 stream.
|
||||
*
|
||||
* The stream is self-terminating: a sequence of 32-bit flag groups
|
||||
* tested from bit 31 down. A clear bit is a literal byte. A set bit is
|
||||
* a match described by an LE16 word, (offset-1) in the top 13 bits and
|
||||
* (length-3) in the low 3 bits. A match whose low 3 bits are 7 uses the
|
||||
* shared-nibble form: the low nibble of the next stream byte extends
|
||||
* the length, and its high nibble extends the next match that also uses
|
||||
* this form. A nibble of 15 selects a raw length: a byte, an LE16 if
|
||||
* the byte is 255, or an LE32 if the LE16 is 0. The final flag group
|
||||
* is padded with set bits; a match flag with no input left is the
|
||||
* end-of-data marker.
|
||||
*
|
||||
* @param {byteArray} input
|
||||
* @returns {byteArray} decompressed data
|
||||
*/
|
||||
export function decompress(input) {
|
||||
const out = [];
|
||||
let pending = -1; // offset of the shared-nibble byte, -1 when none pending
|
||||
let flags = 0;
|
||||
let flagsLeft = 0;
|
||||
let i = 0;
|
||||
|
||||
while (true) {
|
||||
if (flagsLeft === 0) {
|
||||
if (input.length - i < 4)
|
||||
throw new OperationError("XPRESS: truncated flag group");
|
||||
flags = (input[i] | (input[i + 1] << 8) |
|
||||
(input[i + 2] << 16) | (input[i + 3] << 24)) >>> 0;
|
||||
i += 4;
|
||||
flagsLeft = 32;
|
||||
}
|
||||
flagsLeft--;
|
||||
if (((flags >>> flagsLeft) & 1) === 0) {
|
||||
if (i >= input.length)
|
||||
throw new OperationError("XPRESS: truncated literal");
|
||||
out.push(input[i++]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// A set flag with no input left is the end-of-data marker.
|
||||
if (i >= input.length)
|
||||
return out;
|
||||
if (input.length - i < 2)
|
||||
throw new OperationError("XPRESS: truncated match");
|
||||
const mb = input[i] | (input[i + 1] << 8);
|
||||
i += 2;
|
||||
const moff = (mb >>> 3) + 1;
|
||||
let mlen = (mb & 7) + 3;
|
||||
|
||||
if ((mb & 7) === 7) {
|
||||
let nib;
|
||||
if (pending === -1) {
|
||||
if (i >= input.length)
|
||||
throw new OperationError("XPRESS: truncated shared nibble");
|
||||
nib = input[i] & 0x0f;
|
||||
pending = i++;
|
||||
} else {
|
||||
nib = input[pending] >>> 4;
|
||||
pending = -1;
|
||||
}
|
||||
if (nib === 15) {
|
||||
let v = 0;
|
||||
if (i >= input.length)
|
||||
throw new OperationError("XPRESS: truncated raw length");
|
||||
v = input[i++];
|
||||
if (v === 255) {
|
||||
if (input.length - i < 2)
|
||||
throw new OperationError("XPRESS: truncated raw length");
|
||||
v = input[i] | (input[i + 1] << 8);
|
||||
i += 2;
|
||||
if (v === 0) {
|
||||
if (input.length - i < 4)
|
||||
throw new OperationError("XPRESS: truncated raw length");
|
||||
v = (input[i] | (input[i + 1] << 8) |
|
||||
(input[i + 2] << 16) | (input[i + 3] << 24)) >>> 0;
|
||||
i += 4;
|
||||
}
|
||||
}
|
||||
if (v < 22)
|
||||
throw new OperationError("XPRESS: invalid match length");
|
||||
mlen = v + 3;
|
||||
} else {
|
||||
mlen = nib + 3;
|
||||
}
|
||||
}
|
||||
|
||||
if (moff > 8192 || moff > out.length)
|
||||
throw new OperationError("XPRESS: match offset out of range");
|
||||
if (out.length + mlen > MAX_DECOMPRESSED)
|
||||
throw new OperationError("XPRESS: decompression ratio too large");
|
||||
|
||||
const start = out.length - moff;
|
||||
for (let j = 0; j < mlen; j++)
|
||||
out.push(out[start + j]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress an XPRESS LZ77+Huffman stream into exactly
|
||||
* decompressedSize bytes.
|
||||
*
|
||||
* The first 256 bytes hold 512 4-bit code lengths, the even symbol in
|
||||
* the low nibble and the odd in the high. Canonical codes are assigned
|
||||
* in (length, symbol) order, most-significant bit first. The bit stream
|
||||
* follows as LE16 words, MSB first, read through a 32-bit register
|
||||
* refilled while fewer than 15 bits remain. Symbols 0..255 are
|
||||
* literals. Symbol 256 is end-of-data; mid-stream it decodes as a match
|
||||
* of length 3 at offset 1. Symbols 257..511 are matches: ((s-256)>>4)
|
||||
* selects the offset bit width, ((s-256)&15) the base length, with a
|
||||
* nibble of 15 selecting a raw length byte (an LE16 if 255, an LE32 if
|
||||
* the LE16 is 0).
|
||||
*
|
||||
* @param {byteArray} input
|
||||
* @param {number} decompressedSize
|
||||
* @returns {byteArray} decompressed data
|
||||
*/
|
||||
export function decompressHuffman(input, decompressedSize) {
|
||||
if (decompressedSize <= 0 || decompressedSize > MAX_DECOMPRESSED)
|
||||
throw new OperationError("XPRESS: invalid decompressed size");
|
||||
if (input.length < 256)
|
||||
throw new OperationError("XPRESS: truncated Huffman table");
|
||||
|
||||
const lens = new Array(512);
|
||||
for (let l = 0; l < 256; l++) {
|
||||
lens[l * 2] = input[l] & 0x0f;
|
||||
lens[l * 2 + 1] = input[l] >>> 4;
|
||||
}
|
||||
|
||||
// Decode table in canonical (length, symbol) order, MSB first.
|
||||
const TABLE_BITS = 15;
|
||||
const TABLE_SIZE = 1 << TABLE_BITS;
|
||||
const table = new Array(TABLE_SIZE);
|
||||
let e = 0;
|
||||
for (let l = 1; l <= TABLE_BITS; l++) {
|
||||
for (let s = 0; s < 512; s++) {
|
||||
if (lens[s] === l) {
|
||||
const n = 1 << (TABLE_BITS - l);
|
||||
for (let k = 0; k < n; k++)
|
||||
table[e++] = s;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (e !== TABLE_SIZE)
|
||||
throw new OperationError("XPRESS: invalid Huffman code lengths");
|
||||
|
||||
// Preload two LE16 words, most-significant bit first.
|
||||
let bits = 0;
|
||||
let nbits = 0;
|
||||
let i = 256;
|
||||
while (nbits < 32) {
|
||||
if (input.length - i < 2)
|
||||
throw new OperationError("XPRESS: truncated bit stream");
|
||||
bits = ((bits >>> 0) | (input[i] | (input[i + 1] << 8)) << (16 - nbits)) >>> 0;
|
||||
i += 2;
|
||||
nbits += 16;
|
||||
}
|
||||
|
||||
const out = [];
|
||||
while (out.length < decompressedSize) {
|
||||
while (nbits < 15) {
|
||||
if (input.length - i < 2)
|
||||
throw new OperationError("XPRESS: truncated bit stream");
|
||||
bits = ((bits >>> 0) | (input[i] | (input[i + 1] << 8)) << (16 - nbits)) >>> 0;
|
||||
i += 2;
|
||||
nbits += 16;
|
||||
}
|
||||
const sym = table[(bits >>> 17) & 0x7fff];
|
||||
const clen = lens[sym];
|
||||
bits = (bits >>> 0) << clen;
|
||||
nbits -= clen;
|
||||
|
||||
if (sym < 256) {
|
||||
out.push(sym);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sym === 256) {
|
||||
// End of data; mid-stream it decodes as a match(3, 1).
|
||||
if (out.length === decompressedSize)
|
||||
break;
|
||||
if (decompressedSize - out.length < 3)
|
||||
throw new OperationError("XPRESS: corrupt end-of-data marker");
|
||||
const start = out.length - 1;
|
||||
for (let j = 0; j < 3; j++)
|
||||
out.push(out[start + j]);
|
||||
continue;
|
||||
}
|
||||
|
||||
const hb = (sym - 256) >>> 4;
|
||||
let mlen = (sym - 256) & 15;
|
||||
if (mlen === 15) {
|
||||
let v = 0;
|
||||
if (i >= input.length)
|
||||
throw new OperationError("XPRESS: truncated raw length");
|
||||
v = input[i++];
|
||||
if (v === 255) {
|
||||
if (input.length - i < 2)
|
||||
throw new OperationError("XPRESS: truncated raw length");
|
||||
v = input[i] | (input[i + 1] << 8);
|
||||
i += 2;
|
||||
if (v === 0) {
|
||||
if (input.length - i < 4)
|
||||
throw new OperationError("XPRESS: truncated raw length");
|
||||
v = (input[i] | (input[i + 1] << 8) |
|
||||
(input[i + 2] << 16) | (input[i + 3] << 24)) >>> 0;
|
||||
i += 4;
|
||||
}
|
||||
mlen = v + 3;
|
||||
} else {
|
||||
mlen = v + 18;
|
||||
}
|
||||
} else {
|
||||
mlen += 3;
|
||||
}
|
||||
|
||||
while (nbits < hb) {
|
||||
if (input.length - i < 2)
|
||||
throw new OperationError("XPRESS: truncated bit stream");
|
||||
bits = ((bits >>> 0) | (input[i] | (input[i + 1] << 8)) << (16 - nbits)) >>> 0;
|
||||
i += 2;
|
||||
nbits += 16;
|
||||
}
|
||||
let moff = 0;
|
||||
if (hb > 0) {
|
||||
moff = (bits >>> (32 - hb)) & ((1 << hb) - 1);
|
||||
bits = (bits >>> 0) << hb;
|
||||
nbits -= hb;
|
||||
}
|
||||
moff += 1 << hb;
|
||||
|
||||
if (moff > out.length)
|
||||
throw new OperationError("XPRESS: match offset out of range");
|
||||
if (out.length + mlen > MAX_DECOMPRESSED)
|
||||
throw new OperationError("XPRESS: decompression ratio too large");
|
||||
|
||||
const start = out.length - moff;
|
||||
for (let j = 0; j < mlen; j++)
|
||||
out.push(out[start + j]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
41
src/core/operations/XPRESSDecompress.mjs
Normal file
41
src/core/operations/XPRESSDecompress.mjs
Normal file
@ -0,0 +1,41 @@
|
||||
/**
|
||||
* @author MP Gowtham [mpgowtham@users.noreply.github.com]
|
||||
* @copyright Crown Copyright 2026
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation.mjs";
|
||||
import {decompress} from "../lib/XPRESS.mjs";
|
||||
|
||||
/**
|
||||
* XPRESS Decompress operation
|
||||
*/
|
||||
class XPRESSDecompress extends Operation {
|
||||
|
||||
/**
|
||||
* XPRESS Decompress constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "XPRESS Decompress";
|
||||
this.module = "Compression";
|
||||
this.description = "Decompresses data using the XPRESS plain LZ77 algorithm (MS-XCA section 2.1).<br><br>Similar to the Windows API <code>RtlDecompressBuffer</code> with <code>COMPRESSION_FORMAT_XPRESS</code>.";
|
||||
this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15";
|
||||
this.inputType = "byteArray";
|
||||
this.outputType = "byteArray";
|
||||
this.args = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {byteArray} input
|
||||
* @param {Object[]} args
|
||||
* @returns {byteArray}
|
||||
*/
|
||||
run(input, args) {
|
||||
return decompress(input);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default XPRESSDecompress;
|
||||
48
src/core/operations/XPRESSHuffmanDecompress.mjs
Normal file
48
src/core/operations/XPRESSHuffmanDecompress.mjs
Normal file
@ -0,0 +1,48 @@
|
||||
/**
|
||||
* @author MP Gowtham [mpgowtham@users.noreply.github.com]
|
||||
* @copyright Crown Copyright 2026
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation.mjs";
|
||||
import {decompressHuffman} from "../lib/XPRESS.mjs";
|
||||
|
||||
/**
|
||||
* XPRESS LZ77+Huffman Decompress operation
|
||||
*/
|
||||
class XPRESSHuffmanDecompress extends Operation {
|
||||
|
||||
/**
|
||||
* XPRESS LZ77+Huffman Decompress constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "XPRESS LZ77+Huffman Decompress";
|
||||
this.module = "Compression";
|
||||
this.description = "Decompresses data using the XPRESS LZ77+Huffman algorithm (MS-XCA section 2.2).<br><br>The uncompressed size must be known in advance, as it is from the WOF chunk table or WIM header, so it is taken as an argument.";
|
||||
this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15";
|
||||
this.inputType = "byteArray";
|
||||
this.outputType = "byteArray";
|
||||
this.args = [
|
||||
{
|
||||
"name": "Decompressed size",
|
||||
"type": "number",
|
||||
"value": 4096
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {byteArray} input
|
||||
* @param {Object[]} args
|
||||
* @returns {byteArray}
|
||||
*/
|
||||
run(input, args) {
|
||||
const size = args[0];
|
||||
return decompressHuffman(input, size);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default XPRESSHuffmanDecompress;
|
||||
79
tests/operations/tests/XPRESS.mjs
Normal file
79
tests/operations/tests/XPRESS.mjs
Normal file
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* XPRESS tests.
|
||||
*
|
||||
* @author MP Gowtham [mpgowtham@users.noreply.github.com]
|
||||
* @copyright Crown Copyright 2026
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
import TestRegister from "../../lib/TestRegister.mjs";
|
||||
|
||||
TestRegister.addTests([
|
||||
|
||||
// MS-XCA section 3.1 worked example (plain LZ77).
|
||||
{
|
||||
name: "XPRESS Decompress: worked example",
|
||||
input: "0000000047484f53542f2f5245434f5645522064617461207265636f7665727920656e67ffffff07696e652e0a",
|
||||
expectedOutput: "GHOST//RECOVER data recovery engine.\n",
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Space"]
|
||||
},
|
||||
{
|
||||
"op": "XPRESS Decompress",
|
||||
"args": []
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
// Matches with shared nibbles, raw lengths and 64 MiB of output.
|
||||
{
|
||||
name: "XPRESS Decompress: repeated string compression",
|
||||
input: "ffffff1f61626317000fff2601",
|
||||
expectedOutput: "abc".repeat(100),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Space"]
|
||||
},
|
||||
{
|
||||
"op": "XPRESS Decompress",
|
||||
"args": []
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
// Literals plus 65536-byte offset matches (anchor example).
|
||||
{
|
||||
name: "XPRESS Decompress: anchor example",
|
||||
input: "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 30 23 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 a8 dc 00 00 ff 26 01",
|
||||
expectedOutput: "abc".repeat(100),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Space"]
|
||||
},
|
||||
{
|
||||
"op": "XPRESS LZ77+Huffman Decompress",
|
||||
"args": [300]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
// MS-XCA section 3.1 worked example (LZ77+Huffman), ten repetitions.
|
||||
{
|
||||
name: "XPRESS LZ77+Huffman Decompress: worked example",
|
||||
input: "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 50 66 55 55 66 65 55 45 65 55 55 65 55 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 b4 e3 a9 8f 5e e7 62 8e bc 5f ac 28 47 19 40 42 98 aa eb 89 7c da 20 5c 61 96 e4 b6 ff 38 01 00 00",
|
||||
expectedOutput: "The quick brown fox jumps over the lazy dog. ".repeat(8),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Space"]
|
||||
},
|
||||
{
|
||||
"op": "XPRESS LZ77+Huffman Decompress",
|
||||
"args": [360]
|
||||
}
|
||||
]
|
||||
}
|
||||
]);
|
||||
Loading…
x
Reference in New Issue
Block a user