Merge 95313d77ae307475d4447e74b08e29925c13efa6 into 4290ea753912378913b1f3f54e0fc5720afeda5d

src/core/config/Categories.json

@@ -68,6 +68,8 @@
             "Swap endianness",
             "To MessagePack",
             "From MessagePack",
+            "To Python Marshal",
+            "From Python Marshal",
             "To Braille",
             "From Braille",
             "Parse TLV",

src/core/lib/PythonMarshal.mjs

@@ -0,0 +1,505 @@
+/**
+ * @author GCHQ
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+
+const FLAG_REF = 0x80;
+const MAX_ITEMS = 1000000;
+const MAX_DEPTH = 1000;
+const NULL = Symbol("marshal null");
+
+const TYPE = {
+    NULL: 0x30,
+    NONE: 0x4e,
+    FALSE: 0x46,
+    TRUE: 0x54,
+    STOPITER: 0x53,
+    ELLIPSIS: 0x2e,
+    INT: 0x69,
+    INT64: 0x49,
+    FLOAT: 0x66,
+    BINARY_FLOAT: 0x67,
+    COMPLEX: 0x78,
+    BINARY_COMPLEX: 0x79,
+    LONG: 0x6c,
+    STRING: 0x73,
+    INTERNED: 0x74,
+    STRINGREF: 0x52,
+    TUPLE: 0x28,
+    SMALL_TUPLE: 0x29,
+    LIST: 0x5b,
+    DICT: 0x7b,
+    CODE: 0x63,
+    UNICODE: 0x75,
+    SET: 0x3c,
+    FROZENSET: 0x3e,
+    ASCII: 0x61,
+    ASCII_INTERNED: 0x41,
+    SHORT_ASCII: 0x7a,
+    SHORT_ASCII_INTERNED: 0x5a,
+    REF: 0x72,
+};
+
+const textDecoder = new TextDecoder("utf-8", { fatal: true });
+const textEncoder = new TextEncoder();
+
+/**
+ * Python marshal format codec for values expressible in JSON.
+ */
+class PythonMarshal {
+
+    /**
+     * @param {ArrayBuffer} input
+     * @returns {Object|Array|string|number|boolean|null}
+     */
+    static decode(input) {
+        const reader = new Reader(new Uint8Array(input));
+        if (!reader.bytes.length) throw new OperationError("Python marshal input is empty");
+
+        const value = reader.readValue(0);
+        if (value === NULL) throw new OperationError("Unexpected null marker in Python marshal input");
+        try {
+            JSON.stringify(value);
+        } catch (err) {
+            throw new OperationError("Python marshal data contains a cyclic value, which is not representable as JSON");
+        }
+        return value;
+    }
+
+    /**
+     * @param {Object|Array|string|number|boolean|null} value
+     * @returns {ArrayBuffer}
+     */
+    static encode(value) {
+        const writer = new Writer();
+        writer.writeValue(value, 0);
+        return writer.finish();
+    }
+
+}
+
+class Reader {
+
+    /**
+     * @param {Uint8Array} bytes
+     */
+    constructor(bytes) {
+        this.bytes = bytes;
+        this.position = 0;
+        this.refs = [];
+    }
+
+    /**
+     * @param {number} length
+     * @returns {Uint8Array}
+     */
+    readBytes(length) {
+        if (!Number.isSafeInteger(length) || length < 0 || length > this.bytes.length - this.position) {
+            throw new OperationError("Unexpected end of Python marshal input");
+        }
+        const start = this.position;
+        this.position += length;
+        return this.bytes.subarray(start, this.position);
+    }
+
+    /**
+     * @returns {number}
+     */
+    readUint8() {
+        return this.readBytes(1)[0];
+    }
+
+    /**
+     * @returns {number}
+     */
+    readInt32() {
+        const bytes = this.readBytes(4);
+        return new DataView(bytes.buffer, bytes.byteOffset, 4).getInt32(0, true);
+    }
+
+    /**
+     * @returns {number}
+     */
+    readUint32() {
+        const bytes = this.readBytes(4);
+        return new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0, true);
+    }
+
+    /**
+     * @returns {number}
+     */
+    readFloat64() {
+        const bytes = this.readBytes(8);
+        return new DataView(bytes.buffer, bytes.byteOffset, 8).getFloat64(0, true);
+    }
+
+    /**
+     * @param {Uint8Array} bytes
+     * @returns {string}
+     */
+    decodeText(bytes) {
+        try {
+            return textDecoder.decode(bytes);
+        } catch (err) {
+            throw new OperationError("Python marshal string is not valid UTF-8");
+        }
+    }
+
+    /**
+     * @param {number} depth
+     * @returns {Object|Array|string|number|boolean|null|symbol}
+     */
+    readValue(depth) {
+        if (depth > MAX_DEPTH) throw new OperationError("Python marshal data is nested too deeply");
+
+        const typeWithFlags = this.readUint8();
+        const type = typeWithFlags & ~FLAG_REF;
+        const shouldStoreReference = (typeWithFlags & FLAG_REF) !== 0;
+        const referenceIndex = shouldStoreReference ? this.refs.length : -1;
+        let value;
+
+        if (shouldStoreReference && (type === TYPE.LIST || type === TYPE.TUPLE || type === TYPE.SMALL_TUPLE || type === TYPE.SET || type === TYPE.FROZENSET)) {
+            value = [];
+            this.refs.push(value);
+            this.readSequence(value, type === TYPE.SMALL_TUPLE ? this.readUint8() : this.readUint32(), depth);
+            return value;
+        }
+        if (shouldStoreReference && type === TYPE.DICT) {
+            value = Object.create(null);
+            this.refs.push(value);
+            this.readDict(value, depth);
+            return value;
+        }
+
+        switch (type) {
+            case TYPE.NULL:
+                value = NULL;
+                break;
+            case TYPE.NONE:
+                value = null;
+                break;
+            case TYPE.FALSE:
+                value = false;
+                break;
+            case TYPE.TRUE:
+                value = true;
+                break;
+            case TYPE.INT:
+                value = this.readInt32();
+                break;
+            case TYPE.INT64:
+                value = this.readInteger64();
+                break;
+            case TYPE.LONG:
+                value = this.readLong();
+                break;
+            case TYPE.FLOAT:
+                value = Number(this.decodeText(this.readBytes(this.readUint8())));
+                break;
+            case TYPE.BINARY_FLOAT:
+                value = this.readFloat64();
+                break;
+            case TYPE.STRING:
+                value = this.readByteString(this.readUint32());
+                break;
+            case TYPE.INTERNED:
+            case TYPE.UNICODE:
+            case TYPE.ASCII:
+            case TYPE.ASCII_INTERNED:
+                value = this.decodeText(this.readBytes(this.readUint32()));
+                break;
+            case TYPE.SHORT_ASCII:
+            case TYPE.SHORT_ASCII_INTERNED:
+                value = this.decodeText(this.readBytes(this.readUint8()));
+                break;
+            case TYPE.STRINGREF:
+            case TYPE.REF:
+                value = this.readReference();
+                break;
+            case TYPE.LIST:
+            case TYPE.TUPLE:
+            case TYPE.SET:
+            case TYPE.FROZENSET:
+                value = [];
+                if (shouldStoreReference) this.refs[referenceIndex] = value;
+                this.readSequence(value, this.readUint32(), depth);
+                break;
+            case TYPE.SMALL_TUPLE:
+                value = [];
+                if (shouldStoreReference) this.refs[referenceIndex] = value;
+                this.readSequence(value, this.readUint8(), depth);
+                break;
+            case TYPE.DICT:
+                value = Object.create(null);
+                if (shouldStoreReference) this.refs[referenceIndex] = value;
+                this.readDict(value, depth);
+                break;
+            case TYPE.STOPITER:
+            case TYPE.ELLIPSIS:
+            case TYPE.COMPLEX:
+            case TYPE.BINARY_COMPLEX:
+            case TYPE.CODE:
+                throw new OperationError(`Python marshal type '${String.fromCharCode(type)}' is not representable as JSON`);
+            default:
+                throw new OperationError(`Unsupported Python marshal type 0x${type.toString(16).padStart(2, "0")}`);
+        }
+
+        if (shouldStoreReference) this.refs[referenceIndex] = value;
+        if (typeof value === "number" && !Number.isFinite(value)) {
+            throw new OperationError("Non-finite Python marshal numbers are not representable as JSON");
+        }
+        return value;
+    }
+
+    /**
+     * @returns {number|Object}
+     */
+    readInteger64() {
+        const bytes = this.readBytes(8);
+        let value = 0n;
+        for (let i = 7; i >= 0; i--) value = (value << 8n) | BigInt(bytes[i]);
+        if (value >= (1n << 63n)) value -= 1n << 64n;
+        return this.toJsonInteger(value);
+    }
+
+    /**
+     * @returns {number|Object}
+     */
+    readLong() {
+        const size = this.readInt32();
+        const digitCount = Math.abs(size);
+        if (digitCount > MAX_ITEMS) throw new OperationError("Python marshal integer has too many digits");
+
+        let value = 0n;
+        for (let i = 0; i < digitCount; i++) {
+            const bytes = this.readBytes(2);
+            const digit = BigInt(bytes[0] | (bytes[1] << 8));
+            value |= digit << BigInt(i * 15);
+        }
+        return this.toJsonInteger(size < 0 ? -value : value);
+    }
+
+    /**
+     * @param {bigint} value
+     * @returns {number|Object}
+     */
+    toJsonInteger(value) {
+        if (value >= BigInt(Number.MIN_SAFE_INTEGER) && value <= BigInt(Number.MAX_SAFE_INTEGER)) return Number(value);
+        return { _pythonMarshalType: "int", value: value.toString() };
+    }
+
+    /**
+     * @param {number} length
+     * @returns {Object}
+     */
+    readByteString(length) {
+        return {
+            _pythonMarshalType: "bytes",
+            hex: Array.from(this.readBytes(length), byte => byte.toString(16).padStart(2, "0")).join(""),
+        };
+    }
+
+    /**
+     * @returns {Object|Array|string|number|boolean|null}
+     */
+    readReference() {
+        const reference = this.readUint32();
+        if (reference >= this.refs.length) throw new OperationError("Python marshal data contains an invalid reference");
+        return this.refs[reference];
+    }
+
+    /**
+     * @param {Array} value
+     * @param {number} length
+     * @param {number} depth
+     */
+    readSequence(value, length, depth) {
+        if (length > MAX_ITEMS) throw new OperationError("Python marshal collection has too many items");
+        for (let i = 0; i < length; i++) value.push(this.readValue(depth + 1));
+    }
+
+    /**
+     * @param {Object} value
+     * @param {number} depth
+     */
+    readDict(value, depth) {
+        for (let i = 0; i < MAX_ITEMS; i++) {
+            const key = this.readValue(depth + 1);
+            if (key === NULL) return;
+            if (typeof key !== "string") throw new OperationError("Python marshal dictionaries with non-string keys are not representable as JSON");
+            value[key] = this.readValue(depth + 1);
+        }
+        throw new OperationError("Python marshal dictionary has too many items");
+    }
+
+}
+
+class Writer {
+
+    constructor() {
+        this.parts = [];
+    }
+
+    /**
+     * @returns {ArrayBuffer}
+     */
+    finish() {
+        const length = this.parts.reduce((total, part) => total + part.length, 0);
+        const result = new Uint8Array(length);
+        let offset = 0;
+        for (const part of this.parts) {
+            result.set(part, offset);
+            offset += part.length;
+        }
+        return result.buffer;
+    }
+
+    /**
+     * @param {number} type
+     */
+    writeType(type) {
+        this.parts.push(Uint8Array.of(type));
+    }
+
+    /**
+     * @param {number} value
+     */
+    writeInt32(value) {
+        const bytes = new Uint8Array(4);
+        new DataView(bytes.buffer).setInt32(0, value, true);
+        this.parts.push(bytes);
+    }
+
+    /**
+     * @param {number} value
+     */
+    writeUint32(value) {
+        const bytes = new Uint8Array(4);
+        new DataView(bytes.buffer).setUint32(0, value, true);
+        this.parts.push(bytes);
+    }
+
+    /**
+     * @param {number} value
+     */
+    writeFloat64(value) {
+        const bytes = new Uint8Array(8);
+        new DataView(bytes.buffer).setFloat64(0, value, true);
+        this.parts.push(bytes);
+    }
+
+    /**
+     * @param {Object|Array|string|number|boolean|null} value
+     * @param {number} depth
+     */
+    writeValue(value, depth) {
+        if (depth > MAX_DEPTH) throw new OperationError("JSON input is nested too deeply");
+
+        if (value === null) {
+            this.writeType(TYPE.NONE);
+        } else if (typeof value === "boolean") {
+            this.writeType(value ? TYPE.TRUE : TYPE.FALSE);
+        } else if (typeof value === "string") {
+            this.writeText(value);
+        } else if (typeof value === "number") {
+            this.writeNumber(value);
+        } else if (Array.isArray(value)) {
+            if (value.length > MAX_ITEMS) throw new OperationError("JSON array has too many items");
+            this.writeType(TYPE.LIST);
+            this.writeUint32(value.length);
+            for (const item of value) this.writeValue(item, depth + 1);
+        } else if (typeof value === "object") {
+            if (value._pythonMarshalType === "bytes") {
+                this.writeBytes(value);
+            } else if (value._pythonMarshalType === "int") {
+                this.writeLong(value);
+            } else {
+                const keys = Object.keys(value);
+                if (keys.length > MAX_ITEMS) throw new OperationError("JSON object has too many properties");
+                this.writeType(TYPE.DICT);
+                for (const key of keys) {
+                    this.writeText(key);
+                    this.writeValue(value[key], depth + 1);
+                }
+                this.writeType(TYPE.NULL);
+            }
+        } else {
+            throw new OperationError(`Cannot encode JSON value of type '${typeof value}' as Python marshal`);
+        }
+    }
+
+    /**
+     * @param {string} value
+     */
+    writeText(value) {
+        const bytes = textEncoder.encode(value);
+        this.writeType(TYPE.UNICODE);
+        this.writeUint32(bytes.length);
+        this.parts.push(bytes);
+    }
+
+    /**
+     * @param {number} value
+     */
+    writeNumber(value) {
+        if (!Number.isFinite(value)) throw new OperationError("JSON numbers must be finite");
+        if (Number.isSafeInteger(value)) {
+            if (value >= -0x80000000 && value <= 0x7fffffff) {
+                this.writeType(TYPE.INT);
+                this.writeInt32(value);
+            } else {
+                this.writeBigInt(BigInt(value));
+            }
+        } else {
+            this.writeType(TYPE.BINARY_FLOAT);
+            this.writeFloat64(value);
+        }
+    }
+
+    /**
+     * @param {Object} value
+     */
+    writeBytes(value) {
+        if (typeof value.hex !== "string" || !/^(?:[0-9a-f]{2})*$/i.test(value.hex)) {
+            throw new OperationError("Python marshal bytes values require an even-length hexadecimal 'hex' string");
+        }
+        const bytes = new Uint8Array(value.hex.length / 2);
+        for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(value.hex.slice(i * 2, i * 2 + 2), 16);
+        this.writeType(TYPE.STRING);
+        this.writeUint32(bytes.length);
+        this.parts.push(bytes);
+    }
+
+    /**
+     * @param {Object} value
+     */
+    writeLong(value) {
+        if (typeof value.value !== "string" || !/^-?\d+$/.test(value.value)) {
+            throw new OperationError("Python marshal integer values require a decimal string 'value'");
+        }
+        this.writeBigInt(BigInt(value.value));
+    }
+
+    /**
+     * @param {bigint} value
+     */
+    writeBigInt(value) {
+        let integer = value;
+        const negative = integer < 0n;
+        if (negative) integer = -integer;
+        const digits = [];
+        while (integer) {
+            digits.push(Number(integer & 0x7fffn));
+            integer >>= 15n;
+        }
+        this.writeType(TYPE.LONG);
+        this.writeInt32(negative ? -digits.length : digits.length);
+        for (const digit of digits) this.parts.push(Uint8Array.of(digit & 0xff, digit >> 8));
+    }
+
+}
+
+export default PythonMarshal;

src/core/operations/FromPythonMarshal.mjs

@@ -0,0 +1,41 @@
+/**
+ * @author GCHQ
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import PythonMarshal from "../lib/PythonMarshal.mjs";
+
+/**
+ * From Python Marshal operation
+ */
+class FromPythonMarshal extends Operation {
+
+    /**
+     * FromPythonMarshal constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Python Marshal";
+        this.module = "Code";
+        this.description = "Decodes CPython marshal data to JSON-compatible values. Supports null, booleans, numbers, strings, byte strings, lists, tuples, sets and dictionaries with string keys. Byte strings are represented as <code>{&quot;_pythonMarshalType&quot;:&quot;bytes&quot;,&quot;hex&quot;:&quot;...&quot;}</code>; integers outside JavaScript's safe range use <code>{&quot;_pythonMarshalType&quot;:&quot;int&quot;,&quot;value&quot;:&quot;...&quot;}</code>. Code objects and other Python-only values are not supported.";
+        this.infoURL = "https://docs.python.org/3/library/marshal.html";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "JSON";
+        this.args = [];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        return PythonMarshal.decode(input);
+    }
+
+}
+
+export default FromPythonMarshal;

src/core/operations/ToPythonMarshal.mjs

@@ -0,0 +1,41 @@
+/**
+ * @author GCHQ
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import PythonMarshal from "../lib/PythonMarshal.mjs";
+
+/**
+ * To Python Marshal operation
+ */
+class ToPythonMarshal extends Operation {
+
+    /**
+     * ToPythonMarshal constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "To Python Marshal";
+        this.module = "Code";
+        this.description = "Encodes JSON-compatible values using CPython marshal format 4. Use <code>{&quot;_pythonMarshalType&quot;:&quot;bytes&quot;,&quot;hex&quot;:&quot;...&quot;}</code> for Python byte strings and <code>{&quot;_pythonMarshalType&quot;:&quot;int&quot;,&quot;value&quot;:&quot;...&quot;}</code> for arbitrary-size integers. Arrays encode as Python lists; objects encode as dictionaries with string keys.";
+        this.infoURL = "https://docs.python.org/3/library/marshal.html";
+        this.inputType = "JSON";
+        this.outputType = "ArrayBuffer";
+        this.args = [];
+    }
+
+    /**
+     * @param {JSON} input
+     * @param {Object[]} args
+     * @returns {ArrayBuffer}
+     */
+    run(input, args) {
+        return PythonMarshal.encode(input);
+    }
+
+}
+
+export default ToPythonMarshal;

tests/operations/tests/PythonMarshal.mjs

@@ -0,0 +1,103 @@
+/**
+ * Python marshal operation tests.
+ *
+ * @author GCHQ
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+/*
+ * The decode fixtures below were generated with CPython 3.14.4. Reproduce them
+ * with the following script (the explicit version keeps the fixtures stable as
+ * CPython's default marshal format evolves):
+ *
+ * import marshal
+ *
+ * for value in [
+ *     {"hello": "world", "data": b"\x00\xff"},
+ *     [1, "two"],
+ * ]:
+ *     print(repr(marshal.dumps(value, 4)))
+ *
+ * all_types = {
+ *     "none": None, "false": False, "true": True, "int32": -42,
+ *     "integer": 9007199254740992, "float": 3.5, "bytes": b"\x00\xff",
+ *     "text": "Γειά", "list": [1, "two"], "tuple": (1, "two"),
+ *     "set": {1, 2}, "dictionary": {"nested": "value"},
+ * }
+ * print(marshal.dumps(all_types, 4).hex())
+ *
+ * The encoder fixture is validated by CPython with:
+ *
+ * assert marshal.loads(bytes.fromhex(
+ *     "7b750500000068656c6c6f7505000000776f726c64750400000064617461730200000000ff30"
+ * )) == {"hello": "world", "data": b"\x00\xff"}
+ *
+ * The test suite deliberately embeds these CPython-generated fixtures instead
+ * of invoking Python, so `npm test` remains a Node.js-only command.
+ */
+const hexToString = (hex) => hex.match(/../g).map((byte) =>
+    String.fromCharCode(parseInt(byte, 16))).join("");
+
+TestRegister.addTests([
+    {
+        name: "From Python Marshal: CPython format 4 dictionary",
+        input: "\xfb\xda\x05hello\xda\x05world\xda\x04data\xf3\x02\x00\x00\x00\x00\xff\x30",
+        expectedOutput: "{\n    \"hello\": \"world\",\n    \"data\": {\n        \"_pythonMarshalType\": \"bytes\",\n        \"hex\": \"00ff\"\n    }\n}",
+        recipeConfig: [
+            {
+                op: "From Python Marshal",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "From Python Marshal: CPython format 4 references",
+        input: "\xdb\x02\x00\x00\x00\xe9\x01\x00\x00\x00\xda\x03two",
+        expectedOutput: "[\n    1,\n    \"two\"\n]",
+        recipeConfig: [
+            {
+                op: "From Python Marshal",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "From Python Marshal: documented value types",
+        input: hexToString("fbda046e6f6e654eda0566616c736546da047472756554da05696e743332e9d6ffffffda07696e7465676572ec040000000000000000000001da05666c6f6174e70000000000000c40da056279746573f30200000000ffda0474657874f508000000ce93ceb5ceb9ceacda046c6973745b02000000e901000000da0374776fda057475706c65a902720f0000007210000000da037365743c02000000720f000000e902000000da0a64696374696f6e6172797bda066e6573746564da0576616c75653030"),
+        expectedOutput: "{\n    \"none\": null,\n    \"false\": false,\n    \"true\": true,\n    \"int32\": -42,\n    \"integer\": {\n        \"_pythonMarshalType\": \"int\",\n        \"value\": \"9007199254740992\"\n    },\n    \"float\": 3.5,\n    \"bytes\": {\n        \"_pythonMarshalType\": \"bytes\",\n        \"hex\": \"00ff\"\n    },\n    \"text\": \"Γειά\",\n    \"list\": [\n        1,\n        \"two\"\n    ],\n    \"tuple\": [\n        1,\n        \"two\"\n    ],\n    \"set\": [\n        1,\n        2\n    ],\n    \"dictionary\": {\n        \"nested\": \"value\"\n    }\n}",
+        recipeConfig: [
+            {
+                op: "From Python Marshal",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Python Marshal: CPython format 4 dictionary",
+        input: "{\"hello\":\"world\",\"data\":{\"_pythonMarshalType\":\"bytes\",\"hex\":\"00ff\"}}",
+        expectedOutput: "{u\x05\x00\x00\x00hellou\x05\x00\x00\x00worldu\x04\x00\x00\x00datas\x02\x00\x00\x00\x00\xff0",
+        recipeConfig: [
+            {
+                op: "To Python Marshal",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "Python Marshal: arbitrary-size integer round trip",
+        input: "{\"_pythonMarshalType\":\"int\",\"value\":\"9007199254740992\"}",
+        expectedOutput: "{\n    \"_pythonMarshalType\": \"int\",\n    \"value\": \"9007199254740992\"\n}",
+        recipeConfig: [
+            {
+                op: "To Python Marshal",
+                args: [],
+            },
+            {
+                op: "From Python Marshal",
+                args: [],
+            },
+        ],
+    },
+]);