Consolidate HTML entity tables into a single spec-generated source (#2645) (#2671)

.gitignore

@@ -8,6 +8,7 @@ build
 src/core/config/modules/*
 src/core/config/OperationConfig.json
 src/core/operations/index.mjs
+src/core/lib/HTMLEntities.mjs
 src/node/config/OperationConfig.json
 src/node/index.mjs
 tests/operations/index.mjs

Gruntfile.js

@@ -367,6 +367,7 @@ module.exports = function (grunt) {
                 command: chainCommands([
                     "echo '\n--- Regenerating config files. ---'",
                     "echo [] > src/core/config/OperationConfig.json",
+                    `node ${nodeFlags} src/core/config/scripts/generateHTMLEntities.mjs`,
                     `node ${nodeFlags} src/core/config/scripts/generateOpsIndex.mjs`,
                     `node ${nodeFlags} src/core/config/scripts/generateConfig.mjs`,
                     "echo '--- Config scripts finished. ---\n'"

src/core/config/scripts/generateHTMLEntities.mjs

@@ -0,0 +1,139 @@
+/**
+ * This script automatically generates src/core/lib/HTMLEntities.mjs, the shared
+ * HTML entity lookup tables used by the "To HTML Entity" and "From HTML Entity"
+ * operations.
+ *
+ * The data is derived from the vendored WHATWG named character reference set
+ * (src/core/vendor/htmlEntities/entity.json, from
+ * https://html.spec.whatwg.org/entities.json) so the
+ * two operations cannot drift apart and every entity is spec-conformant:
+ *
+ *   - HTML_ENTITY_REVERSE_LOOKUP (decode) is every single-code-point spec name.
+ *   - HTML_ENTITY_LOOKUP (encode) picks one canonical name per code point via a
+ *     deterministic tiebreak, overridden by htmlEntityOverrides.mjs where a
+ *     specific historical name is preferred.
+ *
+ * @author roberson-io [michaelroberson@gmail.com]
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+
+/* eslint no-console: ["off"] */
+
+import path from "path";
+import fs from "fs";
+import process from "process";
+import { fileURLToPath } from "url";
+import { HTML_ENTITY_CANONICAL_OVERRIDES } from "./htmlEntityOverrides.mjs";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+
+if (!fs.existsSync(path.join(process.cwd(), "src/core/lib"))) {
+    console.log("\nCWD: " + process.cwd());
+    console.log("Error: generateHTMLEntities.mjs should be run from the project root");
+    console.log("Example> node src/core/config/scripts/generateHTMLEntities.mjs");
+    process.exit(1);
+}
+
+const SPEC = JSON.parse(fs.readFileSync(
+    path.join(scriptDir, "..", "..", "vendor", "htmlEntities", "entity.json"), "utf8"));
+
+// Build code point -> [spec names] for single-code-point, semicolon-terminated
+// references (the representable subset; multi-code-point entities are skipped).
+const codePointNames = {};
+for (const [key, val] of Object.entries(SPEC)) {
+    if (!key.endsWith(";") || val.codepoints.length !== 1) continue;
+    const name = key.slice(1, -1); // strip leading "&" and trailing ";"
+    (codePointNames[val.codepoints[0]] ??= []).push(name);
+}
+
+// Deterministic canonical-name tiebreak: prefer a lower-case name, then the
+// shortest, then alphabetical. Overridden per code point where a specific
+// historical name is preferred.
+const isAllUpper = s => s === s.toUpperCase() && s !== s.toLowerCase();
+
+/**
+ * Choose the single canonical entity name for a code point.
+ *
+ * @param {number} codePoint - the Unicode code point being encoded
+ * @param {string[]} names - all valid WHATWG names for that code point
+ * @returns {string} the canonical name to emit when encoding
+ */
+function canonicalName(codePoint, names) {
+    const override = HTML_ENTITY_CANONICAL_OVERRIDES[codePoint];
+    if (override !== undefined) {
+        if (!names.includes(override))
+            throw new Error(`Override &${override}; is not a spec name for code point ${codePoint} (spec: ${names})`);
+        return override;
+    }
+    return [...names].sort((a, b) =>
+        (isAllUpper(a) - isAllUpper(b)) ||
+        (a.length - b.length) ||
+        (a < b ? -1 : a > b ? 1 : 0)
+    )[0];
+}
+
+const forward = {};   // code point -> canonical name (encode)
+const reverse = {};   // name -> code point (decode, every spec name)
+for (const [cpStr, names] of Object.entries(codePointNames)) {
+    const cp = Number(cpStr);
+    forward[cp] = canonicalName(cp, names);
+    for (const name of names) reverse[name] = cp;
+}
+
+// Decode-only aliases = spec names that are not the canonical encode name.
+const aliases = {};
+for (const [name, cp] of Object.entries(reverse)) {
+    if (forward[cp] !== name) aliases[name] = cp;
+}
+
+// --- emit -----------------------------------------------------------------
+const fwdEntries = Object.keys(forward).map(Number).sort((a, b) => a - b)
+    .map(cp => `    ${cp}: "${forward[cp]}",`).join("\n").replace(/,$/, "");
+const aliasEntries = Object.keys(aliases).sort()
+    .map(n => `    ${JSON.stringify(n)}: ${aliases[n]},`).join("\n").replace(/,$/, "");
+
+const code = `/**
+ * THIS FILE IS AUTOMATICALLY GENERATED BY src/core/config/scripts/generateHTMLEntities.mjs
+ *
+ * HTML entity lookup tables shared by the "To HTML Entity" and "From HTML Entity"
+ * operations, derived from the WHATWG named character reference set
+ * (https://html.spec.whatwg.org/entities.json). Do not edit by hand — change the
+ * vendored src/core/vendor/htmlEntities/entity.json or htmlEntityOverrides.mjs
+ * and regenerate.
+ *
+ * @author roberson-io [michaelroberson@gmail.com]
+ * @copyright Crown Copyright ${new Date().getUTCFullYear()}
+ * @license Apache-2.0
+ */
+
+/**
+ * Canonical lookup: Unicode code point -> entity name (without "&" and ";"),
+ * used for ENCODING. One canonical name per code point.
+ */
+export const HTML_ENTITY_LOOKUP = {
+${fwdEntries}
+};
+
+/**
+ * Legacy / alias names that only DECODE (spelling variants, deprecated names,
+ * box-drawing aliases, etc.); not used for encoding.
+ */
+export const HTML_ENTITY_DECODE_ALIASES = {
+${aliasEntries}
+};
+
+/**
+ * Derived reverse lookup: entity name -> code point, used for DECODING. Built
+ * from the canonical lookup plus the legacy aliases.
+ */
+export const HTML_ENTITY_REVERSE_LOOKUP = {...HTML_ENTITY_DECODE_ALIASES};
+for (const codePoint in HTML_ENTITY_LOOKUP) {
+    HTML_ENTITY_REVERSE_LOOKUP[HTML_ENTITY_LOOKUP[codePoint]] = Number(codePoint);
+}
+`;
+
+fs.writeFileSync(path.join(process.cwd(), "src/core/lib/HTMLEntities.mjs"), code);
+console.log(`generateHTMLEntities: ${Object.keys(forward).length} encode names, ` +
+    `${Object.keys(reverse).length} decode names, ${Object.keys(aliases).length} aliases, ` +
+    `${Object.keys(HTML_ENTITY_CANONICAL_OVERRIDES).length} overrides applied.`);

src/core/config/scripts/htmlEntityOverrides.mjs

@@ -0,0 +1,86 @@
+/**
+ * Canonical entity-name overrides for generateHTMLEntities.mjs.
+ *
+ * The WHATWG named character reference set assigns MANY names to some code
+ * points (e.g. U+2211 is both ∑ and ∑), but does not designate a
+ * canonical one. The generator therefore needs a rule to pick a single name per
+ * code point for ENCODING. It uses a deterministic tiebreak (prefer a
+ * lower-case name, then the shortest, then alphabetical), which reproduces the
+ * historically-emitted name for ~1355 of the ~1414 encodable code points.
+ *
+ * This file pins the canonical name for the code points where the tiebreak
+ * would otherwise change the emitted entity. Every value here is still a valid
+ * WHATWG name for that code point (the generator asserts this) — these are
+ * editorial choices, not correctness fixes, kept so "To HTML Entity" output
+ * stays stable. Trim an entry to let the tiebreak decide instead.
+ *
+ * @author roberson-io [michaelroberson@gmail.com]
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+
+/**
+ * @constant
+ * @type {Object.<number, string>}
+ */
+export const HTML_ENTITY_CANONICAL_OVERRIDES = {
+    124: "verbar",
+    168: "uml",
+    177: "plusmn",
+    189: "frac12",
+    247: "divide",
+    711: "caron",
+    728: "breve",
+    937: "Omega",
+    949: "epsilon",
+    965: "upsilon",
+    977: "thetasym",
+    978: "upsih",
+    981: "straightphi",
+    8208: "hyphen",
+    8214: "Verbar",
+    8230: "hellip",
+    8289: "ApplyFunction",
+    8290: "InvisibleTimes",
+    8291: "InvisibleComma",
+    8459: "hamilt",
+    8461: "quaternions",
+    8463: "planck",
+    8465: "image",
+    8472: "weierp",
+    8474: "rationals",
+    8476: "real",
+    8477: "reals",
+    8484: "integers",
+    8492: "bernou",
+    8499: "phmmat",
+    8500: "order",
+    8501: "alefsym",
+    8518: "DifferentialD",
+    8519: "ExponentialE",
+    8520: "ImaginaryI",
+    8612: "LeftTeeArrow",
+    8613: "UpTeeArrow",
+    8615: "DownTeeArrow",
+    8624: "lsh",
+    8625: "rsh",
+    8660: "hArr",
+    8704: "forall",
+    8711: "nabla",
+    8712: "isin",
+    8721: "sum",
+    8723: "mnplus",
+    8730: "radic",
+    8750: "conint",
+    8768: "wreath",
+    8776: "asymp",
+    8781: "asympeq",
+    8784: "esdot",
+    8788: "colone",
+    8869: "perp",
+    8896: "xwedge",
+    8897: "xvee",
+    8902: "sstarf",
+    10536: "nesear",
+    10537: "seswar"
+};

src/core/vendor/htmlEntities/entity.txt

@@ -0,0 +1,14 @@
+entity.json is the WHATWG named character reference set, retrieved verbatim from:
+
+    https://html.spec.whatwg.org/entities.json
+
+It is used by src/core/config/scripts/generateHTMLEntities.mjs to generate the
+shared HTML entity lookup tables (src/core/lib/HTMLEntities.mjs) consumed by the
+"To HTML Entity" and "From HTML Entity" operations.
+
+Source: HTML Standard — 13.5 Named character references
+    https://html.spec.whatwg.org/multipage/named-characters.html
+
+Copyright and licence
+---------------------
+Copyright © WHATWG (Apple, Google, Mozilla, Microsoft). This work is licensed under a Creative Commons Attribution 4.0 International License. To the extent portions of it are incorporated into source code, such portions in the source code are licensed under the BSD 3-Clause License instead.

tests/node/tests/ToHTMLEntity.mjs

@@ -1,7 +1,20 @@
 import TestRegister from "../../lib/TestRegister.mjs";
 import ToHTMLEntity from "../../../src/core/operations/ToHTMLEntity.mjs";
+import FromHTMLEntity from "../../../src/core/operations/FromHTMLEntity.mjs";
+import { HTML_ENTITY_LOOKUP, HTML_ENTITY_REVERSE_LOOKUP } from "../../../src/core/lib/HTMLEntities.mjs";
 import it from "../assertionHandler.mjs";
 import assert from "assert";
+import { readFileSync } from "fs";
+import { fileURLToPath } from "url";
+import path from "path";
+
+// Vendored WHATWG named character reference set (https://html.spec.whatwg.org/entities.json).
+const SPEC = JSON.parse(readFileSync(path.join(
+    path.dirname(fileURLToPath(import.meta.url)),
+    "../../../src/core/vendor/htmlEntities/entity.json"), "utf8"));
+const specByName = {};
+for (const [key, val] of Object.entries(SPEC))
+    if (key.endsWith(";")) specByName[key.slice(1, -1)] = val.codepoints;
 
 TestRegister.addApiTests([
     it("To HTML Entity: every named entity in the table is well-formed", () => {
@@ -30,4 +43,40 @@ TestRegister.addApiTests([
         }
         assert.deepStrictEqual(malformed, [], `Malformed entity value(s) near: ${JSON.stringify(malformed)}`);
     }),
+
+    it("HTML Entity: named encoding round-trips through From HTML Entity", () => {
+        // Because both operations share one lookup table, encoding a character to
+        // a named entity and decoding it must return the original character for
+        // every BMP code point: FromHTMLEntity(ToHTMLEntity(x)) === x.
+        const toOp = new ToHTMLEntity(),
+            fromOp = new FromHTMLEntity();
+        const mismatches = [];
+        for (let cp = 0; cp <= 0xFFFF; cp++) {
+            if (cp >= 0xD800 && cp <= 0xDFFF) continue; // skip surrogate range
+            const char = String.fromCodePoint(cp);
+            const encoded = toOp.run(char, [true, "Named entities"]);
+            const decoded = fromOp.run(encoded, []);
+            if (decoded !== char)
+                mismatches.push(`U+${cp.toString(16).toUpperCase().padStart(4, "0")} -> ${encoded} -> U+${decoded.codePointAt(0).toString(16).toUpperCase()}`);
+        }
+        assert.deepStrictEqual(mismatches, [], `Round-trip failed for: ${JSON.stringify(mismatches.slice(0, 20))}`);
+    }),
+
+    it("HTML Entity: every table entry is conformant with the WHATWG spec", () => {
+        // Both lookup tables must agree with entities.json: every encode name is a
+        // real spec name mapping to exactly that code point, and every decode name
+        // maps to the spec code point.
+        const violations = [];
+        for (const [cp, name] of Object.entries(HTML_ENTITY_LOOKUP)) {
+            const spec = specByName[name];
+            if (!spec || spec.length !== 1 || spec[0] !== Number(cp))
+                violations.push(`encode ${cp} -> &${name}; (spec: ${spec ? JSON.stringify(spec) : "none"})`);
+        }
+        for (const [name, cp] of Object.entries(HTML_ENTITY_REVERSE_LOOKUP)) {
+            const spec = specByName[name];
+            if (!spec || spec.length !== 1 || spec[0] !== cp)
+                violations.push(`decode &${name}; -> ${cp} (spec: ${spec ? JSON.stringify(spec) : "none"})`);
+        }
+        assert.deepStrictEqual(violations, [], `Spec violations: ${JSON.stringify(violations.slice(0, 20))}`);
+    }),
 ]);

tests/operations/tests/HTMLEntity.mjs

@@ -0,0 +1,126 @@
+/**
+ * To/From HTML Entity tests.
+ *
+ * @author roberson-io [michaelroberson@gmail.com]
+ * @copyright Crown Copyright 2026
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "To HTML Entity: named",
+        input: "<a href='#'>\"&\"</a>",
+        expectedOutput: "&lt;a href&equals;&apos;&num;&apos;&gt;&quot;&amp;&quot;&lt;&sol;a&gt;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [false, "Named entities"] },
+        ],
+    },
+    {
+        // Regression: this value was "&rlhar;;" (stray trailing semicolon) in the
+        // original byteToEntity table. See issue #2645.
+        name: "To HTML Entity: malformed value fixed (U+21CC)",
+        input: "⇌",
+        expectedOutput: "&rlhar;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [false, "Named entities"] },
+        ],
+    },
+    {
+        // Regression: U+03F5 emitted "&epsi;," before; the spec name is &epsiv;
+        // (&epsi; is U+03B5). See issue #2645.
+        name: "To HTML Entity: spec-conformant name (U+03F5)",
+        input: "ϵ",
+        expectedOutput: "&epsiv;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [false, "Named entities"] },
+        ],
+    },
+    {
+        // Regression: the OHM SIGN previously emitted the non-conformant &ohm;
+        // (spec &ohm; is U+03A9). It now encodes numerically.
+        name: "To HTML Entity: non-conformant name dropped (U+2126 ohm sign)",
+        input: "Ω",
+        expectedOutput: "&#8486;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [false, "Named entities"] },
+        ],
+    },
+    {
+        // The FULL BLOCK previously decoded to &block; but never encoded to it.
+        // Generating both tables from the spec fixes this asymmetry.
+        name: "To HTML Entity: encode/decode symmetry (U+2588 block)",
+        input: "█",
+        expectedOutput: "&block;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [false, "Named entities"] },
+        ],
+    },
+    {
+        name: "To HTML Entity: numeric, convert all",
+        input: "A<",
+        expectedOutput: "&#65;&#60;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [true, "Numeric entities"] },
+        ],
+    },
+    {
+        name: "To HTML Entity: hex, convert all",
+        input: "A<",
+        expectedOutput: "&#x41;&#x3c;",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [true, "Hex entities"] },
+        ],
+    },
+    {
+        name: "From HTML Entity: named",
+        input: "&lt;a href=&apos;#&apos;&gt;&quot;&amp;&quot;&lt;/a&gt;",
+        expectedOutput: "<a href='#'>\"&\"</a>",
+        recipeConfig: [
+            { op: "From HTML Entity", args: [] },
+        ],
+    },
+    {
+        // Legacy/alias names still decode, now to the spec code point.
+        name: "From HTML Entity: alias to spec code point (&ohm;)",
+        input: "&ohm;",
+        expectedOutput: "Ω",
+        recipeConfig: [
+            { op: "From HTML Entity", args: [] },
+        ],
+    },
+    {
+        name: "From HTML Entity: decimal numeric entity",
+        input: "&#65;",
+        expectedOutput: "A",
+        recipeConfig: [
+            { op: "From HTML Entity", args: [] },
+        ],
+    },
+    {
+        name: "From HTML Entity: hex numeric entity",
+        input: "&#x41;",
+        expectedOutput: "A",
+        recipeConfig: [
+            { op: "From HTML Entity", args: [] },
+        ],
+    },
+    {
+        // Unknown entities are passed through unchanged.
+        name: "From HTML Entity: invalid entity passed through",
+        input: "a &notreal; b",
+        expectedOutput: "a &notreal; b",
+        recipeConfig: [
+            { op: "From HTML Entity", args: [] },
+        ],
+    },
+    {
+        name: "HTML Entity: round-trips through both operations",
+        input: "Hello <World> & \"friends\" — ⇌ █",
+        expectedOutput: "Hello <World> & \"friends\" — ⇌ █",
+        recipeConfig: [
+            { op: "To HTML Entity", args: [true, "Named entities"] },
+            { op: "From HTML Entity", args: [] },
+        ],
+    },
+]);