diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index dd6918dc..d3e7648a 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -49,6 +49,7 @@ "Escape Unicode Characters", "Unescape Unicode Characters", "Normalise Unicode", + "Escape Smart Characters", "To Quoted Printable", "From Quoted Printable", "To Punycode", diff --git a/src/core/operations/EscapeSmartCharacters.mjs b/src/core/operations/EscapeSmartCharacters.mjs new file mode 100644 index 00000000..39444e94 --- /dev/null +++ b/src/core/operations/EscapeSmartCharacters.mjs @@ -0,0 +1,129 @@ +/** + * @author HarelKatz [github.com/HarelKatz] + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; + +/** + * Escape Smart Characters operation + */ +class EscapeSmartCharacters extends Operation { + + /** + * EscapeSmartCharacters constructor + */ + constructor() { + super(); + + this.name = "Escape Smart Characters"; + this.module = "Default"; + this.description = "Converts smart (typographic) Unicode characters — e.g. smart quotes, em/en dashes, ellipses, ©, ®, ™, arrows — into their plain ASCII equivalents.

Characters with no ASCII mapping (e.g. ) are handled according to the 'Unmappable characters' option.

e.g. “Hello” — world… becomes \"Hello\" -- world..."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Unmappable characters", + type: "option", + value: ["Include", "Remove", "Replace with '.'"] + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const [unmappable] = args; + let result = ""; + for (const ch of input) { + if (ch.codePointAt(0) < 128) { + result += ch; + } else if (Object.prototype.hasOwnProperty.call(SMART_MAP, ch)) { + result += SMART_MAP[ch]; + } else { + switch (unmappable) { + case "Remove": + break; + case "Replace with '.'": + result += "."; + break; + case "Include": + default: + result += ch; + break; + } + } + } + return result; + } + +} + +const SMART_MAP = { + // Smart double quotes + "“": "\"", // “ left double quotation mark + "”": "\"", // ” right double quotation mark + "„": "\"", // „ double low-9 quotation mark + "‟": "\"", // ‟ double high-reversed-9 quotation mark + "″": "\"", // ″ double prime + + // Smart single quotes / apostrophes + "‘": "'", // ‘ left single quotation mark + "’": "'", // ’ right single quotation mark / apostrophe + "‚": "'", // ‚ single low-9 quotation mark + "‛": "'", // ‛ single high-reversed-9 quotation mark + "′": "'", // ′ prime + + // Dashes & hyphens + "‐": "-", // ‐ hyphen + "‑": "-", // ‑ non-breaking hyphen + "‒": "-", // ‒ figure dash + "–": "-", // – en dash + "—": "--", // — em dash + "―": "--", // ― horizontal bar + + // Ellipsis + "…": "...", // … + + // Trademark / copyright symbols + "©": "(c)", // © + "®": "(r)", // ® + "™": "(tm)", // ™ + + // Arrows + "←": "<--", // ← + "→": "-->", // → + "↑": "^", // ↑ + "↓": "v", // ↓ + "↔": "<->", // ↔ + "⇐": "<==", // ⇐ + "⇒": "==>", // ⇒ + "⇔": "<=>", // ⇔ + + // Guillemets + "«": "<<", // « + "»": ">>", // » + "‹": "<", // ‹ + "›": ">", // › + + // Math & misc symbols + "×": "x", // × + "÷": "/", // ÷ + "±": "+/-", // ± + "•": "*", // • + "·": ".", // · + + // Non-ASCII spaces + "\u00A0": " ", // NBSP + "\u2002": " ", // en space + "\u2003": " ", // em space + "\u2009": " ", // thin space + "\u200A": " " // hair space +}; + +export default EscapeSmartCharacters; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index c44270fd..44792560 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -69,6 +69,7 @@ import "./tests/DropNthBytes.mjs"; import "./tests/ECDSA.mjs"; import "./tests/ELFInfo.mjs"; import "./tests/Enigma.mjs"; +import "./tests/EscapeSmartCharacters.mjs"; import "./tests/ExtractAudioMetadata.mjs"; import "./tests/ExtractEmailAddresses.mjs"; import "./tests/ExtractHashes.mjs"; diff --git a/tests/operations/tests/EscapeSmartCharacters.mjs b/tests/operations/tests/EscapeSmartCharacters.mjs new file mode 100644 index 00000000..428c3ad8 --- /dev/null +++ b/tests/operations/tests/EscapeSmartCharacters.mjs @@ -0,0 +1,132 @@ +/** + * Escape Smart Characters tests + * + * @author HarelKatz [github.com/HarelKatz] + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + "name": "Escape Smart Characters: smart quotes and apostrophes", + "input": "“Hello,” she said, ‘yes.’", + "expectedOutput": "\"Hello,\" she said, 'yes.'", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: em dash, en dash and ellipsis", + "input": "page 1–3 — wait…", + "expectedOutput": "page 1-3 -- wait...", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: trademark symbols", + "input": "Foo© Bar® Baz™", + "expectedOutput": "Foo(c) Bar(r) Baz(tm)", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: arrows and guillemets", + "input": "← → ↔ ⇒ « »", + "expectedOutput": "<-- --> <-> ==> << >>", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: math and misc", + "input": "3 × 4 ÷ 2 = 6, ±0.5 • item", + "expectedOutput": "3 x 4 / 2 = 6, +/-0.5 * item", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: NBSP becomes regular space", + "input": "a b c", + "expectedOutput": "a b c", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: unmappable Include preserves char", + "input": "warning: ☣ hazard", + "expectedOutput": "warning: ☣ hazard", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: unmappable Remove drops char", + "input": "warning: ☣ hazard", + "expectedOutput": "warning: hazard", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Remove"] + } + ] + }, + { + "name": "Escape Smart Characters: unmappable Replace substitutes dot", + "input": "warning: ☣ hazard", + "expectedOutput": "warning: . hazard", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Replace with '.'"] + } + ] + }, + { + "name": "Escape Smart Characters: pure ASCII passes through", + "input": "hello world! 123", + "expectedOutput": "hello world! 123", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + }, + { + "name": "Escape Smart Characters: empty input", + "input": "", + "expectedOutput": "", + "recipeConfig": [ + { + "op": "Escape Smart Characters", + "args": ["Include"] + } + ] + } +]);