From 5105aad91d9091be64182f1d79678fdbecfcfc15 Mon Sep 17 00:00:00 2001 From: J L <57751845+jl5193@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:36:07 +0100 Subject: [PATCH] New operation improvements (#1431) Co-authored-by: GCHQDeveloper581 <63102987+GCHQDeveloper581@users.noreply.github.com> --- .gitignore | 1 + src/core/config/scripts/generateOpsIndex.mjs | 63 ++++++ src/core/config/scripts/newOperation.mjs | 35 ++- tests/operations/index.mjs | 217 ------------------- tests/operations/tests/FlaskSession.mjs | 19 +- tests/operations/tests/IPv6Transition.mjs | 8 +- tests/operations/tests/ParseX509CRL.mjs | 32 +-- 7 files changed, 126 insertions(+), 249 deletions(-) delete mode 100644 tests/operations/index.mjs diff --git a/.gitignore b/.gitignore index 157558e9..a1e9dab6 100755 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ src/core/config/OperationConfig.json src/core/operations/index.mjs src/node/config/OperationConfig.json src/node/index.mjs +tests/operations/index.mjs **/*.DS_Store tests/browser/output/* .node-version diff --git a/src/core/config/scripts/generateOpsIndex.mjs b/src/core/config/scripts/generateOpsIndex.mjs index d8dd6a70..ccf3da7b 100644 --- a/src/core/config/scripts/generateOpsIndex.mjs +++ b/src/core/config/scripts/generateOpsIndex.mjs @@ -58,3 +58,66 @@ fs.writeFileSync( code ); console.log("Written operation index."); + +// find all test files +const testsDir = path.join(process.cwd() + "/tests/operations/tests/"); +const testObjs = []; +fs.readdirSync(testsDir).forEach(file => { + if (!file.endsWith(".mjs")) return; + testObjs.push(file.split(".mjs")[0]); +}); + +// Construct test index file +code = `/** +* THIS FILE IS AUTOMATICALLY GENERATED BY src/core/config/scripts/generateOpsIndex.mjs +* +* @author john [john19696@protonmail.com] +* @author tlwr [toby@toby.codes] +* @author n1474335 [n1474335@gmail.com] +* @copyright Crown Copyright ${new Date().getUTCFullYear()} +* @license Apache-2.0 +*/ + +import { + setLongTestFailure, + logTestReport, +} from "../lib/utils.mjs"; + +import "../lib/wasmFetchPolyfill.mjs"; + +import TestRegister from "../lib/TestRegister.mjs"; +`; + +testObjs.forEach(obj => { + if (obj !== "SplitColourChannels") + code += `import "./tests/${obj}.mjs";\n`; + else + code += `// Cannot test operations that use the File type yet +// import "./tests/SplitColourChannels.mjs";\n`; +}); + +code += ` + +const testStatus = { + allTestsPassing: true, + counts: { + total: 0, + } +}; + +setLongTestFailure(); + +const logOpsTestReport = logTestReport.bind(null, testStatus); + +(async function() { + const results = await TestRegister.runTests(); + logOpsTestReport(results); +})(); +`; + +// Write tests file +fs.writeFileSync( + path.join(testsDir, "../index.mjs"), + code +); +console.log("Written operation tests index."); diff --git a/src/core/config/scripts/newOperation.mjs b/src/core/config/scripts/newOperation.mjs index 1686f6eb..e46dffcc 100644 --- a/src/core/config/scripts/newOperation.mjs +++ b/src/core/config/scripts/newOperation.mjs @@ -23,7 +23,7 @@ if (!fs.existsSync(dir)) { console.log("Example> node --experimental-modules src/core/config/scripts/newOperation.mjs"); process.exit(1); } - +const testDir = path.join(process.cwd() + "/tests/operations/tests/"); const ioTypes = ["string", "byteArray", "number", "html", "ArrayBuffer", "BigNumber", "JSON", "File", "List"]; const schema = { @@ -123,6 +123,30 @@ prompt.get(schema, (err, result) => { return txt.charAt(0).toUpperCase() + txt.substr(1); }).replace(/[\s-()./]/g, ""); + const testTemplate = `/** +* ${moduleName} tests +* +* @author ${result.authorName} [${result.authorEmail}] +* @copyright Crown Copyright ${(new Date()).getFullYear()} +* @license Apache-2.0 +*/ + +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "${result.opName}: test", + input: "Example input", + expectedOutput: "Expected output", + recipeConfig: [ + { + op: "${result.opName}", + args: [], + }, + ], + }, +]); +`; const template = `/** * @author ${result.authorName} [${result.authorEmail}] @@ -218,13 +242,16 @@ export default ${moduleName}; } fs.writeFileSync(filename, template); + const testFilename = path.join(testDir, `./${moduleName}.mjs`); + fs.writeFileSync(testFilename, testTemplate); + console.log(`\nOperation template written to ${colors.green(filename)}`); + console.log(`\nOperation test template written to ${colors.green(testFilename)}`); console.log(`\nNext steps: 1. Add your operation to ${colors.green("src/core/config/Categories.json")} -2. Write your operation code. -3. Write tests in ${colors.green("tests/operations/tests/")} +2. Write your operation code in ${colors.green(filename)} +3. Write your operation test code in ${colors.green(testFilename)} 4. Run ${colors.cyan("npm run lint")} and ${colors.cyan("npm run test")} 5. Submit a Pull Request to get your operation added to the official CyberChef repository.`); }); - diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs deleted file mode 100644 index 44792560..00000000 --- a/tests/operations/index.mjs +++ /dev/null @@ -1,217 +0,0 @@ -/* eslint no-console: 0 */ - -/** - * Test Runner - * - * For running the tests in the test register. - * - * @author tlwr [toby@toby.codes] - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2017 - * @license Apache-2.0 - */ - -import "../lib/wasmFetchPolyfill.mjs"; -import { setLongTestFailure, logTestReport } from "../lib/utils.mjs"; - -import TestRegister from "../lib/TestRegister.mjs"; -import "./tests/A1Z26CipherDecode.mjs"; -import "./tests/AESKeyWrap.mjs"; -import "./tests/AnalyseUUID.mjs"; -import "./tests/AlternatingCaps.mjs"; -import "./tests/AvroToJSON.mjs"; -import "./tests/BaconCipher.mjs"; -import "./tests/Base32.mjs"; -import "./tests/Base45.mjs"; -import "./tests/Base58.mjs"; -import "./tests/Base62.mjs"; -import "./tests/Base64.mjs"; -import "./tests/Base85.mjs"; -import "./tests/Base92.mjs"; -import "./tests/BCD.mjs"; -import "./tests/Bech32.mjs"; -import "./tests/BitwiseOp.mjs"; -import "./tests/BLAKE2b.mjs"; -import "./tests/BLAKE2s.mjs"; -import "./tests/BLAKE3.mjs"; -import "./tests/Bombe.mjs"; -import "./tests/BSON.mjs"; -import "./tests/ByteRepr.mjs"; -import "./tests/CaesarBoxCipher.mjs"; -import "./tests/CaretMdecode.mjs"; -import "./tests/CartesianProduct.mjs"; -import "./tests/CBORDecode.mjs"; -import "./tests/CBOREncode.mjs"; -import "./tests/CetaceanCipherDecode.mjs"; -import "./tests/CetaceanCipherEncode.mjs"; -import "./tests/ChaCha.mjs"; -import "./tests/ChangeIPFormat.mjs"; -import "./tests/CharEnc.mjs"; -import "./tests/Charts.mjs"; -import "./tests/Ciphers.mjs"; -import "./tests/CipherSaber2.mjs"; -import "./tests/CMAC.mjs"; -import "./tests/Code.mjs"; -import "./tests/Colossus.mjs"; -import "./tests/Comment.mjs"; -import "./tests/Compress.mjs"; -import "./tests/ConditionalJump.mjs"; -import "./tests/ConvertCoordinateFormat.mjs"; -import "./tests/ConvertLeetSpeak.mjs"; -import "./tests/ConvertToNATOAlphabet.mjs"; -import "./tests/CRCChecksum.mjs"; -import "./tests/Crypt.mjs"; -import "./tests/CSV.mjs"; -import "./tests/DateTime.mjs"; -import "./tests/DefangIP.mjs"; -import "./tests/DisassembleARM.mjs"; -import "./tests/DropNthBytes.mjs"; -import "./tests/ECDSA.mjs"; -import "./tests/ELFInfo.mjs"; -import "./tests/Enigma.mjs"; -import "./tests/EscapeSmartCharacters.mjs"; -import "./tests/ExtractAudioMetadata.mjs"; -import "./tests/ExtractEmailAddresses.mjs"; -import "./tests/ExtractHashes.mjs"; -import "./tests/ExtractIPAddresses.mjs"; -import "./tests/Fernet.mjs"; -import "./tests/Float.mjs"; -import "./tests/FileTree.mjs"; -import "./tests/FletcherChecksum.mjs"; -import "./tests/Fork.mjs"; -import "./tests/FromDecimal.mjs"; -import "./tests/GenerateAllChecksums.mjs"; -import "./tests/GenerateAllHashes.mjs"; -import "./tests/GenerateDeBruijnSequence.mjs"; -import "./tests/GenerateQRCode.mjs"; -import "./tests/GetAllCasings.mjs"; -import "./tests/GOST.mjs"; -import "./tests/Gunzip.mjs"; -import "./tests/Gzip.mjs"; -import "./tests/Hash.mjs"; -import "./tests/HASSH.mjs"; -import "./tests/HaversineDistance.mjs"; -import "./tests/Hex.mjs"; -import "./tests/Hexdump.mjs"; -import "./tests/HKDF.mjs"; -import "./tests/Image.mjs"; -import "./tests/IndexOfCoincidence.mjs"; -import "./tests/JA3Fingerprint.mjs"; -import "./tests/JA4.mjs"; -import "./tests/JA3SFingerprint.mjs"; -import "./tests/Jsonata.mjs"; -import "./tests/JSONBeautify.mjs"; -import "./tests/JSONMinify.mjs"; -import "./tests/JSONtoCSV.mjs"; -import "./tests/Jump.mjs"; -import "./tests/JWK.mjs"; -import "./tests/JWTDecode.mjs"; -import "./tests/JWTSign.mjs"; -import "./tests/JWTVerify.mjs"; -import "./tests/LevenshteinDistance.mjs"; -import "./tests/Lorenz.mjs"; -import "./tests/LS47.mjs"; -import "./tests/LuhnChecksum.mjs"; -import "./tests/LZNT1Decompress.mjs"; -import "./tests/LZString.mjs"; -import "./tests/Magic.mjs"; -import "./tests/Media.mjs"; -import "./tests/MIMEDecoding.mjs"; -import "./tests/Modhex.mjs"; -import "./tests/MorseCode.mjs"; -import "./tests/MS.mjs"; -import "./tests/MultipleBombe.mjs"; -import "./tests/MurmurHash3.mjs"; -import "./tests/NetBIOS.mjs"; -import "./tests/NormaliseUnicode.mjs"; -import "./tests/NTLM.mjs"; -import "./tests/OTP.mjs"; -import "./tests/ParseEthernetFrame.mjs"; -import "./tests/ParseIPv4Header.mjs"; -import "./tests/ParseIPRange.mjs"; -import "./tests/ParseObjectIDTimestamp.mjs"; -import "./tests/ParseQRCode.mjs"; -import "./tests/ParseSSHHostKey.mjs"; -import "./tests/ParseTCP.mjs"; -import "./tests/ParseTLSRecord.mjs"; -import "./tests/ParseTLV.mjs"; -import "./tests/ParseUDP.mjs"; -import "./tests/PEMtoHex.mjs"; -import "./tests/PGP.mjs"; -import "./tests/PHP.mjs"; -import "./tests/ParityBit.mjs"; -import "./tests/PHPSerialize.mjs"; -import "./tests/PowerSet.mjs"; -import "./tests/Protobuf.mjs"; -import "./tests/PubKeyFromCert.mjs"; -import "./tests/PubKeyFromPrivKey.mjs"; -import "./tests/Rabbit.mjs"; -import "./tests/RAKE.mjs"; -import "./tests/Regex.mjs"; -import "./tests/Register.mjs"; -import "./tests/RemoveANSIEscapeCodes.mjs"; -import "./tests/RegularExpression.mjs"; -import "./tests/RenderMarkdown.mjs"; -import "./tests/RisonEncodeDecode.mjs"; -import "./tests/Rotate.mjs"; -import "./tests/RSA.mjs"; -import "./tests/Salsa20.mjs"; -import "./tests/XSalsa20.mjs"; -import "./tests/SeqUtils.mjs"; -import "./tests/SetDifference.mjs"; -import "./tests/SetIntersection.mjs"; -import "./tests/SetUnion.mjs"; -import "./tests/Shuffle.mjs"; -import "./tests/SIGABA.mjs"; -import "./tests/SM2.mjs"; -import "./tests/SM4.mjs"; -import "./tests/RC6.mjs"; -// import "./tests/SplitColourChannels.mjs"; // Cannot test operations that use the File type yet -import "./tests/SQLBeautify.mjs"; -import "./tests/StrUtils.mjs"; -import "./tests/StripIPv4Header.mjs"; -import "./tests/StripTCPHeader.mjs"; -import "./tests/StripUDPHeader.mjs"; -import "./tests/Subsection.mjs"; -import "./tests/SwapCase.mjs"; -import "./tests/SymmetricDifference.mjs"; -import "./tests/TakeNthBytes.mjs"; -import "./tests/Template.mjs"; -import "./tests/TextEncodingBruteForce.mjs"; -import "./tests/TextIntegerConverter.mjs"; -import "./tests/ToFromInsensitiveRegex.mjs"; -import "./tests/TranslateDateTimeFormat.mjs"; -import "./tests/Typex.mjs"; -import "./tests/UnescapeString.mjs"; -import "./tests/Unicode.mjs"; -import "./tests/Wrap.mjs"; -import "./tests/URLEncodeDecode.mjs"; -import "./tests/RSA.mjs"; -import "./tests/CBOREncode.mjs"; -import "./tests/CBORDecode.mjs"; -import "./tests/JA3Fingerprint.mjs"; -import "./tests/JA3SFingerprint.mjs"; -import "./tests/HASSH.mjs"; -import "./tests/JSONtoYAML.mjs"; - -// Cannot test operations that use the File type yet -// import "./tests/SplitColourChannels.mjs"; -import "./tests/YARA.mjs"; -import "./tests/ParseCSR.mjs"; -import "./tests/XXTEA.mjs"; - -const testStatus = { - allTestsPassing: true, - counts: { - total: 0, - }, -}; - -setLongTestFailure(); - -const logOpsTestReport = logTestReport.bind(null, testStatus); - -(async function () { - const results = await TestRegister.runTests(); - logOpsTestReport(results); -})(); diff --git a/tests/operations/tests/FlaskSession.mjs b/tests/operations/tests/FlaskSession.mjs index 7becf400..427f5f1e 100644 --- a/tests/operations/tests/FlaskSession.mjs +++ b/tests/operations/tests/FlaskSession.mjs @@ -14,15 +14,18 @@ const validTokenSha256 = "eyJyb2xlIjoic3VwZXJ1c2VyIiwidXNlciI6ImFkbWluIn0.aab3Ew const validKey = "mysecretkey"; const wrongKey = "notTheKey"; -const outputObject = { - user: "admin", - role: "superuser", -}; +const outputObject = `{ + "role": "superuser", + "user": "admin" +}`; -const outputVerify = { - valid: true, - payload: outputObject, -}; +const outputVerify = `{ + "valid": true, + "payload": { + "role": "superuser", + "user": "admin" + } +}`; TestRegister.addTests([ { diff --git a/tests/operations/tests/IPv6Transition.mjs b/tests/operations/tests/IPv6Transition.mjs index f7558c31..75aabefe 100644 --- a/tests/operations/tests/IPv6Transition.mjs +++ b/tests/operations/tests/IPv6Transition.mjs @@ -12,7 +12,7 @@ TestRegister.addTests([ { name: "IPv6 Transition: IPv4 to IPv6", input: "198.51.100.7", - expectedOutput: "6to4: 2002:c633:6407::/48\nIPv4 Mapped: ::ffff:c633:6407\nIPv4 Translated: ::ffff:0:c633:6407\nNat 64: 64:ff9b::c633:6407", + expectedOutput: "6to4: 2002:c633:6407::/48\nIPv4 Mapped: ::ffff:c633:6407\nIPv4 Translated: ::ffff:0:c633:6407\nNat 64: 64:ff9b::c633:6407\n", recipeConfig: [ { op: "IPv6 Transition Addresses", @@ -22,7 +22,7 @@ TestRegister.addTests([ }, { name: "IPv6 Transition: IPv4 /24 Range to IPv6", input: "198.51.100.0/24", - expectedOutput: "6to4: 2002:c633:6400::/40\nIPv4 Mapped: ::ffff:c633:6400/120\nIPv4 Translated: ::ffff:0:c633:6400/120\nNat 64: 64:ff9b::c633:6400/120", + expectedOutput: "6to4: 2002:c633:6400::/40\nIPv4 Mapped: ::ffff:c633:6400/120\nIPv4 Translated: ::ffff:0:c633:6400/120\nNat 64: 64:ff9b::c633:6400/120\n", recipeConfig: [ { op: "IPv6 Transition Addresses", @@ -32,7 +32,7 @@ TestRegister.addTests([ }, { name: "IPv6 Transition: IPv4 to IPv6 Remove headers", input: "198.51.100.7", - expectedOutput: "2002:c633:6407::/48\n::ffff:c633:6407\n::ffff:0:c633:6407\n64:ff9b::c633:6407", + expectedOutput: "2002:c633:6407::/48\n::ffff:c633:6407\n::ffff:0:c633:6407\n64:ff9b::c633:6407\n", recipeConfig: [ { op: "IPv6 Transition Addresses", @@ -42,7 +42,7 @@ TestRegister.addTests([ }, { name: "IPv6 Transition: IPv6 to IPv4", input: "64:ff9b::c633:6407", - expectedOutput: "IPv4: 198.51.100.7", + expectedOutput: "IPv4: 198.51.100.7\n", recipeConfig: [ { op: "IPv6 Transition Addresses", diff --git a/tests/operations/tests/ParseX509CRL.mjs b/tests/operations/tests/ParseX509CRL.mjs index 33de5e38..a889a498 100644 --- a/tests/operations/tests/ParseX509CRL.mjs +++ b/tests/operations/tests/ParseX509CRL.mjs @@ -51,16 +51,16 @@ const OUT_CRL_PEM_RSA = `Certificate Revocation List (CRL): DirName:/C=UK/ST=London/O=BB/CN=Test Root CA serial:37:5D:4B:F6:BD:7C:11:7C:CC:46:1A:FF:D7:2F:2C:26:F8:1E:4B:3D X509v3 CRL Distribution Points: - Full Name: - URI:http://example.com/full-crl - Full Name: - URI:ldap://example.com/full-crl - Full Name: - IP:127.0.0.1 + Full Name: + URI:http://example.com/full-crl + Full Name: + URI:ldap://example.com/full-crl + Full Name: + IP:127.0.0.1 X509v3 CRL Number: 1E3C - issuerAltName: - Unsupported CRL extension. Try openssl CLI. + X509v3 Issuer Alternative Name: + Revoked Certificates: Serial Number: 1000 Revocation Date: Sun, 25 Aug 2024 03:23:08 GMT @@ -143,16 +143,16 @@ const OUT_CRL_PEM_RSA_CRL_REASON_AND_INVALIDITY_DATE = `Certificate Revocation L DirName:/C=UK/ST=London/O=BB/CN=Test Root CA serial:37:5D:4B:F6:BD:7C:11:7C:CC:46:1A:FF:D7:2F:2C:26:F8:1E:4B:3D X509v3 CRL Distribution Points: - Full Name: - URI:http://example.com/full-crl - Full Name: - URI:ldap://example.com/full-crl - Full Name: - IP:127.0.0.1 + Full Name: + URI:http://example.com/full-crl + Full Name: + URI:ldap://example.com/full-crl + Full Name: + IP:127.0.0.1 X509v3 CRL Number: 1E3D - issuerAltName: - Unsupported CRL extension. Try openssl CLI. + X509v3 Issuer Alternative Name: + Revoked Certificates: Serial Number: 1000 Revocation Date: Sun, 25 Aug 2024 12:08:48 GMT