firstt try to update new op. new test fix, see email

2022-09-30 14:01:49 +01:00 · 2022-09-30 14:01:49 +01:00 · e5501269b3
commit e5501269b3
parent c2cf535f88
4 changed files with 177 additions and 66 deletions
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "cyberchef",
-  "version": "9.46.5",
+  "version": "9.46.0",
  "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
  "author": "n1474335 <n1474335@gmail.com>",
  "homepage": "https://gchq.github.io/CyberChef",
@ -49,7 +49,7 @@
    "babel-loader": "^8.2.5",
    "babel-plugin-dynamic-import-node": "^2.3.3",
    "babel-plugin-transform-builtin-extend": "1.1.2",
-    "chromedriver": "^103.0.0",
+    "chromedriver": "^101.0.0",
    "cli-progress": "^3.11.1",
    "colors": "^1.4.0",
    "copy-webpack-plugin": "^11.0.0",
@ -77,6 +77,7 @@
    "postcss-import": "^14.1.0",
    "postcss-loader": "^7.0.0",
    "prompt": "^1.3.0",
+    "sass-loader": "^13.0.0",
    "sitemap": "^7.1.1",
    "terser": "^5.14.0",
    "webpack": "^5.73.0",
@ -116,13 +117,14 @@
    "flat": "^5.0.2",
    "geodesy": "1.1.3",
    "highlight.js": "^11.5.1",
+    "iniparser": "^1.0.5",
    "jimp": "^0.16.1",
    "jquery": "3.6.0",
    "js-crc": "^0.2.0",
    "js-sha3": "^0.8.0",
    "jsesc": "^3.0.2",
    "json5": "^2.2.1",
-    "jsonpath-plus": "^7.2.0",
+    "jsonpath": "^1.1.1",
    "jsonwebtoken": "^8.5.1",
    "jsqr": "^1.4.0",
    "jsrsasign": "^10.5.23",
@ -139,6 +141,7 @@
    "ngeohash": "^0.6.3",
    "node-forge": "^1.3.1",
    "node-md6": "^0.1.0",
+    "node-sass": "^7.0.1",
    "nodom": "^2.4.0",
    "notepack.io": "^3.0.1",
    "nwmatcher": "^1.4.4",
@ -154,7 +157,7 @@
    "split.js": "^1.6.5",
    "ssdeep.js": "0.0.3",
    "stream-browserify": "^3.0.0",
-    "tesseract.js": "3.0.2",
+    "tesseract.js": "2.1.5",
    "ua-parser-js": "^1.0.2",
    "unorm": "^1.6.0",
    "utf8": "^3.0.0",
@ -167,9 +170,8 @@
  "scripts": {
    "start": "npx grunt dev",
    "build": "npx grunt prod",
-    "node": "npx grunt node",
    "repl": "node --experimental-modules --experimental-json-modules --experimental-specifier-resolution=node --no-warnings src/node/repl.mjs",
-    "test": "npx grunt configTests && node --experimental-modules --experimental-json-modules --no-warnings --no-deprecation tests/node/index.mjs && node --experimental-modules --experimental-json-modules --no-warnings --no-deprecation tests/operations/index.mjs",
+    "test": "npx grunt configTests && node --trace-uncaught --experimental-modules --experimental-json-modules --no-warnings --no-deprecation tests/node/index.mjs && node --experimental-modules --experimental-json-modules --no-warnings --no-deprecation tests/operations/index.mjs",
    "testnodeconsumer": "npx grunt testnodeconsumer",
    "testui": "npx grunt testui",
    "testuidev": "npx nightwatch --env=dev",
--- a/src/core/config/scripts/generateOpsIndex.mjs
+++ b/src/core/config/scripts/generateOpsIndex.mjs
@ -58,3 +58,64 @@ fs.writeFileSync(
    code
 );
 console.log("Written operation index.");
+
+// find all test files
+const testsDir = path.join(process.cwd() + "/tests/operations/tests/");
+const testObjs = [];
+fs.readdirSync(testsDir).forEach(file => {
+    if (!file.endsWith(".mjs")) return;
+    testObjs.push(file.split(".mjs")[0]);
+});
+
+// Construct test index file
+code = `/**
+* THIS FILE IS AUTOMATICALLY GENERATED BY src/core/config/scripts/generateOpsIndex.mjs
+*
+* @author john [john19696@protonmail.com]
+* @author tlwr [toby@toby.codes]
+* @author n1474335 [n1474335@gmail.com]
+* @copyright Crown Copyright ${new Date().getUTCFullYear()}
+* @license Apache-2.0
+*/
+
+import {
+    setLongTestFailure,
+    logTestReport,
+} from "../lib/utils.mjs";
+
+import TestRegister from "../lib/TestRegister.mjs";
+`;
+
+testObjs.forEach(obj => {
+    code += `import "./tests/${obj}.mjs";\n`;
+});
+
+code += `
+// Cannot test operations that use the File type yet
+// import "./tests/SplitColourChannels.mjs";
+
+const testStatus = {
+    allTestsPassing: true,
+    counts: {
+        total: 0,
+    }
+};
+
+setLongTestFailure();
+
+const logOpsTestReport = logTestReport.bind(null, testStatus);
+
+(async function() {
+    const results = await TestRegister.runTests();
+    logOpsTestReport(results);
+})();
+`;
+
+// Write tests file
+fs.writeFileSync(
+    path.join(testsDir, "../index.mjs"),
+    code
+);
+console.log("Written operation tests index.");
+
+
--- a/src/core/config/scripts/newOperation.mjs
+++ b/src/core/config/scripts/newOperation.mjs
@ -14,6 +14,7 @@ import process from "process";
 import fs from "fs";
 import path from "path";
 import EscapeString from "../../operations/EscapeString.mjs";
+import iniparser from "iniparser";


 const dir = path.join(process.cwd() + "/src/core/operations/");
@ -23,6 +24,23 @@ if (!fs.existsSync(dir)) {
    console.log("Example> node --experimental-modules src/core/config/scripts/newOperation.mjs");
    process.exit(1);
 }
+const testDir = path.join(process.cwd() + "/tests/operations/tests/");
+
+const configFile = "/.gitconfig";
+let gitUserEmail, gitUserName;
+// gitconfig in root is better than global one
+let gitConfig = process.cwd() + configFile;
+if (!fs.existsSync()) {
+    gitConfig = process.env.HOME +  configFile;
+}
+// get user settings from git
+if (fs.existsSync(gitConfig)) {
+    const config = iniparser.parseSync(gitConfig);
+    if ("user" in config) {
+        if ("email" in config.user) gitUserEmail = config.user.email;
+        if ("name" in config.user) gitUserName = config.user.name; 
+    }
+}

 const ioTypes = ["string", "byteArray", "number", "html", "ArrayBuffer", "BigNumber", "JSON", "File", "List<File>"];

@ -88,12 +106,14 @@ If your operation does not rely on a library, just leave this blank and it will
        authorName: {
            description: "Your name or username will be added to the @author tag for this operation.",
            example: "n1474335",
+            default: gitUserName,
            prompt: "Username",
            type: "string"
        },
        authorEmail: {
            description: "Your email address will also be added to the @author tag for this operation.",
            example: "n1474335@gmail.com",
+            default: gitUserEmail,
            prompt: "Email",
            type: "string"
        }
@ -123,6 +143,30 @@ prompt.get(schema, (err, result) => {
        return txt.charAt(0).toUpperCase() + txt.substr(1);
    }).replace(/[\s-()./]/g, "");

+    const testTemplate = `/**
+* ${moduleName} tests
+*    
+* @author ${result.authorName} [${result.authorEmail}]
+* @copyright Crown Copyright ${(new Date()).getFullYear()}
+* @license Apache-2.0
+*/
+
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: " ${moduleName}: test",
+        input: "Example input",
+        expectedOutput: "Expected output",
+        recipeConfig: [
+            {
+                op: " ${moduleName}",
+                args: [],
+            },
+        ],
+    },
+]);
+`;

    const template = `/**
 * @author ${result.authorName} [${result.authorEmail}]
@ -218,13 +262,18 @@ export default ${moduleName};
    }
    fs.writeFileSync(filename, template);

+    const testFilename = path.join(testDir, `./${moduleName}.mjs`);
+    fs.writeFileSync(testFilename, testTemplate);
+
    console.log(`\nOperation template written to ${colors.green(filename)}`);
+    console.log(`\nOperation test template written to ${colors.green(testFilename)}`);
    console.log(`\nNext steps:
 1. Add your operation to ${colors.green("src/core/config/Categories.json")}
-2. Write your operation code.
-3. Write tests in ${colors.green("tests/operations/tests/")}
-4. Run ${colors.cyan("npm run lint")} and ${colors.cyan("npm run test")}
-5. Submit a Pull Request to get your operation added to the official CyberChef repository.`);
+2. Write your operation code in ${colors.green(filename)}
+3. Add your operation to ${colors.green("tests/operations/index.mjs")}
+4. Write your operation test code in ${colors.green(testFilename)}
+5. Write tests in ${colors.green("tests/operations/tests/")}
+6. Run ${colors.cyan("npm run lint")} and ${colors.cyan("npm run test")}
+7. Submit a Pull Request to get your operation aldded to the official CyberChef repository.`);

 });
-
--- a/tests/operations/index.mjs
+++ b/tests/operations/index.mjs
@ -1,15 +1,12 @@
-/* eslint no-console: 0 */
-
 /**
- * Test Runner
- *
- * For running the tests in the test register.
- *
- * @author tlwr [toby@toby.codes]
- * @author n1474335 [n1474335@gmail.com]
- * @copyright Crown Copyright 2017
- * @license Apache-2.0
- */
+* THIS FILE IS AUTOMATICALLY GENERATED BY src/core/config/scripts/generateOpsIndex.mjs
+*
+* @author john [john19696@protonmail.com]
+* @author tlwr [toby@toby.codes]
+* @author n1474335 [n1474335@gmail.com]
+* @copyright Crown Copyright 2022
+* @license Apache-2.0
+*/

 import {
    setLongTestFailure,
@ -17,111 +14,113 @@ import {
 } from "../lib/utils.mjs";

 import TestRegister from "../lib/TestRegister.mjs";
+import "./tests/AvroToJSON.mjs";
 import "./tests/BCD.mjs";
+import "./tests/BLAKE2b.mjs";
+import "./tests/BLAKE2s.mjs";
 import "./tests/BSON.mjs";
 import "./tests/BaconCipher.mjs";
 import "./tests/Base45.mjs";
 import "./tests/Base58.mjs";
-import "./tests/Base64.mjs";
 import "./tests/Base62.mjs";
+import "./tests/Base64.mjs";
 import "./tests/Base85.mjs";
 import "./tests/BitwiseOp.mjs";
+import "./tests/Bombe.mjs";
 import "./tests/ByteRepr.mjs";
+import "./tests/CBORDecode.mjs";
+import "./tests/CBOREncode.mjs";
+import "./tests/CSV.mjs";
+import "./tests/CaesarBoxCipher.mjs";
 import "./tests/CartesianProduct.mjs";
-import "./tests/CetaceanCipherEncode.mjs";
 import "./tests/CetaceanCipherDecode.mjs";
-import "./tests/CharEnc.mjs";
+import "./tests/CetaceanCipherEncode.mjs";
 import "./tests/ChangeIPFormat.mjs";
+import "./tests/CharEnc.mjs";
 import "./tests/Charts.mjs";
 import "./tests/Checksum.mjs";
+import "./tests/CipherSaber2.mjs";
 import "./tests/Ciphers.mjs";
 import "./tests/Code.mjs";
+import "./tests/Colossus.mjs";
 import "./tests/Comment.mjs";
 import "./tests/Compress.mjs";
 import "./tests/ConditionalJump.mjs";
+import "./tests/ConvertCoordinateFormat.mjs";
+import "./tests/ConvertToNATOAlphabet.mjs";
 import "./tests/Crypt.mjs";
-import "./tests/CSV.mjs";
 import "./tests/DateTime.mjs";
+import "./tests/DefangIP.mjs";
+import "./tests/ELFInfo.mjs";
+import "./tests/Enigma.mjs";
 import "./tests/ExtractEmailAddresses.mjs";
 import "./tests/Fork.mjs";
 import "./tests/FromDecimal.mjs";
-import "./tests/Gzip.mjs";
+import "./tests/FromGeohash.mjs";
+import "./tests/GetAllCasings.mjs";
 import "./tests/Gunzip.mjs";
+import "./tests/Gzip.mjs";
+import "./tests/HASSH.mjs";
 import "./tests/Hash.mjs";
 import "./tests/HaversineDistance.mjs";
 import "./tests/Hex.mjs";
 import "./tests/Hexdump.mjs";
 import "./tests/Image.mjs";
 import "./tests/IndexOfCoincidence.mjs";
-import "./tests/Jump.mjs";
+import "./tests/JA3Fingerprint.mjs";
+import "./tests/JA3SFingerprint.mjs";
 import "./tests/JSONBeautify.mjs";
 import "./tests/JSONMinify.mjs";
 import "./tests/JSONtoCSV.mjs";
 import "./tests/JWTDecode.mjs";
 import "./tests/JWTSign.mjs";
 import "./tests/JWTVerify.mjs";
+import "./tests/Jump.mjs";
+import "./tests/LS47.mjs";
+import "./tests/LZString.mjs";
+import "./tests/Lorenz.mjs";
+import "./tests/LuhnChecksum.mjs";
 import "./tests/MS.mjs";
 import "./tests/Magic.mjs";
+import "./tests/Media.mjs";
 import "./tests/MorseCode.mjs";
+import "./tests/MultipleBombe.mjs";
 import "./tests/NetBIOS.mjs";
 import "./tests/NormaliseUnicode.mjs";
 import "./tests/OTP.mjs";
+import "./tests/PEMtoHex.mjs";
 import "./tests/PGP.mjs";
 import "./tests/PHP.mjs";
 import "./tests/ParseIPRange.mjs";
+import "./tests/ParseObjectIDTimestamp.mjs";
 import "./tests/ParseQRCode.mjs";
-import "./tests/PEMtoHex.mjs";
+import "./tests/ParseSSHHostKey.mjs";
+import "./tests/ParseTCP.mjs";
+import "./tests/ParseTLV.mjs";
+import "./tests/ParseUDP.mjs";
 import "./tests/PowerSet.mjs";
+import "./tests/Protobuf.mjs";
+import "./tests/RSA.mjs";
 import "./tests/Regex.mjs";
 import "./tests/Register.mjs";
 import "./tests/Rotate.mjs";
+import "./tests/SIGABA.mjs";
+import "./tests/SM4.mjs";
 import "./tests/SeqUtils.mjs";
 import "./tests/SetDifference.mjs";
 import "./tests/SetIntersection.mjs";
 import "./tests/SetUnion.mjs";
-import "./tests/SM4.mjs";
+import "./tests/SplitColourChannels.mjs";
 import "./tests/StrUtils.mjs";
+import "./tests/Subsection.mjs";
 import "./tests/SymmetricDifference.mjs";
 import "./tests/TextEncodingBruteForce.mjs";
-import "./tests/TranslateDateTimeFormat.mjs";
-import "./tests/Magic.mjs";
-import "./tests/ParseTLV.mjs";
-import "./tests/Media.mjs";
 import "./tests/ToFromInsensitiveRegex.mjs";
-import "./tests/YARA.mjs";
-import "./tests/ConvertCoordinateFormat.mjs";
-import "./tests/Enigma.mjs";
-import "./tests/Bombe.mjs";
-import "./tests/MultipleBombe.mjs";
+import "./tests/ToGeohash.mjs";
+import "./tests/TranslateDateTimeFormat.mjs";
 import "./tests/Typex.mjs";
-import "./tests/BLAKE2b.mjs";
-import "./tests/BLAKE2s.mjs";
-import "./tests/Protobuf.mjs";
-import "./tests/ParseSSHHostKey.mjs";
-import "./tests/DefangIP.mjs";
-import "./tests/ParseUDP.mjs";
-import "./tests/ParseTCP.mjs";
-import "./tests/AvroToJSON.mjs";
-import "./tests/Lorenz.mjs";
-import "./tests/LuhnChecksum.mjs";
-import "./tests/CipherSaber2.mjs";
-import "./tests/Colossus.mjs";
-import "./tests/ParseObjectIDTimestamp.mjs";
 import "./tests/Unicode.mjs";
-import "./tests/RSA.mjs";
-import "./tests/CBOREncode.mjs";
-import "./tests/CBORDecode.mjs";
-import "./tests/JA3Fingerprint.mjs";
-import "./tests/JA3SFingerprint.mjs";
-import "./tests/HASSH.mjs";
-import "./tests/GetAllCasings.mjs";
-import "./tests/SIGABA.mjs";
-import "./tests/ELFInfo.mjs";
-import "./tests/Subsection.mjs";
-import "./tests/CaesarBoxCipher.mjs";
-import "./tests/LS47.mjs";
-import "./tests/LZString.mjs";
-
+import "./tests/YARA.mjs";

 // Cannot test operations that use the File type yet
 // import "./tests/SplitColourChannels.mjs";