8.9.2

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 All notable changes to CyberChef will be documented in this file.
 
+### [8.9.0] - 2018-11-07
+- 'Defang URL' operation added [@arnydo] | [#394]
+
 ### [8.8.0] - 2018-10-10
 - 'Parse TLV' operation added [@GCHQ77703] | [#351]
 
@@ -76,6 +79,7 @@ All notable changes to CyberChef will be documented in this file.
 [@JustAnotherMark]: https://github.com/JustAnotherMark
 [@sevzero]: https://github.com/sevzero
 [@PenguinGeorge]: https://github.com/PenguinGeorge
+[@arnydo]: https://github.com/arnydo
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -95,3 +99,4 @@ All notable changes to CyberChef will be documented in this file.
 [#344]: https://github.com/gchq/CyberChef/pull/344
 [#348]: https://github.com/gchq/CyberChef/pull/348
 [#351]: https://github.com/gchq/CyberChef/pull/351
+[#394]: https://github.com/gchq/CyberChef/pull/394

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.8.8",
+  "version": "8.9.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.8.8",
+  "version": "8.9.2",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/config/Categories.json

@@ -159,7 +159,8 @@
             "Change IP format",
             "Group IP addresses",
             "Encode NetBIOS Name",
-            "Decode NetBIOS Name"
+            "Decode NetBIOS Name",
+            "Defang URL"
         ]
     },
     {

src/core/lib/Extract.mjs

@@ -39,3 +39,21 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
     return output;
 }
+
+
+/**
+ * URL regular expression
+ */
+const protocol = "[A-Z]+://",
+    hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
+    port = ":\\d+",
+    path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*" +
+        "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
+
+export const URL_REGEX = new RegExp(protocol + hostname + "(?:" + port + ")?(?:" + path + ")?", "ig");
+
+
+/**
+ * Domain name regular expression
+ */
+export const DOMAIN_REGEX = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;

src/core/operations/DefangURL.mjs

@@ -0,0 +1,102 @@
+/**
+ * @author arnydo [arnydo@protonmail.com]
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {URL_REGEX, DOMAIN_REGEX} from "../lib/Extract";
+
+/**
+ * DefangURL operation
+ */
+class DefangURL extends Operation {
+
+    /**
+     * DefangURL constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Defang URL";
+        this.module = "Default";
+        this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning the URL becomes invalid, neutralising the risk of accidentally clicking on a malicious link.<br><br>This is often used when dealing with malicious links or IOCs.<br><br>Works well when combined with the 'Extract URLs' operation.";
+        this.infoURL = "https://isc.sans.edu/forums/diary/Defang+all+the+things/22744/";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Escape dots",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape http",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape ://",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Process",
+                type: "option",
+                value: ["Valid domains and full URLs", "Only full URLs", "Everything"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [dots, http, slashes, process] = args;
+
+        switch (process) {
+            case "Valid domains and full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                input = input.replace(DOMAIN_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Only full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Everything":
+                input = defangURL(input, dots, http, slashes);
+                break;
+        }
+
+        return input;
+    }
+
+}
+
+
+/**
+ * Defangs a given URL
+ *
+ * @param {string} url
+ * @param {boolean} dots
+ * @param {boolean} http
+ * @param {boolean} slashes
+ * @returns {string}
+ */
+function defangURL(url, dots, http, slashes) {
+    if (dots) url = url.replace(/\./g, "[.]");
+    if (http) url = url.replace(/http/gi, "hxxp");
+    if (slashes) url = url.replace(/:\/\//g, "[://]");
+
+    return url;
+}
+
+export default DefangURL;

src/core/operations/ExtractDomains.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, DOMAIN_REGEX } from "../lib/Extract";
 
 /**
  * Extract domains operation
@@ -38,10 +38,8 @@ class ExtractDomains extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            regex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;
-
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, DOMAIN_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/ExtractURLs.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, URL_REGEX } from "../lib/Extract";
 
 /**
  * Extract URLs operation
@@ -38,16 +38,8 @@ class ExtractURLs extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            protocol = "[A-Z]+://",
-            hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
-            port = ":\\d+";
-        let path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*";
-
-        path += "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
-        const regex = new RegExp(protocol + hostname + "(?:" + port +
-            ")?(?:" + path + ")?", "ig");
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, URL_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/FromDecimal.mjs

@@ -29,38 +29,43 @@ class FromDecimal extends Operation {
                 "name": "Delimiter",
                 "type": "option",
                 "value": DELIM_OPTIONS
+            },
+            {
+                "name": "Support signed values",
+                "type": "boolean",
+                "value": false
             }
         ];
         this.patterns = [
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Space"]
+                args: ["Space", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Comma"]
+                args: ["Comma", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Semi-colon"]
+                args: ["Semi-colon", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Colon"]
+                args: ["Colon", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Line feed"]
+                args: ["Line feed", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["CRLF"]
+                args: ["CRLF", false]
             },
         ];
     }
@@ -71,7 +76,11 @@ class FromDecimal extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        return fromDecimal(input, args[0]);
+        let data = fromDecimal(input, args[0]);
+        if (args[1]) { // Convert negatives
+            data = data.map(v => v < 0 ? 0xFF + v + 1 : v);
+        }
+        return data;
     }
 
 }

src/core/operations/HMAC.mjs

@@ -28,8 +28,9 @@ class HMAC extends Operation {
         this.args = [
             {
                 "name": "Key",
-                "type": "binaryString",
-                "value": ""
+                "type": "toggleString",
+                "value": "",
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             },
             {
                 "name": "Hashing function",
@@ -66,7 +67,7 @@ class HMAC extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const key = args[0],
+        const key = Utils.convertToByteString(args[0].string || "", args[0].option),
             hashFunc = args[1].toLowerCase(),
             msg = Utils.arrayBufferToStr(input, false),
             hasher = CryptoApi.getHasher(hashFunc);

src/core/operations/ToDecimal.mjs

@@ -30,6 +30,11 @@ class ToDecimal extends Operation {
                 "name": "Delimiter",
                 "type": "option",
                 "value": DELIM_OPTIONS
+            },
+            {
+                "name": "Support signed values",
+                "type": "boolean",
+                "value": false
             }
         ];
     }
@@ -40,7 +45,11 @@ class ToDecimal extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const delim = Utils.charRep(args[0]);
+        const delim = Utils.charRep(args[0]),
+            signed = args[1];
+        if (signed) {
+            input = input.map(v => v > 0x7F ? v - 0xFF - 1 : v);
+        }
         return input.join(delim);
     }
 

test/index.mjs

@@ -42,7 +42,8 @@ import "./tests/operations/Crypt";
 import "./tests/operations/DateTime";
 import "./tests/operations/ExtractEmailAddresses";
 import "./tests/operations/Fork";
-import "./tests/operations/FromGeohash.mjs";
+import "./tests/operations/FromDecimal";
+import "./tests/operations/FromGeohash";
 import "./tests/operations/Hash";
 import "./tests/operations/HaversineDistance";
 import "./tests/operations/Hexdump";

test/tests/operations/FromDecimal.mjs

@@ -0,0 +1,33 @@
+/**
+ * From Decimal tests
+ *
+ * @author qistoph
+ * @copyright Crown Copyright 2018
+ * @licence Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "From Decimal",
+        input: "83 97 109 112 108 101 32 84 101 120 116",
+        expectedOutput: "Sample Text",
+        recipeConfig: [
+            {
+                op: "From Decimal",
+                args: ["Space", false]
+            },
+        ],
+    },
+    {
+        name: "From Decimal with negatives",
+        input: "-130,-140,-152,-151,115,33,0,-1",
+        expectedOutput: "~this!\u0000\u00ff",
+        recipeConfig: [
+            {
+                op: "From Decimal",
+                args: ["Comma", true]
+            },
+        ],
+    },
+]);

test/tests/operations/Hash.mjs

@@ -411,7 +411,7 @@ TestRegister.addTests([
         recipeConfig: [
             {
                 "op": "HMAC",
-                "args": ["test", "SHA256"]
+                "args": [{"option": "Latin1", "string": "test"}, "SHA256"]
             }
         ]
     },