7.7.7

.travis.yml

@@ -11,6 +11,7 @@ script:
   - grunt node
   - grunt prod --msg="$COMPILE_MSG"
 before_deploy:
+  - grunt exec:sitemap
   - grunt copy:ghPages
 deploy:
   - provider: pages

Gruntfile.js

@@ -185,6 +185,7 @@ module.exports = function (grunt) {
         webpack: {
             options: webpackConfig,
             metaConf: {
+                mode: "production",
                 target: "node",
                 entry: [
                     "babel-polyfill",
@@ -200,6 +201,7 @@ module.exports = function (grunt) {
                 externals: [NodeExternals()],
             },
             metaConfDev: {
+                mode: "development",
                 target: "node",
                 entry: [
                     "babel-polyfill",
@@ -216,9 +218,11 @@ module.exports = function (grunt) {
                 watch: true
             },
             web: {
+                mode: "production",
                 target: "web",
                 entry: Object.assign({
-                    main: "./src/web/index.js"
+                    main: "./src/web/index.js",
+                    sitemap: "./src/web/static/sitemap.js"
                 }, moduleEntryPoints),
                 output: {
                     path: __dirname + "/build/prod"
@@ -230,15 +234,6 @@ module.exports = function (grunt) {
                 },
                 plugins: [
                     new webpack.DefinePlugin(BUILD_CONSTANTS),
-                    new webpack.optimize.UglifyJsPlugin({
-                        compress: {
-                            "screw_ie8": true,
-                            "dead_code": true,
-                            "unused": true,
-                            "warnings": false
-                        },
-                        comments: false,
-                    }),
                     new HtmlWebpackPlugin({
                         filename: "index.html",
                         template: "./src/web/html/index.html",
@@ -255,6 +250,7 @@ module.exports = function (grunt) {
                 ]
             },
             webInline: {
+                mode: "production",
                 target: "web",
                 entry: "./src/web/index.js",
                 output: {
@@ -263,15 +259,6 @@ module.exports = function (grunt) {
                 },
                 plugins: [
                     new webpack.DefinePlugin(BUILD_CONSTANTS),
-                    new webpack.optimize.UglifyJsPlugin({
-                        compress: {
-                            "screw_ie8": true,
-                            "dead_code": true,
-                            "unused": true,
-                            "warnings": false
-                        },
-                        comments: false,
-                    }),
                     new HtmlWebpackPlugin({
                         filename: "cyberchef.htm",
                         template: "./src/web/html/index.html",
@@ -288,6 +275,7 @@ module.exports = function (grunt) {
                 ]
             },
             tests: {
+                mode: "development",
                 target: "node",
                 entry: "./test/index.js",
                 externals: [NodeExternals()],
@@ -300,6 +288,7 @@ module.exports = function (grunt) {
                 ]
             },
             node: {
+                mode: "production",
                 target: "node",
                 entry: "./src/node/index.js",
                 externals: [NodeExternals()],
@@ -326,11 +315,13 @@ module.exports = function (grunt) {
                     children: false,
                     chunks: false,
                     modules: false,
+                    entrypoints: false,
                     warningsFilter: /source-map/,
                 }
             },
             start: {
                 webpack: {
+                    mode: "development",
                     target: "web",
                     entry: Object.assign({
                         main: "./src/web/index.js"
@@ -377,7 +368,7 @@ module.exports = function (grunt) {
                         expand: true,
                         src: "docs/**",
                         dest: "build/prod/"
-                    }
+                    },
                 ]
             }
         },
@@ -406,6 +397,9 @@ module.exports = function (grunt) {
             cleanGit: {
                 command: "git gc --prune=now --aggressive"
             },
+            sitemap: {
+                command: "node build/prod/sitemap.js > build/prod/sitemap.xml"
+            }
         },
         execute: {
             test: "build/test/index.js"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "7.7.2",
+  "version": "7.7.7",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -31,15 +31,15 @@
   "bugs": "https://github.com/gchq/CyberChef/issues",
   "devDependencies": {
     "babel-core": "^6.26.0",
-    "babel-loader": "^7.1.2",
+    "babel-loader": "^7.1.3",
     "babel-preset-env": "^1.6.1",
-    "css-loader": "^0.28.9",
-    "eslint": "^4.17.0",
-    "exports-loader": "^0.6.4",
-    "extract-text-webpack-plugin": "^3.0.2",
-    "file-loader": "^1.1.6",
-    "grunt": ">=1.0.1",
-    "grunt-accessibility": "~5.0.0",
+    "css-loader": "^0.28.10",
+    "eslint": "^4.18.1",
+    "exports-loader": "^0.7.0",
+    "extract-text-webpack-plugin": "^4.0.0-alpha0",
+    "file-loader": "^1.1.10",
+    "grunt": ">=1.0.2",
+    "grunt-accessibility": "~6.0.0",
     "grunt-chmod": "~1.1.1",
     "grunt-concurrent": "^2.3.1",
     "grunt-contrib-clean": "~1.1.0",
@@ -49,23 +49,24 @@
     "grunt-execute": "^0.2.2",
     "grunt-jsdoc": "^2.2.1",
     "grunt-webpack": "^3.0.2",
-    "html-webpack-plugin": "^2.30.1",
-    "imports-loader": "^0.7.1",
+    "html-webpack-plugin": "^3.0.4",
+    "imports-loader": "^0.8.0",
     "ink-docstrap": "^1.3.2",
     "jsdoc-babel": "^0.3.0",
-    "less": "^2.7.3",
-    "less-loader": "^4.0.5",
+    "less": "^3.0.1",
+    "less-loader": "^4.0.6",
     "postcss-css-variables": "^0.8.0",
-    "postcss-import": "^11.0.0",
-    "postcss-loader": "^2.0.10",
-    "style-loader": "^0.19.1",
+    "postcss-import": "^11.1.0",
+    "postcss-loader": "^2.1.1",
+    "sitemap": "^1.13.0",
+    "style-loader": "^0.20.2",
     "url-loader": "^0.6.2",
     "val-loader": "^1.1.0",
     "web-resource-inliner": "^4.2.1",
-    "webpack": "^3.10.0",
-    "webpack-dev-server": "^2.11.1",
+    "webpack": "^4.0.1",
+    "webpack-dev-server": "^3.1.0",
     "webpack-node-externals": "^1.6.0",
-    "worker-loader": "^1.1.0"
+    "worker-loader": "^1.1.1"
   },
   "dependencies": {
     "babel-polyfill": "^6.26.0",
@@ -73,10 +74,10 @@
     "bootstrap": "^3.3.7",
     "bootstrap-colorpicker": "^2.5.2",
     "bootstrap-switch": "^3.3.4",
-    "crypto-api": "^0.7.5",
+    "crypto-api": "^0.8.0",
     "crypto-js": "^3.1.9-1",
     "diff": "^3.4.0",
-    "escodegen": "^1.9.0",
+    "escodegen": "^1.9.1",
     "esmangle": "^1.0.1",
     "esprima": "^4.0.0",
     "exif-parser": "^0.1.12",
@@ -87,24 +88,25 @@
     "js-sha3": "^0.7.0",
     "jsbn": "^1.1.0",
     "jsonpath": "^1.0.0",
-    "jsrsasign": "8.0.4",
-    "lodash": "^4.17.4",
+    "jsrsasign": "8.0.6",
+    "lodash": "^4.17.5",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",
     "moment": "^2.20.1",
     "moment-timezone": "^0.5.14",
-    "node-forge": "^0.7.1",
+    "node-forge": "^0.7.2",
     "node-md6": "^0.1.0",
     "nwmatcher": "^1.4.3",
     "otp": "^0.1.3",
     "sladex-blowfish": "^0.8.1",
     "sortablejs": "^1.7.0",
     "split.js": "^1.3.5",
+    "ua-parser-js": "^0.7.17",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",
     "xpath": "0.0.27",
-    "xregexp": "^4.0.0",
+    "xregexp": "^4.1.1",
     "zlibjs": "^0.3.1"
   },
   "scripts": {

src/core/ChefWorker.js

@@ -138,6 +138,7 @@ function loadRequiredModules(recipeConfig) {
             log.info("Loading module " + module);
             self.sendStatusMessage("Loading module " + module);
             self.importScripts(self.docURL + "/" + module + ".js");
+            self.sendStatusMessage("");
         }
     });
 }

src/core/Utils.js

@@ -952,7 +952,7 @@ const Utils = {
         // Parse bespoke recipe format
         recipe = recipe.replace(/\n/g, "");
         let m,
-            recipeRegex = /([^(]+)\(((?:'[^'\\]*(?:\\.[^'\\]*)*'|[^)/])*)(\/[^)]+)?\)/g,
+            recipeRegex = /([^(]+)\(((?:'[^'\\]*(?:\\.[^'\\]*)*'|[^)/'])*)(\/[^)]+)?\)/g,
             recipeConfig = [],
             args;
 

src/core/config/OperationConfig.js

@@ -1825,6 +1825,11 @@ const OperationConfig = {
                 name: "Cisco style",
                 type: "boolean",
                 value: MAC.CISCO_STYLE
+            },
+            {
+                name: "IPv6 interface ID",
+                type: "boolean",
+                value: MAC.IPV6_INTERFACE_ID
             }
         ]
     },
@@ -3176,7 +3181,7 @@ const OperationConfig = {
         outputType: "string",
         args: [
             {
-                name: "Password",
+                name: "Key",
                 type: "binaryString",
                 value: ""
             },

src/core/operations/Extract.js

@@ -207,7 +207,7 @@ const Extract = {
      */
     runEmail: function(input, args) {
         let displayTotal = args[0],
-            regex = /\w[-.\w]*@[-\w]+(?:\.[-\w]+)*\.[A-Z]{2,4}/ig;
+            regex = /\b\w[-.\w]*@[-\w]+(?:\.[-\w]+)*\.[A-Z]{2,4}\b/ig;
 
         return Extract._search(input, regex, null, displayTotal);
     },

src/core/operations/HTTP.js

@@ -1,4 +1,4 @@
-import {UAS_parser as UAParser} from "../lib/uas_parser.js";
+import UAParser from "ua-parser-js";
 
 
 /**
@@ -46,20 +46,22 @@ const HTTP = {
      * @returns {string}
      */
     runParseUserAgent: function(input, args) {
-        const ua = UAParser.parse(input);
-
-        return "Type: " + ua.type + "\n" +
-            "Family: " + ua.uaFamily + "\n" +
-            "Name: " + ua.uaName + "\n" +
-            "URL: " + ua.uaUrl + "\n" +
-            "Company: " + ua.uaCompany + "\n" +
-            "Company URL: " + ua.uaCompanyUrl + "\n\n" +
-            "OS Family: " + ua.osFamily + "\n" +
-            "OS Name: " + ua.osName + "\n" +
-            "OS URL: " + ua.osUrl + "\n" +
-            "OS Company: " + ua.osCompany + "\n" +
-            "OS Company URL: " + ua.osCompanyUrl + "\n" +
-            "Device Type: " + ua.deviceType + "\n";
+        const ua = UAParser(input);
+        return `Browser
+    Name: ${ua.browser.name || "unknown"}
+    Version: ${ua.browser.version || "unknown"}
+Device
+    Model: ${ua.device.model || "unknown"}
+    Type: ${ua.device.type || "unknown"}
+    Vendor: ${ua.device.vendor || "unknown"}
+Engine
+    Name: ${ua.engine.name || "unknown"}
+    Version: ${ua.engine.version || "unknown"}
+OS
+    Name: ${ua.os.name || "unknown"}
+    Version: ${ua.os.version || "unknown"}
+CPU
+    Architecture: ${ua.cpu.architecture || "unknown"}`;
     },
 
 

src/core/operations/Hash.js

@@ -1,5 +1,5 @@
 import Utils from "../Utils.js";
-import CryptoApi from "crypto-api";
+import CryptoApi from "babel-loader!crypto-api";
 import MD6 from "node-md6";
 import * as SHA3 from "js-sha3";
 import Checksum from "./Checksum.js";
@@ -21,14 +21,11 @@ const Hash = {
      *
      * @param {string} name
      * @param {string} input
+     * @para, {Object} [options={}]
      * @returns {string}
      */
-    runHash: function(name, input) {
-        const hasher = CryptoApi.hasher(name);
-        hasher.state.message = input;
-        hasher.state.length += input.length;
-        hasher.process();
-        return hasher.finalize().stringify("hex");
+    runHash: function(name, input, options={}) {
+        return CryptoApi.hash(name, input, options);
     },
 
 
@@ -329,9 +326,10 @@ const Hash = {
      * @returns {string}
      */
     runSnefru: function (input, args) {
-        const rounds = args[0],
-            size = args[1];
-        return Hash.runHash(`snefru-${rounds}-${size}`, input);
+        return Hash.runHash("snefru", input, {
+            rounds: args[0],
+            length: args[1]
+        });
     },
 
 
@@ -358,7 +356,8 @@ const Hash = {
         "HAS160",
         "Whirlpool",
         "Whirlpool-0",
-        "Whirlpool-T"
+        "Whirlpool-T",
+        "Snefru"
     ],
 
     /**
@@ -369,12 +368,17 @@ const Hash = {
      * @returns {string}
      */
     runHMAC: function (input, args) {
-        const password = args[0],
-            hashFunc = args[1].toLowerCase(),
-            hmac = CryptoApi.mac("hmac", password, hashFunc, {});
+        const key = args[0],
+            hashFunc = args[1].toLowerCase();
+        let hasher = CryptoApi.getHasher(hashFunc);
 
-        hmac.update(input);
-        return hmac.finalize().stringify("hex");
+        // Horrible shim to fix constructor bug. Reported in nf404/crypto-api#8
+        hasher.reset = () => {
+            hasher.state = {};
+            const tmp = new hasher.constructor();
+            hasher.state = tmp.state;
+        };
+        return CryptoApi.hmac(key, input, hasher);
     },
 
 

src/core/operations/MAC.js

@@ -34,6 +34,11 @@ const MAC = {
      * @default
      */
     CISCO_STYLE: false,
+    /**
+     * @constant
+     * @default
+     */
+    IPV6_INTERFACE_ID: false,
 
     /**
      * Format MAC addresses operation.
@@ -50,6 +55,7 @@ const MAC = {
             dashDelim = args[2],
             colonDelim = args[3],
             ciscoStyle = args[4],
+            ipv6IntID = args[5],
             outputList = [],
             macs = input.toLowerCase().split(/[,\s\r\n]+/);
 
@@ -57,23 +63,32 @@ const MAC = {
             let cleanMac = mac.replace(/[:.-]+/g, ""),
                 macHyphen = cleanMac.replace(/(.{2}(?=.))/g, "$1-"),
                 macColon = cleanMac.replace(/(.{2}(?=.))/g, "$1:"),
-                macCisco = cleanMac.replace(/(.{4}(?=.))/g, "$1.");
+                macCisco = cleanMac.replace(/(.{4}(?=.))/g, "$1."),
+                macIPv6 = cleanMac.slice(0, 6) + "fffe" + cleanMac.slice(6);
+
+            macIPv6 = macIPv6.replace(/(.{4}(?=.))/g, "$1:");
+            let bite = parseInt(macIPv6.slice(0, 2), 16) ^ 2;
+            bite = bite.toString(16).padStart(2, "0");
+            macIPv6 = bite + macIPv6.slice(2);
 
             if (outputCase === "Lower only") {
                 if (noDelim) outputList.push(cleanMac);
                 if (dashDelim) outputList.push(macHyphen);
                 if (colonDelim) outputList.push(macColon);
                 if (ciscoStyle) outputList.push(macCisco);
+                if (ipv6IntID) outputList.push(macIPv6);
             } else if (outputCase === "Upper only") {
                 if (noDelim) outputList.push(cleanMac.toUpperCase());
                 if (dashDelim) outputList.push(macHyphen.toUpperCase());
                 if (colonDelim) outputList.push(macColon.toUpperCase());
                 if (ciscoStyle) outputList.push(macCisco.toUpperCase());
+                if (ipv6IntID) outputList.push(macIPv6.toUpperCase());
             } else {
                 if (noDelim) outputList.push(cleanMac, cleanMac.toUpperCase());
                 if (dashDelim) outputList.push(macHyphen, macHyphen.toUpperCase());
                 if (colonDelim) outputList.push(macColon, macColon.toUpperCase());
                 if (ciscoStyle) outputList.push(macCisco, macCisco.toUpperCase());
+                if (ipv6IntID) outputList.push(macIPv6, macIPv6.toUpperCase());
             }
 
             outputList.push(

src/core/operations/Regex.js

@@ -32,7 +32,7 @@ const Regex = {
         },
         {
             name: "Email address",
-            value: "(\\w[-.\\w]*)@([-\\w]+(?:\\.[-\\w]+)*)\\.([A-Za-z]{2,4})"
+            value: "\\b(\\w[-.\\w]*)@([-\\w]+(?:\\.[-\\w]+)*)\\.([A-Za-z]{2,4})\\b"
         },
         {
             name: "URL",

src/core/operations/Unicode.js

@@ -27,7 +27,7 @@ const Unicode = {
      */
     runUnescape: function(input, args) {
         let prefix = Unicode._prefixToRegex[args[0]],
-            regex = new RegExp(prefix+"([a-f\\d]{4,6})", "ig"),
+            regex = new RegExp(prefix+"([a-f\\d]{4})", "ig"),
             output = "",
             m,
             i = 0;

src/web/App.js

@@ -377,6 +377,7 @@ App.prototype.loadURIParams = function() {
         window.location.href.split("#")[1] ||
         window.location.hash;
     this.uriParams = Utils.parseURIParams(params);
+    this.autoBakePause = true;
 
     // Read in recipe from URI params
     if (this.uriParams.recipe) {
@@ -387,35 +388,29 @@ App.prototype.loadURIParams = function() {
     } else if (this.uriParams.op) {
         // If there's no recipe, look for single operations
         this.manager.recipe.clearRecipe();
-        try {
-            this.manager.recipe.addOperation(this.uriParams.op);
-        } catch (err) {
-            // If no exact match, search for nearest match and add that
-            const matchedOps = this.manager.ops.filterOperations(this.uriParams.op, false);
-            if (matchedOps.length) {
-                this.manager.recipe.addOperation(matchedOps[0].name);
-            }
 
-            // Populate search with the string
-            const search = document.getElementById("search");
-
-            search.value = this.uriParams.op;
-            search.dispatchEvent(new Event("search"));
+        // Search for nearest match and add it
+        const matchedOps = this.manager.ops.filterOperations(this.uriParams.op, false);
+        if (matchedOps.length) {
+            this.manager.recipe.addOperation(matchedOps[0].name);
         }
+
+        // Populate search with the string
+        const search = document.getElementById("search");
+
+        search.value = this.uriParams.op;
+        search.dispatchEvent(new Event("search"));
     }
 
     // Read in input data from URI params
     if (this.uriParams.input) {
-        this.autoBakePause = true;
         try {
             const inputData = Utils.fromBase64(this.uriParams.input);
             this.setInput(inputData);
-        } catch (err) {
-        } finally {
-            this.autoBakePause = false;
-        }
+        } catch (err) {}
     }
 
+    this.autoBakePause = false;
     this.autoBake();
 };
 

src/web/html/index.html

@@ -88,6 +88,7 @@
             window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 2000) + 1500);
         </script>
         <% if (!htmlWebpackPlugin.options.inline) { %>
+            <meta name="robots" content="noindex" />
             <script type="application/ld+json">
                 <% print(JSON.stringify(require("../static/structuredData.json"))); %>
             </script>

src/web/static/sitemap.js

@@ -0,0 +1,33 @@
+import sm from "sitemap";
+import OperationConfig from "../../core/config/MetaConfig.js";
+
+
+/**
+ * Generates an XML sitemap for all CyberChef operations and a number of recipes.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+const sitemap = sm.createSitemap({
+    hostname: "https://gchq.github.io/CyberChef",
+});
+
+sitemap.add({
+    url: "/",
+    changefreq: "weekly",
+    priority: 1.0
+});
+
+for (let op in OperationConfig) {
+    sitemap.add({
+        url: `/?op=${encodeURIComponent(op)}`,
+        changeFreq: "yearly",
+        priority: 0.5
+    });
+}
+
+const xml = sitemap.toString();
+
+console.log(xml); // eslint-disable-line no-console

webpack.config.js

@@ -109,9 +109,13 @@ module.exports = {
         children: false,
         chunks: false,
         modules: false,
+        entrypoints: false,
         warningsFilter: /source-map/,
     },
     node: {
         fs: "empty"
+    },
+    performance: {
+        hints: false
     }
 };