5.9.0

Add repo option to Travis

.travis.yml

@@ -19,6 +19,7 @@ deploy:
     local_dir: build/prod/
     target_branch: gh-pages
     on:
+      repo: gchq/CyberChef
       branch: master
   - provider: releases
     skip_cleaup: true

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "5.7.2",
+  "version": "5.9.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/config/Categories.js

@@ -126,6 +126,7 @@ const Categories = [
     {
         name: "Networking",
         ops: [
+            "HTTP request",
             "Strip HTTP headers",
             "Parse User Agent",
             "Parse IP range",
@@ -288,6 +289,8 @@ const Categories = [
             "Scan for Embedded Files",
             "Generate UUID",
             "Render Image",
+            "Remove EXIF",
+            "Extract EXIF",
             "Numberwang",
         ]
     },

src/core/config/OperationConfig.js

@@ -3370,7 +3370,7 @@ const OperationConfig = {
             "<br><br>",
             "EXIF data from photos usually contains information about the image file itself as well as the device used to create it.",
         ].join("\n"),
-        run: Image.runEXIF,
+        run: Image.runExtractEXIF,
         inputType: "byteArray",
         outputType: "string",
         args: [],
@@ -3388,6 +3388,59 @@ const OperationConfig = {
             }
         ]
     },
+    "Remove EXIF": {
+        description: [
+            "Removes EXIF data from a JPEG image.",
+            "<br><br>",
+            "EXIF data embedded in photos usually contains information about the image file itself as well as the device used to create it.",
+        ].join("\n"),
+        run: Image.runRemoveEXIF,
+        inputType: "byteArray",
+        outputType: "byteArray",
+        args: []
+    },
+    "HTTP request": {
+        description: [
+            "Makes an HTTP request and returns the response.",
+            "<br><br>",
+            "This operation supports different HTTP verbs like GET, POST, PUT, etc.",
+            "<br><br>",
+            "You can add headers line by line in the format <code>Key: Value</code>",
+            "<br><br>",
+            "The status code of the response, along with a limited selection of exposed headers, can be viewed by checking the 'Show response metadata' option. Only a limited set of response headers are exposed by the browser for security reasons.",
+        ].join("\n"),
+        run: HTTP.runHTTPRequest,
+        inputType: "string",
+        outputType: "string",
+        manualBake: true,
+        args: [
+            {
+                name: "Method",
+                type: "option",
+                value: HTTP.METHODS,
+            },
+            {
+                name: "URL",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Headers",
+                type: "text",
+                value: "",
+            },
+            {
+                name: "Mode",
+                type: "option",
+                value: HTTP.MODE,
+            },
+            {
+                name: "Show response metadata",
+                type: "boolean",
+                value: false,
+            }
+        ]
+    },
 };
 
 export default OperationConfig;

src/core/lib/remove-exif.js

@@ -0,0 +1,153 @@
+/* piexifjs
+The MIT License (MIT)
+Copyright (c) 2014, 2015 hMatoba(https://github.com/hMatoba)
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+import Utils from "../Utils.js";
+
+// Param jpeg should be a binaryArray
+function removeEXIF(jpeg) {
+  // Convert binaryArray to char string
+  jpeg = Utils.byteArrayToChars(jpeg);
+  if (jpeg.slice(0, 2) != "\xff\xd8") {
+    throw ("Given data is not jpeg.");
+  }
+
+  var segments = splitIntoSegments(jpeg);
+  if (segments[1].slice(0, 2) == "\xff\xe1" &&
+    segments[1].slice(4, 10) == "Exif\x00\x00") {
+    segments = [segments[0]].concat(segments.slice(2));
+  } else if (segments[2].slice(0, 2) == "\xff\xe1" &&
+    segments[2].slice(4, 10) == "Exif\x00\x00") {
+    segments = segments.slice(0, 2).concat(segments.slice(3));
+  } else {
+    throw ("Exif not found.");
+  }
+
+  var new_data = segments.join("");
+
+  // Convert back to binaryArray
+  new_data = Utils.strToCharcode(new_data);
+
+  return new_data;
+};
+
+function splitIntoSegments(data) {
+  if (data.slice(0, 2) != "\xff\xd8") {
+    throw ("Given data isn't JPEG.");
+  }
+
+  var head = 2;
+  var segments = ["\xff\xd8"];
+  while (true) {
+    if (data.slice(head, head + 2) == "\xff\xda") {
+      segments.push(data.slice(head));
+      break;
+    } else {
+      var length = unpack(">H", data.slice(head + 2, head + 4))[0];
+      var endPoint = head + length + 2;
+      segments.push(data.slice(head, endPoint));
+      head = endPoint;
+    }
+
+    if (head >= data.length) {
+      throw ("Wrong JPEG data.");
+    }
+  }
+  return segments;
+}
+
+function unpack(mark, str) {
+  if (typeof(str) != "string") {
+    throw ("'unpack' error. Got invalid type argument.");
+  }
+  var l = 0;
+  for (var markPointer = 1; markPointer < mark.length; markPointer++) {
+    if (mark[markPointer].toLowerCase() == "b") {
+      l += 1;
+    } else if (mark[markPointer].toLowerCase() == "h") {
+      l += 2;
+    } else if (mark[markPointer].toLowerCase() == "l") {
+      l += 4;
+    } else {
+      throw ("'unpack' error. Got invalid mark.");
+    }
+  }
+
+  if (l != str.length) {
+    throw ("'unpack' error. Mismatch between symbol and string length. " + l + ":" + str.length);
+  }
+
+  var littleEndian;
+  if (mark[0] == "<") {
+    littleEndian = true;
+  } else if (mark[0] == ">") {
+    littleEndian = false;
+  } else {
+    throw ("'unpack' error.");
+  }
+  var unpacked = [];
+  var strPointer = 0;
+  var p = 1;
+  var val = null;
+  var c = null;
+  var length = null;
+  var sliced = "";
+
+  while (c = mark[p]) {
+    if (c.toLowerCase() == "b") {
+      length = 1;
+      sliced = str.slice(strPointer, strPointer + length);
+      val = sliced.charCodeAt(0);
+      if ((c == "b") && (val >= 0x80)) {
+        val -= 0x100;
+      }
+    } else if (c == "H") {
+      length = 2;
+      sliced = str.slice(strPointer, strPointer + length);
+      if (littleEndian) {
+        sliced = sliced.split("").reverse().join("");
+      }
+      val = sliced.charCodeAt(0) * 0x100 +
+        sliced.charCodeAt(1);
+    } else if (c.toLowerCase() == "l") {
+      length = 4;
+      sliced = str.slice(strPointer, strPointer + length);
+      if (littleEndian) {
+        sliced = sliced.split("").reverse().join("");
+      }
+      val = sliced.charCodeAt(0) * 0x1000000 +
+        sliced.charCodeAt(1) * 0x10000 +
+        sliced.charCodeAt(2) * 0x100 +
+        sliced.charCodeAt(3);
+      if ((c == "l") && (val >= 0x80000000)) {
+        val -= 0x100000000;
+      }
+    } else {
+      throw ("'unpack' error. " + c);
+    }
+
+    unpacked.push(val);
+    strPointer += length;
+    p += 1;
+  }
+
+  return unpacked;
+}
+
+export default removeEXIF;

src/core/operations/HTTP.js

@@ -12,6 +12,17 @@ import {UAS_parser as UAParser} from "../lib/uas_parser.js";
  */
 const HTTP = {
 
+    /**
+     * @constant
+     * @default
+     */
+    METHODS: [
+        "GET", "POST", "HEAD",
+        "PUT", "PATCH", "DELETE",
+        "CONNECT", "TRACE", "OPTIONS"
+    ],
+
+
     /**
      * Strip HTTP headers operation.
      *
@@ -51,6 +62,94 @@ const HTTP = {
             "Device Type: " + ua.deviceType + "\n";
     },
 
+
+    /**
+     * @constant
+     * @default
+     */
+    MODE: [
+        "Cross-Origin Resource Sharing",
+        "No CORS (limited to HEAD, GET or POST)",
+    ],
+
+    /**
+     * Lookup table for HTTP modes
+     *
+     * @private
+     * @constant
+     */
+    _modeLookup: {
+        "Cross-Origin Resource Sharing": "cors",
+        "No CORS (limited to HEAD, GET or POST)": "no-cors",
+    },
+
+    /**
+     * HTTP request operation.
+     *
+     * @author tlwr [toby@toby.codes]
+     * @author n1474335 [n1474335@gmail.com]
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runHTTPRequest(input, args) {
+        const method = args[0],
+            url = args[1],
+            headersText = args[2],
+            mode = args[3],
+            showResponseMetadata = args[4];
+
+        if (url.length === 0) return "";
+
+        let headers = new Headers();
+        headersText.split(/\r?\n/).forEach(line => {
+            line = line.trim();
+
+            if (line.length === 0) return;
+
+            let split = line.split(":");
+            if (split.length !== 2) throw `Could not parse header in line: ${line}`;
+
+            headers.set(split[0].trim(), split[1].trim());
+        });
+
+        let config = {
+            method: method,
+            headers: headers,
+            mode: HTTP._modeLookup[mode],
+            cache: "no-cache",
+        };
+
+        if (method !== "GET" && method !== "HEAD") {
+            config.body = input;
+        }
+
+        return fetch(url, config)
+        .then(r => {
+            if (r.status === 0 && r.type === "opaque") {
+                return "Error: Null response. Try setting the connection mode to CORS.";
+            }
+
+            if (showResponseMetadata) {
+                let headers = "";
+                for (let pair of r.headers.entries()) {
+                    headers += "    " + pair[0] + ": " + pair[1] + "\n";
+                }
+                return r.text().then(b => {
+                    return "####\n  Status: " + r.status + " " + r.statusText +
+                        "\n  Exposed headers:\n" + headers + "####\n\n" + b;
+                });
+            }
+            return r.text();
+        })
+        .catch(e => {
+            return e.toString() +
+                "\n\nThis error could be caused by one of the following:\n" +
+                " - An invalid URL\n" +
+                " - Making a cross-origin request to a server which does not support CORS\n";
+        });
+    },
+
 };
 
 export default HTTP;

src/core/operations/Image.js

@@ -1,4 +1,5 @@
 import * as ExifParser from "exif-parser";
+import removeEXIF from "../lib/remove-exif.js";
 import Utils from "../Utils.js";
 import FileType from "./FileType.js";
 
@@ -23,7 +24,7 @@ const Image = {
      * @param {Object[]} args
      * @returns {string}
      */
-    runEXIF(input, args) {
+    runExtractEXIF(input, args) {
         try {
             const bytes = Uint8Array.from(input);
             const parser = ExifParser.create(bytes.buffer);
@@ -44,6 +45,30 @@ const Image = {
     },
 
 
+    /**
+     * Remove EXIF operation.
+     *
+     * Removes EXIF data from a byteArray, representing a JPG.
+     *
+     * @author David Moodie [davidmoodie12@gmail.com]
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runRemoveEXIF(input, args) {
+        // Do nothing if input is empty
+        if (input.length === 0) return input;
+
+        try {
+            return removeEXIF(input);
+        } catch (err) {
+            // Simply return input if no EXIF data is found
+            if (err === "Exif not found.") return input;
+            throw "Could not remove EXIF data from image: " + err;
+        }
+    },
+
+
     /**
      * @constant
      * @default

src/core/operations/SeqUtils.js

@@ -26,7 +26,7 @@ const SeqUtils = {
      * @constant
      * @default
      */
-    SORT_ORDER: ["Alphabetical (case sensitive)", "Alphabetical (case insensitive)", "IP address"],
+    SORT_ORDER: ["Alphabetical (case sensitive)", "Alphabetical (case insensitive)", "IP address", "Numeric"],
 
     /**
      * Sort operation.
@@ -47,6 +47,8 @@ const SeqUtils = {
             sorted = sorted.sort(SeqUtils._caseInsensitiveSort);
         } else if (order === "IP address") {
             sorted = sorted.sort(SeqUtils._ipSort);
+        } else if (order === "Numeric") {
+            sorted = sorted.sort(SeqUtils._numericSort);
         }
 
         if (sortReverse) sorted.reverse();
@@ -221,6 +223,35 @@ const SeqUtils = {
         return a_ - b_;
     },
 
+
+    /**
+     * Comparison operation for sorting of numeric values.
+     *
+     * @author Chris van Marle
+     * @private
+     * @param {string} a
+     * @param {string} b
+     * @returns {number}
+     */
+    _numericSort: function _numericSort(a, b) {
+        let a_ = a.split(/([^\d]+)/),
+            b_ = b.split(/([^\d]+)/);
+
+        for (let i = 0; i < a_.length && i < b.length; ++i) {
+            if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+            if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+            if (isNaN(a_[i]) && isNaN(b_[i])) {
+                let ret = a_[i].localeCompare(b_[i]); // Compare strings
+                if (ret !== 0) return ret;
+            }
+            if (!isNaN(a_[i]) && !isNaN(a_[i])) { // Compare numbers
+                if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+            }
+        }
+
+        return 0;
+    },
+
 };
 
 export default SeqUtils;

src/web/ControlsWaiter.js

@@ -355,9 +355,11 @@ ControlsWaiter.prototype.supportButtonClick = function() {
     const reportBugInfo = document.getElementById("report-bug-info");
     const saveLink = this.generateStateUrl(true, true, null, "https://gchq.github.io/CyberChef/");
 
-    reportBugInfo.innerHTML = "* CyberChef compile time: " + COMPILE_TIME + "\n" +
-        "* User-Agent: \n" + navigator.userAgent + "\n" +
-        "* [Link to reproduce](" + saveLink + ")\n\n";
+    if (reportBugInfo) {
+        reportBugInfo.innerHTML = "* CyberChef compile time: " + COMPILE_TIME + "\n" +
+            "* User-Agent: \n" + navigator.userAgent + "\n" +
+            "* [Link to reproduce](" + saveLink + ")\n\n";
+    }
 };
 
 export default ControlsWaiter;

src/web/OptionsWaiter.js

@@ -47,6 +47,8 @@ OptionsWaiter.prototype.load = function(options) {
         if (val) {
             selects[i].value = val;
             selects[i].dispatchEvent(new CustomEvent("change", {bubbles: true}));
+        } else {
+            selects[i].selectedIndex = 0;
         }
     }
 };

src/web/stylesheets/vendors/bootstrap.less

@@ -19,7 +19,7 @@
 @import "~bootstrap/less/scaffolding.less";
 @import "~bootstrap/less/type.less";
 @import "~bootstrap/less/code.less";
-// @import "~bootstrap/less/grid.less";
+@import "~bootstrap/less/grid.less";
 @import "~bootstrap/less/tables.less";
 @import "~bootstrap/less/forms.less";
 @import "~bootstrap/less/buttons.less";

test/index.js

@@ -20,6 +20,7 @@ import "./tests/operations/FlowControl.js";
 import "./tests/operations/Image.js";
 import "./tests/operations/MorseCode.js";
 import "./tests/operations/StrUtils.js";
+import "./tests/operations/SeqUtils.js";
 
 let allTestsPassing = true;
 const testStatusCounts = {

test/tests/operations/SeqUtils.js

@@ -0,0 +1,33 @@
+/**
+ * SeqUtils tests.
+ *
+ * @author Chris van Marle
+ * @copyright Copyright 2017
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister.js";
+
+TestRegister.addTests([
+    {
+        name: "SeqUtils - Numeric sort photos",
+        input: "Photo-1.jpg\nPhoto-4.jpg\nPhoto-2.jpg\nPhoto-3.jpg\n",
+        expectedOutput: "Photo-1.jpg\nPhoto-2.jpg\nPhoto-3.jpg\nPhoto-4.jpg\n",
+        recipeConfig: [
+            {
+                "op": "Sort",
+                "args": ["Line feed", false, "Numeric"]
+            }
+        ],
+    },
+    {
+        name: "SeqUtils - Numeric sort CVE IDs",
+        input: "CVE-2017-1234,CVE-2017-9999,CVE-2017-10000,CVE-2017-10001,CVE-2017-12345,CVE-2016-1234,CVE-2016-4321,CVE-2016-10000,CVE-2016-9999,CVE-2016-10001",
+        expectedOutput: "CVE-2017-12345,CVE-2017-10001,CVE-2017-10000,CVE-2017-9999,CVE-2017-1234,CVE-2016-10001,CVE-2016-10000,CVE-2016-9999,CVE-2016-4321,CVE-2016-1234",
+        recipeConfig: [
+            {
+                "op": "Sort",
+                "args": ["Comma", true, "Numeric"]
+            }
+        ],
+    },
+]);