8.1.2

This decoder will join up a HTTP response sent using chunked transfer encoding, raised in issue #168.

This is useful when attempting to extract files or gzipped responses sent using chunked transfer encoding, particularly when combined with the gunzip operation.

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+### [8.1.0] - 2018-08-19
+- 'Dechunk HTTP response' operation added @sevzero #311
+
 ## [8.0.0] - 2018-08-05
 - Codebase rewritten using [ES modules](https://hacks.mozilla.org/2018/03/es-modules-a-cartoon-deep-dive/) and [classes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes) #284
 - Operation architecture restructured to make adding new operations a lot simpler #284
@@ -29,6 +32,7 @@ All notable changes to this project will be documented in this file.
 -  Initial open source commit @n1474335
 
 
+[8.1.0]: https://github.com/gchq/CyberChef/releases/tag/v8.1.0
 [8.0.0]: https://github.com/gchq/CyberChef/releases/tag/v8.0.0
 [7.0.0]: https://github.com/gchq/CyberChef/releases/tag/v7.0.0
 [6.0.0]: https://github.com/gchq/CyberChef/releases/tag/v6.0.0

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.0.2",
+  "version": "8.1.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.0.2",
+  "version": "8.1.2",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/Utils.mjs

@@ -8,6 +8,7 @@ import utf8 from "utf8";
 import moment from "moment-timezone";
 import {fromBase64} from "./lib/Base64";
 import {fromHex} from "./lib/Hex";
+import {fromDecimal} from "./lib/Decimal";
 
 
 /**
@@ -297,7 +298,7 @@ class Utils {
      * Accepts hex, Base64, UTF8 and Latin1 strings.
      *
      * @param {string} str
-     * @param {string} type - One of "Hex", "Base64", "UTF8" or "Latin1"
+     * @param {string} type - One of "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
      * @returns {byteArray}
      *
      * @example
@@ -314,6 +315,8 @@ class Utils {
         switch (type.toLowerCase()) {
             case "hex":
                 return fromHex(str);
+            case "decimal":
+                return fromDecimal(str);
             case "base64":
                 return fromBase64(str, null, "byteArray");
             case "utf8":
@@ -330,7 +333,7 @@ class Utils {
      * Accepts hex, Base64, UTF8 and Latin1 strings.
      *
      * @param {string} str
-     * @param {string} type - One of "Hex", "Base64", "UTF8" or "Latin1"
+     * @param {string} type - One of "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
      * @returns {string}
      *
      * @example
@@ -347,6 +350,8 @@ class Utils {
         switch (type.toLowerCase()) {
             case "hex":
                 return Utils.byteArrayToChars(fromHex(str));
+            case "decimal":
+                return Utils.byteArrayToChars(fromDecimal(str));
             case "base64":
                 return Utils.byteArrayToChars(fromBase64(str, null, "byteArray"));
             case "utf8":

src/core/config/Categories.json

@@ -137,6 +137,7 @@
         "ops": [
             "HTTP request",
             "Strip HTTP headers",
+            "Dechunk HTTP response",
             "Parse User Agent",
             "Parse IP range",
             "Parse IPv6 address",

src/core/lib/Decimal.mjs

@@ -0,0 +1,37 @@
+/**
+ * Decimal functions.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Utils from "../Utils";
+
+
+/**
+ * Convert a string of decimal values into a byte array.
+ *
+ * @param {string} data
+ * @param {string} [delim]
+ * @returns {byteArray}
+ *
+ * @example
+ * // returns [10,20,30]
+ * fromDecimal("10 20 30");
+ *
+ * // returns [10,20,30]
+ * fromDecimal("10:20:30", "Colon");
+ */
+export function fromDecimal(data, delim="Auto") {
+    delim = Utils.charRep(delim);
+    const output = [];
+    let byteStr = data.split(delim);
+    if (byteStr[byteStr.length-1] === "")
+        byteStr = byteStr.slice(0, byteStr.length-1);
+
+    for (let i = 0; i < byteStr.length; i++) {
+        output[i] = parseInt(byteStr[i], 10);
+    }
+    return output;
+}

src/core/lib/Hex.mjs

@@ -1,5 +1,5 @@
 /**
- * Byte representation functions.
+ * Hexadecimal functions.
  *
  * @author n1474335 [n1474335@gmail.com]
  * @copyright Crown Copyright 2016
@@ -83,8 +83,7 @@ export function toHexFast(data) {
  * // returns [10,20,30]
  * fromHex("0a:14:1e", "Colon");
  */
-export function fromHex(data, delim, byteLen=2) {
-    delim = delim || "Auto";
+export function fromHex(data, delim="Auto", byteLen=2) {
     if (delim !== "None") {
         const delimRegex = delim === "Auto" ? /[^a-f\d]/gi : Utils.regexRep(delim);
         data = data.replace(delimRegex, "");

src/core/operations/ADD.mjs

@@ -29,7 +29,7 @@ class ADD extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Base64", "UTF8", "Latin1"]
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             }
         ];
     }

src/core/operations/AND.mjs

@@ -29,7 +29,7 @@ class AND extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Base64", "UTF8", "Latin1"]
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             }
         ];
     }

src/core/operations/ChangeIPFormat.mjs

@@ -21,7 +21,7 @@ class ChangeIPFormat extends Operation {
         super();
 
         this.name = "Change IP format";
-        this.module = "JSBN";
+        this.module = "Default";
         this.description = "Convert an IP address from one format to another, e.g. <code>172.20.23.54</code> to <code>ac141736</code>";
         this.inputType = "string";
         this.outputType = "string";
@@ -29,12 +29,12 @@ class ChangeIPFormat extends Operation {
             {
                 "name": "Input format",
                 "type": "option",
-                "value": ["Hex", "Raw"]
+                "value": ["Dotted Decimal", "Decimal", "Hex"]
             },
             {
                 "name": "Output format",
                 "type": "option",
-                "value": ["Hex", "Raw"]
+                "value": ["Dotted Decimal", "Decimal", "Hex"]
             }
         ];
     }

src/core/operations/DechunkHTTPResponse.mjs

@@ -0,0 +1,50 @@
+/**
+ * @author sevzero [sevzero@protonmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * Dechunk HTTP response operation
+ */
+class DechunkHTTPResponse extends Operation {
+
+    /**
+     * DechunkHTTPResponse constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Dechunk HTTP response";
+        this.module = "Default";
+        this.description = "Parses an HTTP response transferred using Transfer-Encoding: Chunked";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const chunks = [];
+        let chunkSizeEnd = input.indexOf("\n") + 1;
+        const lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n";
+        const lineEndingsLength = lineEndings.length;
+        let chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        while (!isNaN(chunkSize)) {
+            chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd));
+            input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength);
+            chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength;
+            chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        }
+        return chunks.join("") + input;
+    }
+
+}
+
+export default DechunkHTTPResponse;

src/core/operations/FromDecimal.mjs

@@ -5,8 +5,8 @@
  */
 
 import Operation from "../Operation";
-import Utils from "../Utils";
 import {DELIM_OPTIONS} from "../lib/Delim";
+import {fromDecimal} from "../lib/Decimal";
 
 /**
  * From Decimal operation
@@ -71,16 +71,7 @@ class FromDecimal extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        const delim = Utils.charRep(args[0]),
-            output = [];
-        let byteStr = input.split(delim);
-        if (byteStr[byteStr.length-1] === "")
-            byteStr = byteStr.slice(0, byteStr.length-1);
-
-        for (let i = 0; i < byteStr.length; i++) {
-            output[i] = parseInt(byteStr[i], 10);
-        }
-        return output;
+        return fromDecimal(input, args[0]);
     }
 
 }

src/core/operations/FromHex.mjs

@@ -86,7 +86,7 @@ class FromHex extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        const delim = args[0] || "Space";
+        const delim = args[0] || "Auto";
         return fromHex(input, delim, 2);
     }
 

src/core/operations/OR.mjs

@@ -29,7 +29,7 @@ class OR extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Base64", "UTF8", "Latin1"]
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             }
         ];
     }

src/core/operations/SUB.mjs

@@ -29,7 +29,7 @@ class SUB extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Base64", "UTF8", "Latin1"]
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             }
         ];
     }

src/core/operations/XOR.mjs

@@ -29,7 +29,7 @@ class XOR extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Base64", "UTF8", "Latin1"]
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             },
             {
                 "name": "Scheme",