9.12.0

Certain RTF files may attempt to thwart detection by having a malformed RTF header, such as **{\rt000**. Removing 0x66 will result in detecting these malformed yet valid RTFs as well.

Additional reading:
https://www.decalage.info/rtf_tricks#Trick_1:_Incomplete_RTF_Header

CHANGELOG.md

@@ -2,6 +2,9 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [9.12.0] - 2019-12-20
+- 'Normalise Unicode' operation added [@matthieuxyz] | [#912]
+
 ### [9.11.0] - 2019-11-06
 - Implemented CFB, OFB, and CTR modes for Blowfish operations [@cbeuw] | [#653]
 
@@ -197,6 +200,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[9.12.0]: https://github.com/gchq/CyberChef/releases/tag/v9.12.0
 [9.11.0]: https://github.com/gchq/CyberChef/releases/tag/v9.11.0
 [9.10.0]: https://github.com/gchq/CyberChef/releases/tag/v9.10.0
 [9.9.0]: https://github.com/gchq/CyberChef/releases/tag/v9.9.0
@@ -281,6 +285,7 @@ All major and minor version changes will be documented in this file. Details of
 [@jarrodconnolly]: https://github.com/jarrodconnolly
 [@VirtualColossus]: https://github.com/VirtualColossus
 [@cbeuw]: https://github.com/cbeuw
+[@matthieuxyz]: https://github.com/matthieuxyz
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -344,3 +349,4 @@ All major and minor version changes will be documented in this file. Details of
 [#632]: https://github.com/gchq/CyberChef/pull/632
 [#653]: https://github.com/gchq/CyberChef/pull/653
 [#865]: https://github.com/gchq/CyberChef/pull/865
+[#912]: https://github.com/gchq/CyberChef/pull/912

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.11.18",
+  "version": "9.12.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -7118,28 +7118,28 @@
       "dependencies": {
         "abbrev": {
           "version": "1.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
           "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
           "dev": true,
           "optional": true
         },
         "ansi-regex": {
           "version": "2.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
           "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
           "dev": true,
           "optional": true
         },
         "aproba": {
           "version": "1.2.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
           "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
           "dev": true,
           "optional": true
         },
         "are-we-there-yet": {
           "version": "1.1.5",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
           "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
           "dev": true,
           "optional": true,
@@ -7150,14 +7150,14 @@
         },
         "balanced-match": {
           "version": "1.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
           "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
           "dev": true,
           "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
           "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
           "dev": true,
           "optional": true,
@@ -7168,42 +7168,42 @@
         },
         "chownr": {
           "version": "1.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
           "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==",
           "dev": true,
           "optional": true
         },
         "code-point-at": {
           "version": "1.1.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
           "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
           "dev": true,
           "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
           "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
           "dev": true,
           "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
           "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
           "dev": true,
           "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
           "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
           "dev": true,
           "optional": true
         },
         "debug": {
           "version": "4.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
           "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
           "dev": true,
           "optional": true,
@@ -7213,28 +7213,28 @@
         },
         "deep-extend": {
           "version": "0.6.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
           "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
           "dev": true,
           "optional": true
         },
         "delegates": {
           "version": "1.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
           "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
           "dev": true,
           "optional": true
         },
         "detect-libc": {
           "version": "1.0.3",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
           "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=",
           "dev": true,
           "optional": true
         },
         "fs-minipass": {
           "version": "1.2.5",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz",
           "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==",
           "dev": true,
           "optional": true,
@@ -7244,14 +7244,14 @@
         },
         "fs.realpath": {
           "version": "1.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
           "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
           "dev": true,
           "optional": true
         },
         "gauge": {
           "version": "2.7.4",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
           "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
           "dev": true,
           "optional": true,
@@ -7268,7 +7268,7 @@
         },
         "glob": {
           "version": "7.1.3",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
           "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
           "dev": true,
           "optional": true,
@@ -7283,14 +7283,14 @@
         },
         "has-unicode": {
           "version": "2.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
           "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
           "dev": true,
           "optional": true
         },
         "iconv-lite": {
           "version": "0.4.24",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
           "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
           "dev": true,
           "optional": true,
@@ -7300,7 +7300,7 @@
         },
         "ignore-walk": {
           "version": "3.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz",
           "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==",
           "dev": true,
           "optional": true,
@@ -7310,7 +7310,7 @@
         },
         "inflight": {
           "version": "1.0.6",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
           "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
           "dev": true,
           "optional": true,
@@ -7321,21 +7321,21 @@
         },
         "inherits": {
           "version": "2.0.3",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
           "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
           "dev": true,
           "optional": true
         },
         "ini": {
           "version": "1.3.5",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
           "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
           "dev": true,
           "optional": true
         },
         "is-fullwidth-code-point": {
           "version": "1.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
           "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
           "dev": true,
           "optional": true,
@@ -7345,14 +7345,14 @@
         },
         "isarray": {
           "version": "1.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
           "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
           "dev": true,
           "optional": true
         },
         "minimatch": {
           "version": "3.0.4",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
           "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
           "dev": true,
           "optional": true,
@@ -7362,14 +7362,14 @@
         },
         "minimist": {
           "version": "0.0.8",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
           "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
           "dev": true,
           "optional": true
         },
         "minipass": {
           "version": "2.3.5",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
           "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
           "dev": true,
           "optional": true,
@@ -7380,7 +7380,7 @@
         },
         "minizlib": {
           "version": "1.2.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz",
           "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==",
           "dev": true,
           "optional": true,
@@ -7390,7 +7390,7 @@
         },
         "mkdirp": {
           "version": "0.5.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
           "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
           "dev": true,
           "optional": true,
@@ -7400,14 +7400,14 @@
         },
         "ms": {
           "version": "2.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
           "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
           "dev": true,
           "optional": true
         },
         "needle": {
           "version": "2.3.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.0.tgz",
           "integrity": "sha512-QBZu7aAFR0522EyaXZM0FZ9GLpq6lvQ3uq8gteiDUp7wKdy0lSd2hPlgFwVuW1CBkfEs9PfDQsQzZghLs/psdg==",
           "dev": true,
           "optional": true,
@@ -7419,7 +7419,7 @@
         },
         "node-pre-gyp": {
           "version": "0.12.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz",
           "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==",
           "dev": true,
           "optional": true,
@@ -7438,7 +7438,7 @@
         },
         "nopt": {
           "version": "4.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz",
           "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=",
           "dev": true,
           "optional": true,
@@ -7449,14 +7449,14 @@
         },
         "npm-bundled": {
           "version": "1.0.6",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz",
           "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==",
           "dev": true,
           "optional": true
         },
         "npm-packlist": {
           "version": "1.4.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz",
           "integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==",
           "dev": true,
           "optional": true,
@@ -7467,7 +7467,7 @@
         },
         "npmlog": {
           "version": "4.1.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
           "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
           "dev": true,
           "optional": true,
@@ -7480,21 +7480,21 @@
         },
         "number-is-nan": {
           "version": "1.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
           "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
           "dev": true,
           "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
           "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=",
           "dev": true,
           "optional": true
         },
         "once": {
           "version": "1.4.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
           "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
           "dev": true,
           "optional": true,
@@ -7504,21 +7504,21 @@
         },
         "os-homedir": {
           "version": "1.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
           "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=",
           "dev": true,
           "optional": true
         },
         "os-tmpdir": {
           "version": "1.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
           "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
           "dev": true,
           "optional": true
         },
         "osenv": {
           "version": "0.1.5",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
           "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
           "dev": true,
           "optional": true,
@@ -7529,21 +7529,21 @@
         },
         "path-is-absolute": {
           "version": "1.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
           "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
           "dev": true,
           "optional": true
         },
         "process-nextick-args": {
           "version": "2.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
           "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
           "dev": true,
           "optional": true
         },
         "rc": {
           "version": "1.2.8",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
           "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
           "dev": true,
           "optional": true,
@@ -7556,7 +7556,7 @@
           "dependencies": {
             "minimist": {
               "version": "1.2.0",
-              "resolved": false,
+              "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
               "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
               "dev": true,
               "optional": true
@@ -7565,7 +7565,7 @@
         },
         "readable-stream": {
           "version": "2.3.6",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
           "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
           "dev": true,
           "optional": true,
@@ -7581,7 +7581,7 @@
         },
         "rimraf": {
           "version": "2.6.3",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
           "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
           "dev": true,
           "optional": true,
@@ -7591,49 +7591,49 @@
         },
         "safe-buffer": {
           "version": "5.1.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
           "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
           "dev": true,
           "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
           "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
           "dev": true,
           "optional": true
         },
         "sax": {
           "version": "1.2.4",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
           "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
           "dev": true,
           "optional": true
         },
         "semver": {
           "version": "5.7.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
           "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
           "dev": true,
           "optional": true
         },
         "set-blocking": {
           "version": "2.0.0",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
           "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
           "dev": true,
           "optional": true
         },
         "signal-exit": {
           "version": "3.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
           "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
           "dev": true,
           "optional": true
         },
         "string-width": {
           "version": "1.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
           "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
           "dev": true,
           "optional": true,
@@ -7645,7 +7645,7 @@
         },
         "string_decoder": {
           "version": "1.1.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
           "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
           "dev": true,
           "optional": true,
@@ -7655,7 +7655,7 @@
         },
         "strip-ansi": {
           "version": "3.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
           "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
           "dev": true,
           "optional": true,
@@ -7665,14 +7665,14 @@
         },
         "strip-json-comments": {
           "version": "2.0.1",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
           "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
           "dev": true,
           "optional": true
         },
         "tar": {
           "version": "4.4.8",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
           "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
           "dev": true,
           "optional": true,
@@ -7688,14 +7688,14 @@
         },
         "util-deprecate": {
           "version": "1.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
           "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
           "dev": true,
           "optional": true
         },
         "wide-align": {
           "version": "1.1.3",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
           "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
           "dev": true,
           "optional": true,
@@ -7705,14 +7705,14 @@
         },
         "wrappy": {
           "version": "1.0.2",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
           "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
           "dev": true,
           "optional": true
         },
         "yallist": {
           "version": "3.0.3",
-          "resolved": false,
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
           "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
           "dev": true,
           "optional": true
@@ -14947,6 +14947,11 @@
         "normalize-path": "^2.1.1"
       }
     },
+    "unorm": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/unorm/-/unorm-1.6.0.tgz",
+      "integrity": "sha512-b2/KCUlYZUeA7JFUuRJZPUtr4gZvBh7tavtv4fvk4+KV9pfGiR6CQAQAWl49ZpR3ts2dk4FYkP7EIgDJoiOLDA=="
+    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.11.18",
+  "version": "9.12.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -147,6 +147,7 @@
     "ssdeep.js": "0.0.2",
     "tesseract.js": "^2.0.0-alpha.15",
     "ua-parser-js": "^0.7.20",
+    "unorm": "^1.6.0",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",

src/core/config/Categories.json

@@ -39,6 +39,7 @@
             "URL Decode",
             "Escape Unicode Characters",
             "Unescape Unicode Characters",
+            "Normalise Unicode",
             "To Quoted Printable",
             "From Quoted Printable",
             "To Punycode",

src/core/dishTypes/DishListFile.mjs

@@ -6,7 +6,6 @@
 
 import DishType from "./DishType.mjs";
 import Utils, { isNodeEnvironment } from "../Utils.mjs";
-import Dish from "../Dish.mjs";
 
 
 /**

src/core/lib/ChrEnc.mjs

@@ -164,3 +164,15 @@ export const IO_FORMAT = {
     "Simplified Chinese GB18030 (54936)": 54936,
 };
 
+/**
+ * Unicode Normalisation Forms
+ *
+ * @author Matthieu [m@tthieu.xyz]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+/**
+ * Character encoding format mappings.
+ */
+export const UNICODE_NORMALISATION_FORMS = ["NFD", "NFC", "NFKD", "NFKC"];

src/core/lib/FileSignatures.mjs

@@ -1008,8 +1008,7 @@ export const FILE_SIGNATURES = {
                 0: 0x7b,
                 1: 0x5c,
                 2: 0x72,
-                3: 0x74,
-                4: 0x66
+                3: 0x74
             },
             extractor: extractRTF
         },

src/core/operations/NormaliseUnicode.mjs

@@ -0,0 +1,62 @@
+/**
+ * @author Matthieu [m@tthieu.xyz]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import {UNICODE_NORMALISATION_FORMS} from "../lib/ChrEnc.mjs";
+import unorm from "unorm";
+
+/**
+ * Normalise Unicode operation
+ */
+class NormaliseUnicode extends Operation {
+
+    /**
+     * NormaliseUnicode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Normalise Unicode";
+        this.module = "Encodings";
+        this.description = "Transform Unicode characters to one of the Normalisation Forms";
+        this.infoURL = "https://wikipedia.org/wiki/Unicode_equivalence#Normal_forms";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Normal Form",
+                type: "option",
+                value: UNICODE_NORMALISATION_FORMS
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [normalForm] = args;
+
+        switch (normalForm) {
+            case "NFD":
+                return unorm.nfd(input);
+            case "NFC":
+                return unorm.nfc(input);
+            case "NFKD":
+                return unorm.nfkd(input);
+            case "NFKC":
+                return unorm.nfc(input);
+            default:
+                throw new OperationError("Unknown Normalisation Form");
+        }
+    }
+
+}
+
+export default NormaliseUnicode;

src/core/operations/ToCaseInsensitiveRegex.mjs

@@ -1,10 +1,12 @@
 /**
  * @author masq [github.cyberchef@masq.cc]
+ * @author n1073645
  * @copyright Crown Copyright 2018
  * @license Apache-2.0
  */
 
 import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 /**
  * To Case Insensitive Regex operation
@@ -32,7 +34,61 @@ class ToCaseInsensitiveRegex extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        return input.replace(/[a-z]/ig, m => `[${m.toLowerCase()}${m.toUpperCase()}]`);
+
+        /**
+         * Simulates look behind behaviour since javascript doesn't support it.
+         *
+         * @param {string} input
+         * @returns {string}
+         */
+        function preProcess(input) {
+            let result = "";
+            for (let i = 0; i < input.length; i++) {
+                const temp = input.charAt(i);
+                if (temp.match(/[a-zA-Z]/g) && (input.charAt(i-1) !== "-") && (input.charAt(i+1) !== "-"))
+                    result += "[" + temp.toLowerCase() + temp.toUpperCase() + "]";
+                else
+                    result += temp;
+            }
+            return result;
+        }
+
+        try {
+            RegExp(input);
+        } catch (error) {
+            throw new OperationError("Invalid Regular Expression (Please note this version of node does not support look behinds).");
+        }
+
+        // Example: [test] -> [[tT][eE][sS][tT]]
+        return preProcess(input)
+
+            // Example: [A-Z] -> [A-Za-z]
+            .replace(/([A-Z]-[A-Z]|[a-z]-[a-z])/g, m => `${m[0].toUpperCase()}-${m[2].toUpperCase()}${m[0].toLowerCase()}-${m[2].toLowerCase()}`)
+
+            // Example: [H-d] -> [A-DH-dh-z]
+            .replace(/[A-Z]-[a-z]/g, m => `A-${m[2].toUpperCase()}${m}${m[0].toLowerCase()}-z`)
+
+            // Example: [!-D] -> [!-Da-d]
+            .replace(/\\?[ -@]-[A-Z]/g, m => `${m}a-${m[2].toLowerCase()}`)
+
+            // Example: [%-^] -> [%-^a-z]
+            .replace(/\\?[ -@]-\\?[[-`]/g, m => `${m}a-z`)
+
+            // Example: [K-`] -> [K-`k-z]
+            .replace(/[A-Z]-\\?[[-`]/g, m => `${m}${m[0].toLowerCase()}-z`)
+
+            // Example: [[-}] -> [[-}A-Z]
+            .replace(/\\?[[-`]-\\?[{-~]/g, m => `${m}A-Z`)
+
+            // Example: [b-}] -> [b-}B-Z]
+            .replace(/[a-z]-\\?[{-~]/g, m => `${m}${m[0].toUpperCase()}-Z`)
+
+            // Example: [<-j] -> [<-z]
+            .replace(/\\?[ -@]-[a-z]/g, m => `${m[0]}-z`)
+
+            // Example: [^-j] -> [A-J^-j]
+            .replace(/\\?[[-`]-[a-z]/g, m => `A-${m[2].toUpperCase()}${m}`);
+
     }
 }
 

tests/operations/index.mjs

@@ -59,6 +59,7 @@ import "./tests/MS.mjs";
 import "./tests/Magic.mjs";
 import "./tests/MorseCode.mjs";
 import "./tests/NetBIOS.mjs";
+import "./tests/NormaliseUnicode.mjs";
 import "./tests/OTP.mjs";
 import "./tests/PGP.mjs";
 import "./tests/PHP.mjs";

tests/operations/tests/NormaliseUnicode.mjs

@@ -0,0 +1,54 @@
+/**
+ * Text Encoding Brute Force tests.
+ *
+ * @author Matthieu [m@tthieux.xyz]
+ *
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "Normalise Unicode - NFD",
+        input: "\u00c7\u0043\u0327\u2160",
+        expectedMatch: /C\u0327C\u0327\u2160/,
+        recipeConfig: [
+            {
+                op: "Normalise Unicode",
+                args: ["NFD"],
+            },
+        ],
+    }, {
+        name: "Normalise Unicode - NFC",
+        input: "\u00c7\u0043\u0327\u2160",
+        expectedMatch: /\u00C7\u00C7\u2160/,
+        recipeConfig: [
+            {
+                op: "Normalise Unicode",
+                args: ["NFC"],
+            },
+        ],
+    }, {
+        name: "Normalise Unicode - NFKD",
+        input: "\u00c7\u0043\u0327\u2160",
+        expectedMatch: /C\u0327C\u0327I/,
+        recipeConfig: [
+            {
+                op: "Normalise Unicode",
+                args: ["NFKD"],
+            },
+        ],
+    }, {
+        name: "Normalise Unicode - NFKC",
+        input: "\u00c7\u0043\u0327\u2160",
+        expectedMatch: /\u00C7\u00C7\u2160/,
+        recipeConfig: [
+            {
+                op: "Normalise Unicode",
+                args: ["NFKC"],
+            },
+        ],
+    },
+]);
+

tests/operations/tests/ToFromInsensitiveRegex.mjs

@@ -53,4 +53,136 @@ TestRegister.addTests([
             },
         ],
     },
+    {
+        name: "To Case Insensitive Regex: [A-Z] -> [A-Za-z]",
+        input: "[A-Z]",
+        expectedOutput: "[A-Za-z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [a-z] -> [A-Za-z]",
+        input: "[a-z]",
+        expectedOutput: "[A-Za-z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [H-d] -> [A-DH-dh-z]",
+        input: "[H-d]",
+        expectedOutput: "[A-DH-dh-z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [!-D] -> [!-Da-d]",
+        input: "[!-D]",
+        expectedOutput: "[!-Da-d]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [%-^] -> [%-^a-z]",
+        input: "[%-^]",
+        expectedOutput: "[%-^a-z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [K-`] -> [K-`k-z]",
+        input: "[K-`]",
+        expectedOutput: "[K-`k-z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [[-}] -> [[-}A-Z]",
+        input: "[[-}]",
+        expectedOutput: "[[-}A-Z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [b-}] -> [b-}B-Z]",
+        input: "[b-}]",
+        expectedOutput: "[b-}B-Z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [<-j] -> [<-z]",
+        input: "[<-j]",
+        expectedOutput: "[<-z]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: [^-j] -> [A-J^-j]",
+        input: "[^-j]",
+        expectedOutput: "[A-J^-j]",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: not simple test",
+        input: "Mozilla[A-Z0-9]+[A-Z]Mozilla[0-9whatA-Z][H-d][!-H][a-~](.)+",
+        expectedOutput: "[mM][oO][zZ][iI][lL][lL][aA][A-Za-z0-9]+[A-Za-z][mM][oO][zZ][iI][lL][lL][aA][0-9[wW][hH][aA][tT]A-Za-z][A-DH-dh-z][!-Ha-h][a-~A-Z](.)+",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Case Insensitive Regex: erroneous test",
+        input: "Mozilla[A-Z",
+        expectedOutput: "Invalid Regular Expression (Please note this version of node does not support look behinds).",
+        recipeConfig: [
+            {
+                op: "To Case Insensitive Regex",
+                args: [],
+            },
+        ],
+    }
 ]);