From 6aa9d2b492a5900014103faebbc7dac65cf69e02 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 14 Dec 2018 16:43:03 +0000 Subject: [PATCH 001/110] Added 'Extract Files' operation and 'Forensics' category. --- package-lock.json | 142 ++++++++-------- src/core/config/Categories.json | 18 ++- src/core/lib/FileExtraction.mjs | 231 +++++++++++++++++++++++++++ src/core/lib/Stream.mjs | 164 +++++++++++++++++++ src/core/operations/ExtractFiles.mjs | 91 +++++++++++ 5 files changed, 572 insertions(+), 74 deletions(-) create mode 100644 src/core/lib/FileExtraction.mjs create mode 100644 src/core/lib/Stream.mjs create mode 100644 src/core/operations/ExtractFiles.mjs diff --git a/package-lock.json b/package-lock.json index 4f1be0a3..8b236eb9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1171,7 +1171,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1284,7 +1284,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1369,7 +1369,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1457,7 +1457,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -1863,7 +1863,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -1900,7 +1900,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -1950,7 +1950,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2015,7 +2015,7 @@ }, "cacache": { "version": "10.0.4", - "resolved": "http://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==", "dev": true, "requires": { @@ -2092,7 +2092,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2123,7 +2123,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -2590,7 +2590,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -2603,7 +2603,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -2721,7 +2721,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3055,7 +3055,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3119,7 +3119,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3307,7 +3307,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -3731,7 +3731,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -3743,7 +3743,7 @@ }, "events": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4149,7 +4149,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -4377,7 +4377,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -4445,12 +4445,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4470,7 +4472,8 @@ "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", @@ -4618,6 +4621,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5023,7 +5027,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5173,7 +5177,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5221,7 +5225,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -5241,7 +5245,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5269,7 +5273,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -5368,7 +5372,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -5432,7 +5436,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5450,7 +5454,7 @@ }, "handle-thing": { "version": "1.2.5", - "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", "dev": true }, @@ -5677,7 +5681,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -5725,7 +5729,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -5744,7 +5748,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -5773,7 +5777,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -6225,7 +6229,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -6750,7 +6754,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -6856,7 +6860,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -6948,7 +6952,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -6961,7 +6965,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -7196,7 +7200,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -7255,7 +7259,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -7432,7 +7436,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { @@ -7554,7 +7558,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -7617,7 +7621,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -7756,7 +7760,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -8015,13 +8019,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -8030,7 +8034,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -8173,7 +8177,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -8231,7 +8235,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -8272,7 +8276,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8839,7 +8843,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -8864,13 +8868,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -8885,7 +8889,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -9064,7 +9068,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9426,7 +9430,7 @@ }, "require-uncached": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", "dev": true, "requires": { @@ -9593,7 +9597,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -9914,7 +9918,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -9958,7 +9962,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -10610,7 +10614,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -10627,7 +10631,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -10706,7 +10710,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -10734,7 +10738,7 @@ }, "through": { "version": "2.3.8", - "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -11381,7 +11385,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -11407,7 +11411,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -11423,7 +11427,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -11847,7 +11851,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -11984,7 +11988,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 4fac84c9..8eccea12 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -245,7 +245,8 @@ "XPath expression", "JPath expression", "CSS selector", - "Extract EXIF" + "Extract EXIF", + "Extract Files" ] }, { @@ -336,14 +337,23 @@ "From MessagePack" ] }, + { + "name": "Forensics", + "ops": [ + "Detect File Type", + "Scan for Embedded Files", + "Extract Files", + "Remove EXIF", + "Extract EXIF", + "Render Image" + ] + }, { "name": "Other", "ops": [ "Entropy", "Frequency distribution", "Chi Square", - "Detect File Type", - "Scan for Embedded Files", "Disassemble x86", "Pseudo-Random Number Generator", "Generate UUID", @@ -351,8 +361,6 @@ "Generate HOTP", "Haversine distance", "Render Image", - "Remove EXIF", - "Extract EXIF", "Numberwang", "XKCD Random Number" ] diff --git a/src/core/lib/FileExtraction.mjs b/src/core/lib/FileExtraction.mjs new file mode 100644 index 00000000..927aca92 --- /dev/null +++ b/src/core/lib/FileExtraction.mjs @@ -0,0 +1,231 @@ +/** + * File extraction functions + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ +import Stream from "./Stream"; + +/** + * Attempts to extract a file from a data stream given its mime type and offset. + * + * @param {Uint8Array} bytes + * @param {Object} fileDetail + * @param {string} fileDetail.mime + * @param {string} fileDetail.ext + * @param {number} fileDetail.offset + * @returns {File} + */ +export function extractFile(bytes, fileDetail) { + let fileData; + switch (fileDetail.mime) { + case "image/jpeg": + fileData = extractJPEG(bytes, fileDetail.offset); + break; + case "application/x-msdownload": + fileData = extractMZPE(bytes, fileDetail.offset); + break; + case "application/pdf": + fileData = extractPDF(bytes, fileDetail.offset); + break; + case "application/zip": + fileData = extractZIP(bytes, fileDetail.offset); + break; + default: + throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); + } + + return new File([fileData], `extracted_at_0x${fileDetail.offset.toString(16)}.${fileDetail.ext}`); +} + + +/** + * JPEG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractJPEG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + while (stream.hasMore()) { + const marker = stream.getBytes(2); + if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + + let segmentSize = 0; + switch (marker[1]) { + // No length + case 0xd8: // Start of Image + case 0x01: // For temporary use in arithmetic coding + break; + case 0xd9: // End found + return stream.carve(); + + // Variable size segment + case 0xc0: // Start of frame (Baseline DCT) + case 0xc1: // Start of frame (Extended sequential DCT) + case 0xc2: // Start of frame (Progressive DCT) + case 0xc3: // Start of frame (Lossless sequential) + case 0xc4: // Define Huffman Table + case 0xc5: // Start of frame (Differential sequential DCT) + case 0xc6: // Start of frame (Differential progressive DCT) + case 0xc7: // Start of frame (Differential lossless) + case 0xc8: // Reserved for JPEG extensions + case 0xc9: // Start of frame (Extended sequential DCT) + case 0xca: // Start of frame (Progressive DCT) + case 0xcb: // Start of frame (Lossless sequential) + case 0xcc: // Define arithmetic conditioning table + case 0xcd: // Start of frame (Differential sequential DCT) + case 0xce: // Start of frame (Differential progressive DCT) + case 0xcf: // Start of frame (Differential lossless) + case 0xdb: // Define Quantization Table + case 0xde: // Define hierarchical progression + case 0xe0: // Application-specific + case 0xe1: // Application-specific + case 0xe2: // Application-specific + case 0xe3: // Application-specific + case 0xe4: // Application-specific + case 0xe5: // Application-specific + case 0xe6: // Application-specific + case 0xe7: // Application-specific + case 0xe8: // Application-specific + case 0xe9: // Application-specific + case 0xea: // Application-specific + case 0xeb: // Application-specific + case 0xec: // Application-specific + case 0xed: // Application-specific + case 0xee: // Application-specific + case 0xef: // Application-specific + case 0xfe: // Comment + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + break; + + // 1 byte + case 0xdf: // Expand reference image + stream.position++; + break; + + // 2 bytes + case 0xdc: // Define number of lines + case 0xdd: // Define restart interval + stream.position += 2; + break; + + // Start scan + case 0xda: // Start of scan + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + stream.continueUntil(0xff); + break; + + // Continue through encoded data + case 0x00: // Byte stuffing + case 0xd0: // Restart + case 0xd1: // Restart + case 0xd2: // Restart + case 0xd3: // Restart + case 0xd4: // Restart + case 0xd5: // Restart + case 0xd6: // Restart + case 0xd7: // Restart + stream.continueUntil(0xff); + break; + + default: + stream.continueUntil(0xff); + break; + } + } + + throw new Error("Unable to parse JPEG successfully"); +} + + +/** + * Portable executable extractor. + * Assumes that the offset refers to an MZ header. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMZPE(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to PE header pointer + stream.moveTo(0x3c); + const peAddress = stream.readInt(4, "le"); + + // Move to PE header + stream.moveTo(peAddress); + + // Get number of sections + stream.moveForwardsBy(6); + const numSections = stream.readInt(2, "le"); + + // Get optional header size + stream.moveForwardsBy(12); + const optionalHeaderSize = stream.readInt(2, "le"); + + // Move past optional header to section header + stream.moveForwardsBy(2 + optionalHeaderSize); + + // Move to final section header + stream.moveForwardsBy((numSections - 1) * 0x28); + + // Get raw data info + stream.moveForwardsBy(16); + const rawDataSize = stream.readInt(4, "le"); + const rawDataAddress = stream.readInt(4, "le"); + + // Move to end of final section + stream.moveTo(rawDataAddress + rawDataSize); + + return stream.carve(); +} + + +/** + * PDF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPDF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find end-of-file marker (%%EOF) + stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); + stream.moveForwardsBy(5); + stream.consumeIf(0x0d); + stream.consumeIf(0x0a); + + return stream.carve(); +} + + +/** + * ZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find End of central directory record + stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); + + // Get comment length and consume + stream.moveForwardsBy(20); + const commentLength = stream.readInt(2, "le"); + stream.moveForwardsBy(commentLength); + + return stream.carve(); +} diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs new file mode 100644 index 00000000..5f84e13c --- /dev/null +++ b/src/core/lib/Stream.mjs @@ -0,0 +1,164 @@ +/** + * Stream class for parsing binary protocols. + * + * @author n1474335 [n1474335@gmail.com] + * @author tlwr [toby@toby.codes] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ + +/** + * A Stream can be used to traverse a binary blob, interpreting sections of it + * as various data types. + * + * @param {Uint8Array} bytes + * @param {Object} fileDetail + * @param {string} fileDetail.mime + * @param {string} fileDetail.ext + * @param {number} fileDetail.offset + * @returns {File} + */ +export default class Stream { + + /** + * Stream constructor. + * + * @param {Uint8Array} input + */ + constructor(input) { + this.bytes = input; + this.position = 0; + } + + /** + * Get a number of bytes from the current position. + * + * @param {number} numBytes + * @returns {Uint8Array} + */ + getBytes(numBytes) { + const newPosition = this.position + numBytes; + const bytes = this.bytes.slice(this.position, newPosition); + this.position = newPosition; + return bytes; + } + + /** + * Interpret the following bytes as a string, stopping at the next null byte or + * the supplied limit. + * + * @param {number} numBytes + * @returns {string} + */ + readString(numBytes) { + let result = ""; + for (let i = this.position; i < this.position + numBytes; i++) { + const currentByte = this.bytes[i]; + if (currentByte === 0) break; + result += String.fromCharCode(currentByte); + } + this.position += numBytes; + return result; + } + + /** + * Interpret the following bytes as an integer in big or little endian. + * + * @param {number} numBytes + * @param {string} [endianness="be"] + * @returns {number} + */ + readInt(numBytes, endianness="be") { + let val = 0; + if (endianness === "be") { + for (let i = this.position; i < this.position + numBytes; i++) { + val = val << 8; + val |= this.bytes[i]; + } + } else { + for (let i = this.position + numBytes - 1; i >= this.position; i--) { + val = val << 8; + val |= this.bytes[i]; + } + } + this.position += numBytes; + return val; + } + + /** + * Consume the stream until we reach the specified byte or sequence of bytes. + * + * @param {number|List} val + */ + continueUntil(val) { + if (typeof val === "number") { + while (++this.position < this.bytes.length && this.bytes[this.position] !== val) { + continue; + } + return; + } + + // val is an array + let found = false; + while (!found && this.position < this.bytes.length) { + while (++this.position < this.bytes.length && this.bytes[this.position] !== val[0]) { + continue; + } + found = true; + for (let i = 1; i < val.length; i++) { + if (this.position + i > this.bytes.length || this.bytes[this.position + i] !== val[i]) + found = false; + } + } + } + + /** + * Consume the next byte if it matches the supplied value. + * + * @param {number} val + */ + consumeIf(val) { + if (this.bytes[this.position] === val) + this.position++; + } + + /** + * Move forwards through the stream by the specified number of bytes. + * + * @param {number} numBytes + */ + moveForwardsBy(numBytes) { + this.position += numBytes; + } + + /** + * Move to a specified position in the stream. + * + * @param {number} pos + */ + moveTo(pos) { + if (pos < 0 || pos > this.bytes.length - 1) + throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); + this.position = pos; + } + + /** + * Returns true if there are more bytes left in the stream. + * + * @returns {boolean} + */ + hasMore() { + return this.position < this.bytes.length; + } + + /** + * Returns a slice of the stream up to the current position. + * + * @returns {Uint8Array} + */ + carve() { + return this.bytes.slice(0, this.position); + } + +} diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs new file mode 100644 index 00000000..c213b256 --- /dev/null +++ b/src/core/operations/ExtractFiles.mjs @@ -0,0 +1,91 @@ +/** + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +// import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import Utils from "../Utils"; +import {extractFile} from "../lib/FileExtraction"; + +/** + * Extract Files operation + */ +class ExtractFiles extends Operation { + + /** + * ExtractFiles constructor + */ + constructor() { + super(); + + this.name = "Extract Files"; + this.module = "Default"; + this.description = "TODO"; + this.infoURL = "https://forensicswiki.org/wiki/File_Carving"; + this.inputType = "ArrayBuffer"; + this.outputType = "List"; + this.presentType = "html"; + this.args = []; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {List} + */ + run(input, args) { + const bytes = new Uint8Array(input); + + // Scan for embedded files + const fileDetails = scanForEmbeddedFiles(bytes); + + // Extract each file that we support + const files = []; + fileDetails.forEach(fileDetail => { + try { + files.push(extractFile(bytes, fileDetail)); + } catch (err) {} + }); + + return files; + } + + /** + * Displays the files in HTML for web apps. + * + * @param {File[]} files + * @returns {html} + */ + async present(files) { + return await Utils.displayFilesAsHTML(files); + } + +} + +/** + * TODO refactor + * @param data + */ +function scanForEmbeddedFiles(data) { + let type; + const types = []; + + for (let i = 0; i < data.length; i++) { + type = Magic.magicFileType(data.slice(i)); + if (type) { + types.push({ + offset: i, + ext: type.ext, + mime: type.mime, + desc: type.desc + }); + } + } + + return types; +} + +export default ExtractFiles; From e6fb0be1d04bd8c96a827d244f20ea3f01a991aa Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 18 Dec 2018 17:44:42 +0000 Subject: [PATCH 002/110] Refactored file type detection engine --- src/core/lib/BCD.mjs | 0 src/core/lib/Base58.mjs | 0 src/core/lib/Base64.mjs | 0 src/core/lib/CanvasComponents.mjs | 0 src/core/lib/FileExtraction.mjs | 231 ------ src/core/lib/FileType.mjs | 808 +++++++++++++++++++ src/core/lib/Magic.mjs | 455 +---------- src/core/lib/Stream.mjs | 7 - src/core/operations/DetectFileType.mjs | 20 +- src/core/operations/ExtractFiles.mjs | 28 +- src/core/operations/RenderImage.mjs | 12 +- src/core/operations/ScanForEmbeddedFiles.mjs | 33 +- 12 files changed, 867 insertions(+), 727 deletions(-) mode change 100755 => 100644 src/core/lib/BCD.mjs mode change 100755 => 100644 src/core/lib/Base58.mjs mode change 100755 => 100644 src/core/lib/Base64.mjs mode change 100755 => 100644 src/core/lib/CanvasComponents.mjs delete mode 100644 src/core/lib/FileExtraction.mjs create mode 100644 src/core/lib/FileType.mjs diff --git a/src/core/lib/BCD.mjs b/src/core/lib/BCD.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/Base58.mjs b/src/core/lib/Base58.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/Base64.mjs b/src/core/lib/Base64.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/CanvasComponents.mjs b/src/core/lib/CanvasComponents.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/FileExtraction.mjs b/src/core/lib/FileExtraction.mjs deleted file mode 100644 index 927aca92..00000000 --- a/src/core/lib/FileExtraction.mjs +++ /dev/null @@ -1,231 +0,0 @@ -/** - * File extraction functions - * - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - * - */ -import Stream from "./Stream"; - -/** - * Attempts to extract a file from a data stream given its mime type and offset. - * - * @param {Uint8Array} bytes - * @param {Object} fileDetail - * @param {string} fileDetail.mime - * @param {string} fileDetail.ext - * @param {number} fileDetail.offset - * @returns {File} - */ -export function extractFile(bytes, fileDetail) { - let fileData; - switch (fileDetail.mime) { - case "image/jpeg": - fileData = extractJPEG(bytes, fileDetail.offset); - break; - case "application/x-msdownload": - fileData = extractMZPE(bytes, fileDetail.offset); - break; - case "application/pdf": - fileData = extractPDF(bytes, fileDetail.offset); - break; - case "application/zip": - fileData = extractZIP(bytes, fileDetail.offset); - break; - default: - throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); - } - - return new File([fileData], `extracted_at_0x${fileDetail.offset.toString(16)}.${fileDetail.ext}`); -} - - -/** - * JPEG extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractJPEG(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - while (stream.hasMore()) { - const marker = stream.getBytes(2); - if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); - - let segmentSize = 0; - switch (marker[1]) { - // No length - case 0xd8: // Start of Image - case 0x01: // For temporary use in arithmetic coding - break; - case 0xd9: // End found - return stream.carve(); - - // Variable size segment - case 0xc0: // Start of frame (Baseline DCT) - case 0xc1: // Start of frame (Extended sequential DCT) - case 0xc2: // Start of frame (Progressive DCT) - case 0xc3: // Start of frame (Lossless sequential) - case 0xc4: // Define Huffman Table - case 0xc5: // Start of frame (Differential sequential DCT) - case 0xc6: // Start of frame (Differential progressive DCT) - case 0xc7: // Start of frame (Differential lossless) - case 0xc8: // Reserved for JPEG extensions - case 0xc9: // Start of frame (Extended sequential DCT) - case 0xca: // Start of frame (Progressive DCT) - case 0xcb: // Start of frame (Lossless sequential) - case 0xcc: // Define arithmetic conditioning table - case 0xcd: // Start of frame (Differential sequential DCT) - case 0xce: // Start of frame (Differential progressive DCT) - case 0xcf: // Start of frame (Differential lossless) - case 0xdb: // Define Quantization Table - case 0xde: // Define hierarchical progression - case 0xe0: // Application-specific - case 0xe1: // Application-specific - case 0xe2: // Application-specific - case 0xe3: // Application-specific - case 0xe4: // Application-specific - case 0xe5: // Application-specific - case 0xe6: // Application-specific - case 0xe7: // Application-specific - case 0xe8: // Application-specific - case 0xe9: // Application-specific - case 0xea: // Application-specific - case 0xeb: // Application-specific - case 0xec: // Application-specific - case 0xed: // Application-specific - case 0xee: // Application-specific - case 0xef: // Application-specific - case 0xfe: // Comment - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - break; - - // 1 byte - case 0xdf: // Expand reference image - stream.position++; - break; - - // 2 bytes - case 0xdc: // Define number of lines - case 0xdd: // Define restart interval - stream.position += 2; - break; - - // Start scan - case 0xda: // Start of scan - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - stream.continueUntil(0xff); - break; - - // Continue through encoded data - case 0x00: // Byte stuffing - case 0xd0: // Restart - case 0xd1: // Restart - case 0xd2: // Restart - case 0xd3: // Restart - case 0xd4: // Restart - case 0xd5: // Restart - case 0xd6: // Restart - case 0xd7: // Restart - stream.continueUntil(0xff); - break; - - default: - stream.continueUntil(0xff); - break; - } - } - - throw new Error("Unable to parse JPEG successfully"); -} - - -/** - * Portable executable extractor. - * Assumes that the offset refers to an MZ header. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractMZPE(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Move to PE header pointer - stream.moveTo(0x3c); - const peAddress = stream.readInt(4, "le"); - - // Move to PE header - stream.moveTo(peAddress); - - // Get number of sections - stream.moveForwardsBy(6); - const numSections = stream.readInt(2, "le"); - - // Get optional header size - stream.moveForwardsBy(12); - const optionalHeaderSize = stream.readInt(2, "le"); - - // Move past optional header to section header - stream.moveForwardsBy(2 + optionalHeaderSize); - - // Move to final section header - stream.moveForwardsBy((numSections - 1) * 0x28); - - // Get raw data info - stream.moveForwardsBy(16); - const rawDataSize = stream.readInt(4, "le"); - const rawDataAddress = stream.readInt(4, "le"); - - // Move to end of final section - stream.moveTo(rawDataAddress + rawDataSize); - - return stream.carve(); -} - - -/** - * PDF extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractPDF(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find end-of-file marker (%%EOF) - stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); - stream.moveForwardsBy(5); - stream.consumeIf(0x0d); - stream.consumeIf(0x0a); - - return stream.carve(); -} - - -/** - * ZIP extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractZIP(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find End of central directory record - stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); - - // Get comment length and consume - stream.moveForwardsBy(20); - const commentLength = stream.readInt(2, "le"); - stream.moveForwardsBy(commentLength); - - return stream.carve(); -} diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs new file mode 100644 index 00000000..443bdd36 --- /dev/null +++ b/src/core/lib/FileType.mjs @@ -0,0 +1,808 @@ +/** + * File type functions + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ +import Stream from "./Stream"; + +/** + * A categorised table of file types, including signatures to identifying them and functions + * to extract them where possible. + */ +const FILE_SIGNATURES = { + "Pictures": [ + { + name: "JPEG Image", + extension: "jpg", + mime: "image/jpeg", + description: "", + signature: { + 0: 0xff, + 1: 0xd8, + 2: 0xff + }, + extractor: extractJPEG + }, + { + name: "GIF Image", + extension: "gif", + mime: "image/gif", + description: "", + signature: { + 0: 0x47, + 1: 0x49, + 2: 0x46 + }, + extractor: null + }, + { + name: "PNG Image", + extension: "png", + mime: "image/png", + description: "", + signature: { + 0: 0x89, + 1: 0x50, + 2: 0x4e, + 3: 0x47 + }, + extractor: null + }, + + ], + "Documents": [ + { + name: "Portable Document Format", + extension: "pdf", + mime: "application/pdf", + description: "", + signature: { + 0: 0x25, + 1: 0x50, + 2: 0x44, + 3: 0x46 + }, + extractor: extractPDF + }, + ], + "Applications": [ + { + name: "Windows Portable Executable", + extension: "exe", + mime: "application/x-msdownload", + description: "", + signature: { + 0: 0x4d, + 1: 0x5a + }, + extractor: extractMZPE + }, + ], + "Archives": [ + { + name: "ZIP", + extension: "zip", + mime: "application/zip", + description: "", + signature: { + 0: 0x50, + 1: 0x4b, + 2: [0x3, 0x5, 0x7], + 3: [0x4, 0x6, 0x8] + }, + extractor: extractZIP + }, + + ], +}; + + +/** + * Checks whether a signature matches a buffer. + * + * @param {Object} sig - A dictionary of offsets with values assigned to them. These + * values can be numbers for static checks, arrays of potential valid matches, or + * bespoke functions to check the validity of the buffer value at that offset. + * @param {Uint8Array} buf + * @returns {boolean} + */ +function signatureMatches(sig, buf) { + for (const offset in sig) { + switch (typeof sig[offset]) { + case "number": // Static check + if (buf[offset] !== sig[offset]) + return false; + break; + case "object": // Array of options + if (sig[offset].indexOf(buf[offset]) < 0) + return false; + break; + case "function": // More complex calculation + if (!sig[offset](buf[offset])) + return false; + break; + default: + throw new Error(`Unrecognised signature type at offset ${offset}`); + } + } + return true; +} + + +/** + * Given a buffer, detects magic byte sequences at specific positions and returns the + * extension and mime type. + * + * @param {Uint8Array} buf + * @returns {Object[]} type + * @returns {string} type.ext - File extension + * @returns {string} type.mime - Mime type + * @returns {string} [type.desc] - Description + */ +export function detectFileType(buf) { + if (!(buf && buf.length > 1)) { + return []; + } + + const matchingFiles = []; + + // TODO allow user to select which categories to check + for (const cat in FILE_SIGNATURES) { + const category = FILE_SIGNATURES[cat]; + + category.forEach(filetype => { + if (signatureMatches(filetype.signature, buf)) { + matchingFiles.push(filetype); + } + }); + } + return matchingFiles; + + // Delete all below this line once implemented in FILE_SIGNATURES above. + + + /* + if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) { + return { + ext: "jpg", + mime: "image/jpeg" + }; + } + + if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) { + return { + ext: "png", + mime: "image/png" + }; + } + + if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) { + return { + ext: "gif", + mime: "image/gif" + }; + } + + if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) { + return { + ext: "webp", + mime: "image/webp" + }; + } + + // needs to be before `tif` check + if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { + return { + ext: "cr2", + mime: "image/x-canon-cr2" + }; + } + + if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) { + return { + ext: "tif", + mime: "image/tiff" + }; + } + + if (buf[0] === 0x42 && buf[1] === 0x4D) { + return { + ext: "bmp", + mime: "image/bmp" + }; + } + + if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) { + return { + ext: "jxr", + mime: "image/vnd.ms-photo" + }; + } + + if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) { + return { + ext: "psd", + mime: "image/vnd.adobe.photoshop" + }; + } + + // needs to be before `zip` check + if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) { + return { + ext: "epub", + mime: "application/epub+zip" + }; + } + + if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) { + return { + ext: "zip", + mime: "application/zip" + }; + } + + if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { + return { + ext: "tar", + mime: "application/x-tar" + }; + } + + if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) { + return { + ext: "rar", + mime: "application/x-rar-compressed" + }; + } + + if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) { + return { + ext: "gz", + mime: "application/gzip" + }; + } + + if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) { + return { + ext: "bz2", + mime: "application/x-bzip2" + }; + } + + if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) { + return { + ext: "7z", + mime: "application/x-7z-compressed" + }; + } + + if (buf[0] === 0x78 && buf[1] === 0x01) { + return { + ext: "dmg, zlib", + mime: "application/x-apple-diskimage, application/x-deflate" + }; + } + + if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) { + return { + ext: "mp4", + mime: "video/mp4" + }; + } + + if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) { + return { + ext: "m4v", + mime: "video/x-m4v" + }; + } + + if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) { + return { + ext: "mid", + mime: "audio/midi" + }; + } + + // needs to be before the `webm` check + if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) { + return { + ext: "mkv", + mime: "video/x-matroska" + }; + } + + if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) { + return { + ext: "webm", + mime: "video/webm" + }; + } + + if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { + return { + ext: "mov", + mime: "video/quicktime" + }; + } + + if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) { + return { + ext: "avi", + mime: "video/x-msvideo" + }; + } + + if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) { + return { + ext: "wmv", + mime: "video/x-ms-wmv" + }; + } + + if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") { + return { + ext: "mpg", + mime: "video/mpeg" + }; + } + + if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) { + return { + ext: "mp3", + mime: "audio/mpeg" + }; + } + + if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) { + return { + ext: "m4a", + mime: "audio/m4a" + }; + } + + if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) { + return { + ext: "ogg", + mime: "audio/ogg" + }; + } + + if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { + return { + ext: "flac", + mime: "audio/x-flac" + }; + } + + if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) { + return { + ext: "wav", + mime: "audio/x-wav" + }; + } + + if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { + return { + ext: "amr", + mime: "audio/amr" + }; + } + + if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) { + return { + ext: "pdf", + mime: "application/pdf" + }; + } + + if (buf[0] === 0x4D && buf[1] === 0x5A) { + return { + ext: "exe", + mime: "application/x-msdownload" + }; + } + + if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { + return { + ext: "swf", + mime: "application/x-shockwave-flash" + }; + } + + if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) { + return { + ext: "rtf", + mime: "application/rtf" + }; + } + + if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { + return { + ext: "woff", + mime: "application/font-woff" + }; + } + + if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { + return { + ext: "woff2", + mime: "application/font-woff" + }; + } + + if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) { + return { + ext: "eot", + mime: "application/octet-stream" + }; + } + + if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) { + return { + ext: "ttf", + mime: "application/font-sfnt" + }; + } + + if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) { + return { + ext: "otf", + mime: "application/font-sfnt" + }; + } + + if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) { + return { + ext: "ico", + mime: "image/x-icon" + }; + } + + if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) { + return { + ext: "flv", + mime: "video/x-flv" + }; + } + + if (buf[0] === 0x25 && buf[1] === 0x21) { + return { + ext: "ps", + mime: "application/postscript" + }; + } + + if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) { + return { + ext: "xz", + mime: "application/x-xz" + }; + } + + if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) { + return { + ext: "sqlite", + mime: "application/x-sqlite3" + }; + } + */ + + /** + * + * Added by n1474335 [n1474335@gmail.com] from here on + * + */ + /* + if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) { + return { + ext: "z, tar.z", + mime: "application/x-gtar" + }; + } + + if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) { + return { + ext: "none, axf, bin, elf, o, prx, puff, so", + mime: "application/x-executable", + desc: "Executable and Linkable Format file. No standard file extension." + }; + } + + if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) { + return { + ext: "class", + mime: "application/java-vm" + }; + } + + if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) { + return { + ext: "txt", + mime: "text/plain", + desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files." + }; + } + + // Must be before Little-endian UTF-16 BOM + if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) { + return { + ext: "UTF32LE", + mime: "charset/utf32le", + desc: "Little-endian UTF-32 encoded Unicode byte order mark detected." + }; + } + + if (buf[0] === 0xFF && buf[1] === 0xFE) { + return { + ext: "UTF16LE", + mime: "charset/utf16le", + desc: "Little-endian UTF-16 encoded Unicode byte order mark detected." + }; + } + + if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) || + (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) || + (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) { + return { + ext: "iso", + mime: "application/octet-stream", + desc: "ISO 9660 CD/DVD image file" + }; + } + + if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) { + return { + ext: "doc, xls, ppt", + mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint", + desc: "Microsoft Office documents" + }; + } + + if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) { + return { + ext: "dex", + mime: "application/octet-stream", + desc: "Dalvik Executable (Android)" + }; + } + + if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) { + return { + ext: "vmdk", + mime: "application/vmdk, application/x-virtualbox-vmdk" + }; + } + + if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) { + return { + ext: "crx", + mime: "application/crx", + desc: "Google Chrome extension or packaged app" + }; + } + + if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) { + return { + ext: "zlib", + mime: "application/x-deflate" + }; + } + + return null; + */ +} + + +/** + * Attempts to extract a file from a data stream given its offset and extractor function. + * + * @param {Uint8Array} bytes + * @param {Object} fileDetail + * @param {string} fileDetail.mime + * @param {string} fileDetail.extension + * @param {Function} fileDetail.extractor + * @param {number} offset + * @returns {File} + */ +export function extractFile(bytes, fileDetail, offset) { + if (fileDetail.extractor) { + const fileData = fileDetail.extractor(bytes, offset); + return new File([fileData], `extracted_at_0x${offset.toString(16)}.${fileDetail.extension}`); + } + + throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); +} + + +/** + * JPEG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractJPEG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + while (stream.hasMore()) { + const marker = stream.getBytes(2); + if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + + let segmentSize = 0; + switch (marker[1]) { + // No length + case 0xd8: // Start of Image + case 0x01: // For temporary use in arithmetic coding + break; + case 0xd9: // End found + return stream.carve(); + + // Variable size segment + case 0xc0: // Start of frame (Baseline DCT) + case 0xc1: // Start of frame (Extended sequential DCT) + case 0xc2: // Start of frame (Progressive DCT) + case 0xc3: // Start of frame (Lossless sequential) + case 0xc4: // Define Huffman Table + case 0xc5: // Start of frame (Differential sequential DCT) + case 0xc6: // Start of frame (Differential progressive DCT) + case 0xc7: // Start of frame (Differential lossless) + case 0xc8: // Reserved for JPEG extensions + case 0xc9: // Start of frame (Extended sequential DCT) + case 0xca: // Start of frame (Progressive DCT) + case 0xcb: // Start of frame (Lossless sequential) + case 0xcc: // Define arithmetic conditioning table + case 0xcd: // Start of frame (Differential sequential DCT) + case 0xce: // Start of frame (Differential progressive DCT) + case 0xcf: // Start of frame (Differential lossless) + case 0xdb: // Define Quantization Table + case 0xde: // Define hierarchical progression + case 0xe0: // Application-specific + case 0xe1: // Application-specific + case 0xe2: // Application-specific + case 0xe3: // Application-specific + case 0xe4: // Application-specific + case 0xe5: // Application-specific + case 0xe6: // Application-specific + case 0xe7: // Application-specific + case 0xe8: // Application-specific + case 0xe9: // Application-specific + case 0xea: // Application-specific + case 0xeb: // Application-specific + case 0xec: // Application-specific + case 0xed: // Application-specific + case 0xee: // Application-specific + case 0xef: // Application-specific + case 0xfe: // Comment + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + break; + + // 1 byte + case 0xdf: // Expand reference image + stream.position++; + break; + + // 2 bytes + case 0xdc: // Define number of lines + case 0xdd: // Define restart interval + stream.position += 2; + break; + + // Start scan + case 0xda: // Start of scan + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + stream.continueUntil(0xff); + break; + + // Continue through encoded data + case 0x00: // Byte stuffing + case 0xd0: // Restart + case 0xd1: // Restart + case 0xd2: // Restart + case 0xd3: // Restart + case 0xd4: // Restart + case 0xd5: // Restart + case 0xd6: // Restart + case 0xd7: // Restart + stream.continueUntil(0xff); + break; + + default: + stream.continueUntil(0xff); + break; + } + } + + throw new Error("Unable to parse JPEG successfully"); +} + + +/** + * Portable executable extractor. + * Assumes that the offset refers to an MZ header. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMZPE(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to PE header pointer + stream.moveTo(0x3c); + const peAddress = stream.readInt(4, "le"); + + // Move to PE header + stream.moveTo(peAddress); + + // Get number of sections + stream.moveForwardsBy(6); + const numSections = stream.readInt(2, "le"); + + // Get optional header size + stream.moveForwardsBy(12); + const optionalHeaderSize = stream.readInt(2, "le"); + + // Move past optional header to section header + stream.moveForwardsBy(2 + optionalHeaderSize); + + // Move to final section header + stream.moveForwardsBy((numSections - 1) * 0x28); + + // Get raw data info + stream.moveForwardsBy(16); + const rawDataSize = stream.readInt(4, "le"); + const rawDataAddress = stream.readInt(4, "le"); + + // Move to end of final section + stream.moveTo(rawDataAddress + rawDataSize); + + return stream.carve(); +} + + +/** + * PDF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPDF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find end-of-file marker (%%EOF) + stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); + stream.moveForwardsBy(5); + stream.consumeIf(0x0d); + stream.consumeIf(0x0a); + + return stream.carve(); +} + + +/** + * ZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find End of central directory record + stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); + + // Get comment length and consume + stream.moveForwardsBy(20); + const commentLength = stream.readInt(2, "le"); + stream.moveForwardsBy(commentLength); + + return stream.carve(); +} diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index b4d5a7b0..91472e21 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -2,6 +2,7 @@ import OperationConfig from "../config/OperationConfig.json"; import Utils from "../Utils"; import Recipe from "../Recipe"; import Dish from "../Dish"; +import {detectFileType} from "./FileType"; import chiSquared from "chi-squared"; /** @@ -92,7 +93,14 @@ class Magic { * @returns {string} [type.desc] - Description */ detectFileType() { - return Magic.magicFileType(this.inputBuffer); + const fileType = detectFileType(this.inputBuffer); + + if (!fileType.length) return null; + return { + ext: fileType[0].extension, + mime: fileType[0].mime, + desc: fileType[0].description + }; } /** @@ -784,452 +792,9 @@ class Magic { }[code]; } - - /** - * Given a buffer, detects magic byte sequences at specific positions and returns the - * extension and mime type. - * - * @param {Uint8Array} buf - * @returns {Object} type - * @returns {string} type.ext - File extension - * @returns {string} type.mime - Mime type - * @returns {string} [type.desc] - Description - */ - static magicFileType(buf) { - if (!(buf && buf.length > 1)) { - return null; - } - - if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) { - return { - ext: "jpg", - mime: "image/jpeg" - }; - } - - if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) { - return { - ext: "png", - mime: "image/png" - }; - } - - if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) { - return { - ext: "gif", - mime: "image/gif" - }; - } - - if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) { - return { - ext: "webp", - mime: "image/webp" - }; - } - - // needs to be before `tif` check - if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { - return { - ext: "cr2", - mime: "image/x-canon-cr2" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) { - return { - ext: "tif", - mime: "image/tiff" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x4D) { - return { - ext: "bmp", - mime: "image/bmp" - }; - } - - if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) { - return { - ext: "jxr", - mime: "image/vnd.ms-photo" - }; - } - - if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) { - return { - ext: "psd", - mime: "image/vnd.adobe.photoshop" - }; - } - - // needs to be before `zip` check - if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) { - return { - ext: "epub", - mime: "application/epub+zip" - }; - } - - if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) { - return { - ext: "zip", - mime: "application/zip" - }; - } - - if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { - return { - ext: "tar", - mime: "application/x-tar" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) { - return { - ext: "rar", - mime: "application/x-rar-compressed" - }; - } - - if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) { - return { - ext: "gz", - mime: "application/gzip" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) { - return { - ext: "bz2", - mime: "application/x-bzip2" - }; - } - - if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) { - return { - ext: "7z", - mime: "application/x-7z-compressed" - }; - } - - if (buf[0] === 0x78 && buf[1] === 0x01) { - return { - ext: "dmg, zlib", - mime: "application/x-apple-diskimage, application/x-deflate" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) { - return { - ext: "mp4", - mime: "video/mp4" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) { - return { - ext: "m4v", - mime: "video/x-m4v" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) { - return { - ext: "mid", - mime: "audio/midi" - }; - } - - // needs to be before the `webm` check - if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) { - return { - ext: "mkv", - mime: "video/x-matroska" - }; - } - - if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) { - return { - ext: "webm", - mime: "video/webm" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { - return { - ext: "mov", - mime: "video/quicktime" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) { - return { - ext: "avi", - mime: "video/x-msvideo" - }; - } - - if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) { - return { - ext: "wmv", - mime: "video/x-ms-wmv" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") { - return { - ext: "mpg", - mime: "video/mpeg" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) { - return { - ext: "mp3", - mime: "audio/mpeg" - }; - } - - if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) { - return { - ext: "m4a", - mime: "audio/m4a" - }; - } - - if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) { - return { - ext: "ogg", - mime: "audio/ogg" - }; - } - - if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { - return { - ext: "flac", - mime: "audio/x-flac" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) { - return { - ext: "wav", - mime: "audio/x-wav" - }; - } - - if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { - return { - ext: "amr", - mime: "audio/amr" - }; - } - - if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) { - return { - ext: "pdf", - mime: "application/pdf" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x5A) { - return { - ext: "exe", - mime: "application/x-msdownload" - }; - } - - if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { - return { - ext: "swf", - mime: "application/x-shockwave-flash" - }; - } - - if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) { - return { - ext: "rtf", - mime: "application/rtf" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff", - mime: "application/font-woff" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff2", - mime: "application/font-woff" - }; - } - - if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) { - return { - ext: "eot", - mime: "application/octet-stream" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) { - return { - ext: "ttf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) { - return { - ext: "otf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) { - return { - ext: "ico", - mime: "image/x-icon" - }; - } - - if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) { - return { - ext: "flv", - mime: "video/x-flv" - }; - } - - if (buf[0] === 0x25 && buf[1] === 0x21) { - return { - ext: "ps", - mime: "application/postscript" - }; - } - - if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) { - return { - ext: "xz", - mime: "application/x-xz" - }; - } - - if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) { - return { - ext: "sqlite", - mime: "application/x-sqlite3" - }; - } - - /** - * - * Added by n1474335 [n1474335@gmail.com] from here on - * - */ - if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) { - return { - ext: "z, tar.z", - mime: "application/x-gtar" - }; - } - - if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) { - return { - ext: "none, axf, bin, elf, o, prx, puff, so", - mime: "application/x-executable", - desc: "Executable and Linkable Format file. No standard file extension." - }; - } - - if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) { - return { - ext: "class", - mime: "application/java-vm" - }; - } - - if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) { - return { - ext: "txt", - mime: "text/plain", - desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files." - }; - } - - // Must be before Little-endian UTF-16 BOM - if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) { - return { - ext: "UTF32LE", - mime: "charset/utf32le", - desc: "Little-endian UTF-32 encoded Unicode byte order mark detected." - }; - } - - if (buf[0] === 0xFF && buf[1] === 0xFE) { - return { - ext: "UTF16LE", - mime: "charset/utf16le", - desc: "Little-endian UTF-16 encoded Unicode byte order mark detected." - }; - } - - if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) || - (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) || - (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) { - return { - ext: "iso", - mime: "application/octet-stream", - desc: "ISO 9660 CD/DVD image file" - }; - } - - if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) { - return { - ext: "doc, xls, ppt", - mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint", - desc: "Microsoft Office documents" - }; - } - - if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) { - return { - ext: "dex", - mime: "application/octet-stream", - desc: "Dalvik Executable (Android)" - }; - } - - if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) { - return { - ext: "vmdk", - mime: "application/vmdk, application/x-virtualbox-vmdk" - }; - } - - if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) { - return { - ext: "crx", - mime: "application/crx", - desc: "Google Chrome extension or packaged app" - }; - } - - if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) { - return { - ext: "zlib", - mime: "application/x-deflate" - }; - } - - return null; - } - } + /** * Byte frequencies of various languages generated from Wikipedia dumps taken in late 2017 and early 2018. * The Chi-Squared test cannot accept expected values of 0, so 0.0001 has been used to account for bytes diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 5f84e13c..19903117 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -11,13 +11,6 @@ /** * A Stream can be used to traverse a binary blob, interpreting sections of it * as various data types. - * - * @param {Uint8Array} bytes - * @param {Object} fileDetail - * @param {string} fileDetail.mime - * @param {string} fileDetail.ext - * @param {number} fileDetail.offset - * @returns {File} */ export default class Stream { diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index 1d6897a0..bcb4d2a5 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -5,7 +5,7 @@ */ import Operation from "../Operation"; -import Magic from "../lib/Magic"; +import {detectFileType} from "../lib/FileType"; /** * Detect File Type operation @@ -34,17 +34,21 @@ class DetectFileType extends Operation { */ run(input, args) { const data = new Uint8Array(input), - type = Magic.magicFileType(data); + types = detectFileType(data); - if (!type) { + if (!types.length) { return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?"; } else { - let output = "File extension: " + type.ext + "\n" + - "MIME type: " + type.mime; + let output; - if (type.desc && type.desc.length) { - output += "\nDescription: " + type.desc; - } + types.forEach(type => { + output += "File extension: " + type.extension + "\n" + + "MIME type: " + type.mime + "\n"; + + if (type.description && type.description.length) { + output += "\nDescription: " + type.description + "\n"; + } + }); return output; } diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index c213b256..3a87cd5e 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -6,9 +6,8 @@ import Operation from "../Operation"; // import OperationError from "../errors/OperationError"; -import Magic from "../lib/Magic"; import Utils from "../Utils"; -import {extractFile} from "../lib/FileExtraction"; +import {detectFileType, extractFile} from "../lib/FileType"; /** * Extract Files operation @@ -40,13 +39,13 @@ class ExtractFiles extends Operation { const bytes = new Uint8Array(input); // Scan for embedded files - const fileDetails = scanForEmbeddedFiles(bytes); + const detectedFiles = scanForEmbeddedFiles(bytes); // Extract each file that we support const files = []; - fileDetails.forEach(fileDetail => { + detectedFiles.forEach(detectedFile => { try { - files.push(extractFile(bytes, fileDetail)); + files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); } catch (err) {} }); @@ -70,22 +69,21 @@ class ExtractFiles extends Operation { * @param data */ function scanForEmbeddedFiles(data) { - let type; - const types = []; + const detectedFiles = []; for (let i = 0; i < data.length; i++) { - type = Magic.magicFileType(data.slice(i)); - if (type) { - types.push({ - offset: i, - ext: type.ext, - mime: type.mime, - desc: type.desc + const fileDetails = detectFileType(data.slice(i)); + if (fileDetails.length) { + fileDetails.forEach(match => { + detectedFiles.push({ + offset: i, + fileDetails: match, + }); }); } } - return types; + return detectedFiles; } export default ExtractFiles; diff --git a/src/core/operations/RenderImage.mjs b/src/core/operations/RenderImage.mjs index 7edd2072..4554c96b 100644 --- a/src/core/operations/RenderImage.mjs +++ b/src/core/operations/RenderImage.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import {detectFileType} from "../lib/FileType"; /** * Render Image operation @@ -72,8 +72,8 @@ class RenderImage extends Operation { } // Determine file type - const type = Magic.magicFileType(input); - if (!(type && type.mime.indexOf("image") === 0)) { + const types = detectFileType(input); + if (!(types.length && types[0].mime.indexOf("image") === 0)) { throw new OperationError("Invalid file type"); } @@ -92,9 +92,9 @@ class RenderImage extends Operation { let dataURI = "data:"; // Determine file type - const type = Magic.magicFileType(data); - if (type && type.mime.indexOf("image") === 0) { - dataURI += type.mime + ";"; + const types = detectFileType(data); + if (types.length && types[0].mime.indexOf("image") === 0) { + dataURI += types[0].mime + ";"; } else { throw new OperationError("Invalid file type"); } diff --git a/src/core/operations/ScanForEmbeddedFiles.mjs b/src/core/operations/ScanForEmbeddedFiles.mjs index a38c477f..41ea911b 100644 --- a/src/core/operations/ScanForEmbeddedFiles.mjs +++ b/src/core/operations/ScanForEmbeddedFiles.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import {detectFileType} from "../lib/FileType"; /** * Scan for Embedded Files operation @@ -41,7 +41,7 @@ class ScanForEmbeddedFiles extends Operation { */ run(input, args) { let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n", - type, + types, numFound = 0, numCommonFound = 0; const ignoreCommon = args[0], @@ -49,20 +49,23 @@ class ScanForEmbeddedFiles extends Operation { data = new Uint8Array(input); for (let i = 0; i < data.length; i++) { - type = Magic.magicFileType(data.slice(i)); - if (type) { - if (ignoreCommon && commonExts.indexOf(type.ext) > -1) { - numCommonFound++; - continue; - } - numFound++; - output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" + - " File extension: " + type.ext + "\n" + - " MIME type: " + type.mime + "\n"; + types = detectFileType(data.slice(i)); + if (types.length) { + types.forEach(type => { + if (ignoreCommon && commonExts.indexOf(type.extension) > -1) { + numCommonFound++; + return; + } - if (type.desc && type.desc.length) { - output += " Description: " + type.desc + "\n"; - } + numFound++; + output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" + + " File extension: " + type.extension + "\n" + + " MIME type: " + type.mime + "\n"; + + if (type.description && type.description.length) { + output += " Description: " + type.description + "\n"; + } + }); } } From 8d3836cb16453d969d6d8a1e25f5e650e8d10394 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 21 Dec 2018 12:48:08 +0000 Subject: [PATCH 003/110] Added support for a number of further file types and file detection methods. --- src/core/lib/FileType.mjs | 256 +++++++++++++++++-------- src/core/operations/DetectFileType.mjs | 2 +- src/core/operations/PlayMedia.mjs | 14 +- 3 files changed, 189 insertions(+), 83 deletions(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 443bdd36..d38ddc80 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -13,7 +13,7 @@ import Stream from "./Stream"; * to extract them where possible. */ const FILE_SIGNATURES = { - "Pictures": [ + "Images": [ { name: "JPEG Image", extension: "jpg", @@ -51,7 +51,165 @@ const FILE_SIGNATURES = { }, extractor: null }, - + { + name: "WEBP Image", + extension: "webp", + mime: "image/webp", + description: "", + signature: { + 8: 0x57, + 9: 0x45, + 10: 0x42, + 11: 0x50 + }, + extractor: null + }, + { + name: "TIFF Image", + extension: "tif", + mime: "image/tiff", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x49, + 2: 0x2a, + 3: 0x0 + }, + { + 0: 0x4d, + 1: 0x4d, + 2: 0x0, + 3: 0x2a + } + ], + extractor: null + }, /* + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + },*/ + ], + "Video": [ + { + name: "WEBM", + extension: "webm", + mime: "video/webm", + description: "", + signature: { + 0: 0x1a, + 1: 0x45, + 2: 0xdf, + 3: 0xa3 + }, + extractor: null + }, + ], + "Audio": [ + { + name: "WAV", + extension: "wav", + mime: "audio/x-wav", + description: "", + signature: { + 0: 0x52, + 1: 0x49, + 2: 0x46, + 3: 0x46, + 8: 0x57, + 9: 0x41, + 10: 0x56, + 11: 0x45 + }, + extractor: null + }, + { + name: "OGG", + extension: "ogg", + mime: "audio/ogg", + description: "", + signature: { + 0: 0x4f, + 1: 0x67, + 2: 0x67, + 3: 0x53 + }, + extractor: null + }, ], "Documents": [ { @@ -103,13 +261,31 @@ const FILE_SIGNATURES = { /** * Checks whether a signature matches a buffer. * - * @param {Object} sig - A dictionary of offsets with values assigned to them. These - * values can be numbers for static checks, arrays of potential valid matches, or - * bespoke functions to check the validity of the buffer value at that offset. + * @param {Object|Object[]} sig - A dictionary of offsets with values assigned to them. + * These values can be numbers for static checks, arrays of potential valid matches, + * or bespoke functions to check the validity of the buffer value at that offset. * @param {Uint8Array} buf * @returns {boolean} */ function signatureMatches(sig, buf) { + if (sig instanceof Array) { + return sig.reduce((acc, s) => acc || bytesMatch(s, buf), false); + } else { + return bytesMatch(sig, buf); + } +} + + +/** + * Checks whether a set of bytes match the given buffer. + * + * @param {Object} sig - A dictionary of offsets with values assigned to them. + * These values can be numbers for static checks, arrays of potential valid matches, + * or bespoke functions to check the validity of the buffer value at that offset. + * @param {Uint8Array} buf + * @returns {boolean} + */ +function bytesMatch(sig, buf) { for (const offset in sig) { switch (typeof sig[offset]) { case "number": // Static check @@ -165,34 +341,6 @@ export function detectFileType(buf) { /* - if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) { - return { - ext: "jpg", - mime: "image/jpeg" - }; - } - - if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) { - return { - ext: "png", - mime: "image/png" - }; - } - - if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) { - return { - ext: "gif", - mime: "image/gif" - }; - } - - if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) { - return { - ext: "webp", - mime: "image/webp" - }; - } - // needs to be before `tif` check if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { return { @@ -237,13 +385,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) { - return { - ext: "zip", - mime: "application/zip" - }; - } - if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { return { ext: "tar", @@ -315,13 +456,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) { - return { - ext: "webm", - mime: "video/webm" - }; - } - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { return { ext: "mov", @@ -364,13 +498,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) { - return { - ext: "ogg", - mime: "audio/ogg" - }; - } - if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { return { ext: "flac", @@ -378,13 +505,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) { - return { - ext: "wav", - mime: "audio/x-wav" - }; - } - if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { return { ext: "amr", @@ -392,20 +512,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) { - return { - ext: "pdf", - mime: "application/pdf" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x5A) { - return { - ext: "exe", - mime: "application/x-msdownload" - }; - } - if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { return { ext: "swf", diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index bcb4d2a5..55db5edf 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -39,7 +39,7 @@ class DetectFileType extends Operation { if (!types.length) { return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?"; } else { - let output; + let output = ""; types.forEach(type => { output += "File extension: " + type.extension + "\n" + diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs index 81328a73..d0ec78cc 100644 --- a/src/core/operations/PlayMedia.mjs +++ b/src/core/operations/PlayMedia.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import { detectFileType } from "../lib/FileType"; /** * PlayMedia operation @@ -66,8 +66,8 @@ class PlayMedia extends Operation { // Determine file type - const type = Magic.magicFileType(input); - if (!(type && /^audio|video/.test(type.mime))) { + const types = detectFileType(input); + if (!(types && types.length && /^audio|video/.test(types[0].mime))) { throw new OperationError("Invalid or unrecognised file type"); } @@ -84,15 +84,15 @@ class PlayMedia extends Operation { async present(data) { if (!data.length) return ""; - const type = Magic.magicFileType(data); - const matches = /^audio|video/.exec(type.mime); + const types = detectFileType(data); + const matches = /^audio|video/.exec(types[0].mime); if (!matches) { throw new OperationError("Invalid file type"); } - const dataURI = `data:${type.mime};base64,${toBase64(data)}`; + const dataURI = `data:${types[0].mime};base64,${toBase64(data)}`; const element = matches[0]; - let html = `<${element} src='${dataURI}' type='${type.mime}' controls>`; + let html = `<${element} src='${dataURI}' type='${types[0].mime}' controls>`; html += "

Unsupported media type.

"; html += ``; return html; From f4f9b5c91c95f776c2b48a4c0733416af908e63b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 26 Dec 2018 18:40:27 +0000 Subject: [PATCH 004/110] Added 'isImage' and 'isType' functions --- src/core/lib/FileType.mjs | 33 ++++++++ src/core/operations/GenerateQRCode.mjs | 8 +- src/core/operations/ParseQRCode.mjs | 89 ++++++++++----------- src/core/operations/PlayMedia.mjs | 5 +- src/core/operations/RenderImage.mjs | 11 ++- src/core/operations/SplitColourChannels.mjs | 89 ++++++++++----------- 6 files changed, 130 insertions(+), 105 deletions(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index d38ddc80..ef6cfb03 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -703,6 +703,39 @@ export function detectFileType(buf) { } +/** + * Detects whether the given buffer is a file of the type specified. + * + * @param {string|RegExp} type + * @param {Uint8Array} buf + * @returns {string|false} The mime type or false if the type does not match + */ +export function isType(type, buf) { + const types = detectFileType(buf); + + if (!(types && types.length)) return false; + + if (typeof type === "string") { + return types[0].mime.startsWith(type) ? types[0].mime : false; + } else if (type instanceof RegExp) { + return type.test(types[0].mime) ? types[0].mime : false; + } else { + throw new Error("Invalid type input."); + } +} + + +/** + * Detects whether the given buffer contains an image file. + * + * @param {Uint8Array} buf + * @returns {string|false} The mime type or false if the type does not match + */ +export function isImage(buf) { + return isType("image", buf); +} + + /** * Attempts to extract a file from a data stream given its offset and extractor function. * diff --git a/src/core/operations/GenerateQRCode.mjs b/src/core/operations/GenerateQRCode.mjs index edab6d40..ac7e5c5c 100644 --- a/src/core/operations/GenerateQRCode.mjs +++ b/src/core/operations/GenerateQRCode.mjs @@ -8,7 +8,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import qr from "qr-image"; import { toBase64 } from "../lib/Base64"; -import Magic from "../lib/Magic"; +import { isImage } from "../lib/FileType"; import Utils from "../Utils"; /** @@ -100,9 +100,9 @@ class GenerateQRCode extends Operation { if (format === "PNG") { let dataURI = "data:"; - const type = Magic.magicFileType(data); - if (type && type.mime.indexOf("image") === 0){ - dataURI += type.mime + ";"; + const mime = isImage(data); + if (mime){ + dataURI += mime + ";"; } else { throw new OperationError("Invalid PNG file generated by QR image"); } diff --git a/src/core/operations/ParseQRCode.mjs b/src/core/operations/ParseQRCode.mjs index 75a24d55..ef7af6d7 100644 --- a/src/core/operations/ParseQRCode.mjs +++ b/src/core/operations/ParseQRCode.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; -import Magic from "../lib/Magic"; +import { isImage } from "../lib/FileType"; import jsqr from "jsqr"; import jimp from "jimp"; @@ -42,64 +42,61 @@ class ParseQRCode extends Operation { * @returns {string} */ async run(input, args) { - const type = Magic.magicFileType(input); const [normalise] = args; // Make sure that the input is an image - if (type && type.mime.indexOf("image") === 0) { - let image = input; + if (!isImage(input)) throw new OperationError("Invalid file type."); - if (normalise) { - // Process the image to be easier to read by jsqr - // Disables the alpha channel - // Sets the image default background to white - // Normalises the image colours - // Makes the image greyscale - // Converts image to a JPEG - image = await new Promise((resolve, reject) => { - jimp.read(Buffer.from(input)) - .then(image => { - image - .rgba(false) - .background(0xFFFFFFFF) - .normalize() - .greyscale() - .getBuffer(jimp.MIME_JPEG, (error, result) => { - resolve(result); - }); - }) - .catch(err => { - reject(new OperationError("Error reading the image file.")); - }); - }); - } + let image = input; - if (image instanceof OperationError) { - throw image; - } - - return new Promise((resolve, reject) => { - jimp.read(Buffer.from(image)) + if (normalise) { + // Process the image to be easier to read by jsqr + // Disables the alpha channel + // Sets the image default background to white + // Normalises the image colours + // Makes the image greyscale + // Converts image to a JPEG + image = await new Promise((resolve, reject) => { + jimp.read(Buffer.from(input)) .then(image => { - if (image.bitmap != null) { - const qrData = jsqr(image.bitmap.data, image.getWidth(), image.getHeight()); - if (qrData != null) { - resolve(qrData.data); - } else { - reject(new OperationError("Couldn't read a QR code from the image.")); - } - } else { - reject(new OperationError("Error reading the image file.")); - } + image + .rgba(false) + .background(0xFFFFFFFF) + .normalize() + .greyscale() + .getBuffer(jimp.MIME_JPEG, (error, result) => { + resolve(result); + }); }) .catch(err => { reject(new OperationError("Error reading the image file.")); }); }); - } else { - throw new OperationError("Invalid file type."); } + if (image instanceof OperationError) { + throw image; + } + + return new Promise((resolve, reject) => { + jimp.read(Buffer.from(image)) + .then(image => { + if (image.bitmap != null) { + const qrData = jsqr(image.bitmap.data, image.getWidth(), image.getHeight()); + if (qrData != null) { + resolve(qrData.data); + } else { + reject(new OperationError("Couldn't read a QR code from the image.")); + } + } else { + reject(new OperationError("Error reading the image file.")); + } + }) + .catch(err => { + reject(new OperationError("Error reading the image file.")); + }); + }); + } } diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs index d0ec78cc..98b7d088 100644 --- a/src/core/operations/PlayMedia.mjs +++ b/src/core/operations/PlayMedia.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import { detectFileType } from "../lib/FileType"; +import { isType, detectFileType } from "../lib/FileType"; /** * PlayMedia operation @@ -66,8 +66,7 @@ class PlayMedia extends Operation { // Determine file type - const types = detectFileType(input); - if (!(types && types.length && /^audio|video/.test(types[0].mime))) { + if (!isType(/^(audio|video)/, input)) { throw new OperationError("Invalid or unrecognised file type"); } diff --git a/src/core/operations/RenderImage.mjs b/src/core/operations/RenderImage.mjs index 4554c96b..07866eaf 100644 --- a/src/core/operations/RenderImage.mjs +++ b/src/core/operations/RenderImage.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import {detectFileType} from "../lib/FileType"; +import {isImage} from "../lib/FileType"; /** * Render Image operation @@ -72,8 +72,7 @@ class RenderImage extends Operation { } // Determine file type - const types = detectFileType(input); - if (!(types.length && types[0].mime.indexOf("image") === 0)) { + if (!isImage(input)) { throw new OperationError("Invalid file type"); } @@ -92,9 +91,9 @@ class RenderImage extends Operation { let dataURI = "data:"; // Determine file type - const types = detectFileType(data); - if (types.length && types[0].mime.indexOf("image") === 0) { - dataURI += types[0].mime + ";"; + const mime = isImage(data); + if (mime) { + dataURI += mime + ";"; } else { throw new OperationError("Invalid file type"); } diff --git a/src/core/operations/SplitColourChannels.mjs b/src/core/operations/SplitColourChannels.mjs index f11ca14e..c38af409 100644 --- a/src/core/operations/SplitColourChannels.mjs +++ b/src/core/operations/SplitColourChannels.mjs @@ -7,7 +7,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import {isImage} from "../lib/FileType"; import jimp from "jimp"; @@ -38,56 +38,53 @@ class SplitColourChannels extends Operation { * @returns {List} */ async run(input, args) { - const type = Magic.magicFileType(input); // Make sure that the input is an image - if (type && type.mime.indexOf("image") === 0) { - const parsedImage = await jimp.read(Buffer.from(input)); + if (!isImage(input)) throw new OperationError("Invalid file type."); - const red = new Promise(async (resolve, reject) => { - try { - const split = parsedImage - .clone() - .color([ - {apply: "blue", params: [-255]}, - {apply: "green", params: [-255]} - ]) - .getBufferAsync(jimp.MIME_PNG); - resolve(new File([new Uint8Array((await split).values())], "red.png", {type: "image/png"})); - } catch (err) { - reject(new OperationError(`Could not split red channel: ${err}`)); - } - }); + const parsedImage = await jimp.read(Buffer.from(input)); - const green = new Promise(async (resolve, reject) => { - try { - const split = parsedImage.clone() - .color([ - {apply: "red", params: [-255]}, - {apply: "blue", params: [-255]}, - ]).getBufferAsync(jimp.MIME_PNG); - resolve(new File([new Uint8Array((await split).values())], "green.png", {type: "image/png"})); - } catch (err) { - reject(new OperationError(`Could not split green channel: ${err}`)); - } - }); + const red = new Promise(async (resolve, reject) => { + try { + const split = parsedImage + .clone() + .color([ + {apply: "blue", params: [-255]}, + {apply: "green", params: [-255]} + ]) + .getBufferAsync(jimp.MIME_PNG); + resolve(new File([new Uint8Array((await split).values())], "red.png", {type: "image/png"})); + } catch (err) { + reject(new OperationError(`Could not split red channel: ${err}`)); + } + }); - const blue = new Promise(async (resolve, reject) => { - try { - const split = parsedImage - .color([ - {apply: "red", params: [-255]}, - {apply: "green", params: [-255]}, - ]).getBufferAsync(jimp.MIME_PNG); - resolve(new File([new Uint8Array((await split).values())], "blue.png", {type: "image/png"})); - } catch (err) { - reject(new OperationError(`Could not split blue channel: ${err}`)); - } - }); + const green = new Promise(async (resolve, reject) => { + try { + const split = parsedImage.clone() + .color([ + {apply: "red", params: [-255]}, + {apply: "blue", params: [-255]}, + ]).getBufferAsync(jimp.MIME_PNG); + resolve(new File([new Uint8Array((await split).values())], "green.png", {type: "image/png"})); + } catch (err) { + reject(new OperationError(`Could not split green channel: ${err}`)); + } + }); - return await Promise.all([red, green, blue]); - } else { - throw new OperationError("Invalid file type."); - } + const blue = new Promise(async (resolve, reject) => { + try { + const split = parsedImage + .color([ + {apply: "red", params: [-255]}, + {apply: "green", params: [-255]}, + ]).getBufferAsync(jimp.MIME_PNG); + resolve(new File([new Uint8Array((await split).values())], "blue.png", {type: "image/png"})); + } catch (err) { + reject(new OperationError(`Could not split blue channel: ${err}`)); + } + }); + + return await Promise.all([red, green, blue]); } /** From 729307336e58df5beb97221e3e58bc1a2eedd118 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 26 Dec 2018 23:19:46 +0000 Subject: [PATCH 005/110] Converted all previous file signatures to the new format. --- src/core/lib/FileSignatures.mjs | 1136 +++++++++++++++++++++++++++++++ src/core/lib/FileType.mjs | 811 +--------------------- 2 files changed, 1141 insertions(+), 806 deletions(-) create mode 100644 src/core/lib/FileSignatures.mjs diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs new file mode 100644 index 00000000..69699d7d --- /dev/null +++ b/src/core/lib/FileSignatures.mjs @@ -0,0 +1,1136 @@ +/** + * File signatures and extractor functions + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ +import Stream from "./Stream"; + +/** + * A categorised table of file types, including signatures to identify them and functions + * to extract them where possible. + */ +export const FILE_SIGNATURES = { + "Images": [ + { + name: "Joint Photographic Experts Group image", + extension: "jpg", + mime: "image/jpeg", + description: "", + signature: { + 0: 0xff, + 1: 0xd8, + 2: 0xff + }, + extractor: extractJPEG + }, + { + name: "Graphics Interchange Format image", + extension: "gif", + mime: "image/gif", + description: "", + signature: { + 0: 0x47, + 1: 0x49, + 2: 0x46 + }, + extractor: null + }, + { + name: "Portable Network Graphics image", + extension: "png", + mime: "image/png", + description: "", + signature: { + 0: 0x89, + 1: 0x50, + 2: 0x4e, + 3: 0x47 + }, + extractor: null + }, + { + name: "WEBP Image", + extension: "webp", + mime: "image/webp", + description: "", + signature: { + 8: 0x57, + 9: 0x45, + 10: 0x42, + 11: 0x50 + }, + extractor: null + }, + { // Place before tiff check + name: "Canon CR2 raw image", + extension: "cr2", + mime: "image/x-canon-cr2", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x49, + 2: 0x2a, + 3: 0x0, + 8: 0x43, + 9: 0x52 + }, + { + 0: 0x4d, + 1: 0x4d, + 2: 0x0, + 3: 0x2a, + 8: 0x43, + 9: 0x52 + } + ], + extractor: null + }, + { + name: "Tagged Image File Format image", + extension: "tif", + mime: "image/tiff", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x49, + 2: 0x2a, + 3: 0x0 + }, + { + 0: 0x4d, + 1: 0x4d, + 2: 0x0, + 3: 0x2a + } + ], + extractor: null + }, + { + name: "Bitmap image", + extension: "bmp", + mime: "image/bmp", + description: "", + signature: { + 0: 0x42, + 1: 0x4d + }, + extractor: null + }, + { + name: "JPEG Extended Range image", + extension: "jxr", + mime: "image/vnd.ms-photo", + description: "", + signature: { + 0: 0x49, + 1: 0x49, + 2: 0xbc + }, + extractor: null + }, + { + name: "Photoshop image", + extension: "psd", + mime: "image/vnd.adobe.photoshop", + description: "", + signature: { + 0: 0x38, + 1: 0x42, + 2: 0x50, + 3: 0x53 + }, + extractor: null + }, + { + name: "Icon image", + extension: "ico", + mime: "image/x-icon", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x1, + 3: 0x0 + }, + extractor: null + } + ], + "Video": [ + { // Place before webm + name: "Matroska Multimedia Container", + extension: "mkv", + mime: "video/x-matroska", + description: "", + signature: { + 31: 0x6d, + 32: 0x61, + 33: 0x74, + 34: 0x72, + 35: 0x6f, + 36: 0x73, + 37: 0x6b, + 38: 0x61 + }, + extractor: null + }, + { + name: "WEBM video", + extension: "webm", + mime: "video/webm", + description: "", + signature: { + 0: 0x1a, + 1: 0x45, + 2: 0xdf, + 3: 0xa3 + }, + extractor: null + }, + { + name: "MPEG-4 video", + extension: "mp4", + mime: "video/mp4", + description: "", + signature: [ + { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: [0x18, 0x20], + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70 + }, + { + 0: 0x33, // 3gp5 + 1: 0x67, + 2: 0x70, + 3: 0x35 + }, + { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: 0x1c, + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: 0x6d, + 9: 0x70, + 10: 0x34, + 11: 0x32, + 16: 0x6d, // mp41mp42isom + 17: 0x70, + 18: 0x34, + 19: 0x31, + 20: 0x6d, + 21: 0x70, + 22: 0x34, + 23: 0x32, + 24: 0x69, + 25: 0x73, + 26: 0x6f, + 27: 0x6d + } + ], + extractor: null + }, + { + name: "M4V video", + extension: "m4v", + mime: "video/x-m4v", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: 0x1c, + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: 0x4d, + 9: 0x34, + 10: 0x56 + }, + extractor: null + }, + { + name: "Quicktime video", + extension: "mov", + mime: "video/quicktime", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: 0x14, + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70 + }, + extractor: null + }, + { + name: "Audio Video Interleave", + extension: "avi", + mime: "video/x-msvideo", + description: "", + signature: { + 0: 0x52, + 1: 0x49, + 2: 0x46, + 3: 0x46, + 8: 0x41, + 9: 0x56, + 10: 0x49 + }, + extractor: null + }, + { + name: "Windows Media Video", + extension: "wmv", + mime: "video/x-ms-wmv", + description: "", + signature: { + 0: 0x30, + 1: 0x26, + 2: 0xb2, + 3: 0x75, + 4: 0x8e, + 5: 0x66, + 6: 0xcf, + 7: 0x11, + 8: 0xa6, + 9: 0xd9 + }, + extractor: null + }, + { + name: "MPEG video", + extension: "mpg", + mime: "video/mpeg", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x1, + 3: 0xba + }, + extractor: null + }, + { + name: "Flash Video", + extension: "flv", + mime: "video/x-flv", + description: "", + signature: { + 0: 0x46, + 1: 0x4c, + 2: 0x56, + 3: 0x1 + }, + extractor: null + }, + ], + "Audio": [ + { + name: "Waveform Audio", + extension: "wav", + mime: "audio/x-wav", + description: "", + signature: { + 0: 0x52, + 1: 0x49, + 2: 0x46, + 3: 0x46, + 8: 0x57, + 9: 0x41, + 10: 0x56, + 11: 0x45 + }, + extractor: null + }, + { + name: "OGG audio", + extension: "ogg", + mime: "audio/ogg", + description: "", + signature: { + 0: 0x4f, + 1: 0x67, + 2: 0x67, + 3: 0x53 + }, + extractor: null + }, + { + name: "Musical Instrument Digital Interface audio", + extension: "midi", + mime: "audio/midi", + description: "", + signature: { + 0: 0x4d, + 1: 0x54, + 2: 0x68, + 3: 0x64 + }, + extractor: null + }, + { + name: "MPEG-3 audio", + extension: "mp3", + mime: "audio/mpeg", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x44, + 2: 0x33 + }, + { + 0: 0xff, + 1: 0xfb + } + ], + extractor: null + }, + { + name: "MPEG-4 Part 14 audio", + extension: "m4a", + mime: "audio/m4a", + description: "", + signature: [ + { + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: 0x4d, + 9: 0x34, + 10: 0x41 + }, + { + 0: 0x4d, + 1: 0x34, + 2: 0x41, + 3: 0x20 + } + ], + extractor: null + }, + { + name: "Free Lossless Audio Codec", + extension: "flac", + mime: "audio/x-flac", + description: "", + signature: { + 0: 0x66, + 1: 0x4c, + 2: 0x61, + 3: 0x43 + }, + extractor: null + }, + { + name: "Adaptive Multi-Rate audio codec", + extension: "amr", + mime: "audio/amr", + description: "", + signature: { + 0: 0x23, + 1: 0x21, + 2: 0x41, + 3: 0x4d, + 4: 0x52, + 5: 0x0a + }, + extractor: null + }, + ], + "Documents": [ + { + name: "Portable Document Format", + extension: "pdf", + mime: "application/pdf", + description: "", + signature: { + 0: 0x25, + 1: 0x50, + 2: 0x44, + 3: 0x46 + }, + extractor: extractPDF + }, + { + name: "PostScript", + extension: "ps", + mime: "application/postscript", + description: "", + signature: { + 0: 0x25, + 1: 0x21 + }, + extractor: null + }, + { + name: "Rich Text Format", + extension: "rtf", + mime: "application/rtf", + description: "", + signature: { + 0: 0x7b, + 1: 0x5c, + 2: 0x72, + 3: 0x74, + 4: 0x66 + }, + extractor: null + }, + { + name: "Microsoft Office documents/OLE2", + extension: "ole2,doc,xls,dot,ppt,xla,ppa,pps,pot,msi,sdw,db,vsd,msg", + mime: "application/msword,application/vnd.ms-excel,application/vnd.ms-powerpoint", + description: "Microsoft Office documents", + signature: { + 0: 0xd0, + 1: 0xcf, + 2: 0x11, + 3: 0xe0, + 4: 0xa1, + 5: 0xb1, + 6: 0x1a, + 7: 0xe1 + }, + extractor: null + }, + { + name: "EPUB e-book", + extension: "epub", + mime: "application/epub+zip", + description: "", + signature: { + 0: 0x50, + 1: 0x4b, + 2: 0x3, + 3: 0x4, + 30: 0x6d, // mimetypeapplication/epub_zip + 31: 0x69, + 32: 0x6d, + 33: 0x65, + 34: 0x74, + 35: 0x79, + 36: 0x70, + 37: 0x65, + 38: 0x61, + 39: 0x70, + 40: 0x70, + 41: 0x6c, + 42: 0x69, + 43: 0x63, + 44: 0x61, + 45: 0x74, + 46: 0x69, + 47: 0x6f, + 48: 0x6e, + 49: 0x2f, + 50: 0x65, + 51: 0x70, + 52: 0x75, + 53: 0x62, + 54: 0x2b, + 55: 0x7a, + 56: 0x69, + 57: 0x70 + }, + extractor: null + }, + ], + "Applications": [ + { + name: "Windows Portable Executable", + extension: "exe,dll,drv,vxd,sys,ocx,vbx,com,fon,scr", + mime: "application/x-msdownload", + description: "", + signature: { + 0: 0x4d, + 1: 0x5a, + 3: [0x0, 0x1, 0x2], + 5: [0x0, 0x1, 0x2] + }, + extractor: extractMZPE + }, + { + name: "Executable and Linkable Format file", + extension: "elf,bin,axf,o,prx,so", + mime: "application/x-executable", + description: "Executable and Linkable Format file. No standard file extension.", + signature: { + 0: 0x7f, + 1: 0x45, + 2: 0x4c, + 3: 0x46 + }, + extractor: null + }, + { + name: "Adobe Flash", + extension: "swf", + mime: "application/x-shockwave-flash", + description: "", + signature: { + 0: [0x43, 0x46], + 1: 0x57, + 2: 0x53 + }, + extractor: null + }, + { + name: "Java Class", + extension: "class", + mime: "application/java-vm", + description: "", + signature: { + 0: 0xca, + 1: 0xfe, + 2: 0xba, + 3: 0xbe + }, + extractor: null + }, + { + name: "Dalvik Executable", + extension: "dex", + mime: "application/octet-stream", + description: "Dalvik Executable as used by Android", + signature: { + 0: 0x64, + 1: 0x65, + 2: 0x78, + 3: 0x0a, + 4: 0x30, + 5: 0x33, + 6: 0x35, + 7: 0x0 + }, + extractor: null + }, + { + name: "Google Chrome Extension", + extension: "crx", + mime: "application/crx", + description: "Google Chrome extension or packaged app", + signature: { + 0: 0x43, + 1: 0x72, + 2: 0x32, + 3: 0x34 + }, + extractor: null + }, + ], + "Archives": [ + { + name: "PKZIP archive", + extension: "zip", + mime: "application/zip", + description: "", + signature: { + 0: 0x50, + 1: 0x4b, + 2: [0x3, 0x5, 0x7], + 3: [0x4, 0x6, 0x8] + }, + extractor: extractZIP + }, + { + name: "TAR archive", + extension: "tar", + mime: "application/x-tar", + description: "", + signature: { + 257: 0x75, + 258: 0x73, + 259: 0x74, + 260: 0x61, + 261: 0x72 + }, + extractor: null + }, + { + name: "Roshal Archive", + extension: "rar", + mime: "application/x-rar-compressed", + description: "", + signature: { + 0: 0x52, + 1: 0x61, + 2: 0x72, + 3: 0x21, + 4: 0x1a, + 5: 0x7, + 6: [0x0, 0x1] + }, + extractor: null + }, + { + name: "Gzip", + extension: "gz", + mime: "application/gzip", + description: "", + signature: { + 0: 0x1f, + 1: 0x8b, + 2: 0x8 + }, + extractor: null + }, + { + name: "Bzip2", + extension: "bz2", + mime: "application/x-bzip2", + description: "", + signature: { + 0: 0x42, + 1: 0x5a, + 2: 0x68 + }, + extractor: null + }, + { + name: "7zip", + extension: "7z", + mime: "application/x-7z-compressed", + description: "", + signature: { + 0: 0x37, + 1: 0x7a, + 2: 0xbc, + 3: 0xaf, + 4: 0x27, + 5: 0x1c + }, + extractor: null + }, + { + name: "Zlib Deflate", + extension: "zlib", + mime: "application/x-deflate", + description: "", + signature: { + 0: 0x78, + 1: [0x1, 0x9c, 0xda, 0x5e] + }, + extractor: null + }, + { + name: "xz compression", + extension: "xz", + mime: "application/x-xz", + description: "", + signature: { + 0: 0xfd, + 1: 0x37, + 2: 0x7a, + 3: 0x58, + 4: 0x5a, + 5: 0x0 + }, + extractor: null + }, + { + name: "Tarball", + extension: "tar.z", + mime: "application/x-gtar", + description: "", + signature: { + 0: 0x1f, + 1: [0x9d, 0xa0] + }, + extractor: null + }, + { + name: "ISO disk image", + extension: "iso", + mime: "application/octet-stream", + description: "ISO 9660 CD/DVD image file", + signature: [ + { + 0x8001: 0x43, + 0x8002: 0x44, + 0x8003: 0x30, + 0x8004: 0x30, + 0x8005: 0x31 + }, + { + 0x8801: 0x43, + 0x8802: 0x44, + 0x8803: 0x30, + 0x8804: 0x30, + 0x8805: 0x31 + }, + { + 0x9001: 0x43, + 0x9002: 0x44, + 0x9003: 0x30, + 0x9004: 0x30, + 0x9005: 0x31 + } + ], + extractor: null + }, + { + name: "Virtual Machine Disk", + extension: "vmdk", + mime: "application/vmdk,application/x-virtualbox-vmdk", + description: "", + signature: { + 0: 0x4b, + 1: 0x44, + 2: 0x4d + }, + extractor: null + }, + ], + "Miscellaneous": [ + { + name: "UTF-8 text file", + extension: "txt", + mime: "text/plain", + description: "UTF-8 encoded Unicode byte order mark, commonly but not exclusively seen in text files.", + signature: { + 0: 0xef, + 1: 0xbb, + 2: 0xbf + }, + extractor: null + }, + { // Place before UTF-16 LE file + name: "UTF-32 LE file", + extension: "utf32le", + mime: "charset/utf32le", + description: "Little-endian UTF-32 encoded Unicode byte order mark.", + signature: { + 0: 0xff, + 1: 0xfe, + 2: 0x00, + 3: 0x00 + }, + extractor: null + }, + { + name: "UTF-16 LE file", + extension: "utf16le", + mime: "charset/utf16le", + description: "Little-endian UTF-16 encoded Unicode byte order mark.", + signature: { + 0: 0xff, + 1: 0xfe + }, + extractor: null + }, + { + name: "Web Open Font Format", + extension: "woff", + mime: "application/font-woff", + description: "", + signature: { + 0: 0x77, + 1: 0x4f, + 2: 0x46, + 3: 0x46, + 4: 0x0, + 5: 0x1, + 6: 0x0, + 7: 0x0 + }, + extractor: null + }, + { + name: "Web Open Font Format 2", + extension: "woff2", + mime: "application/font-woff", + description: "", + signature: { + 0: 0x77, + 1: 0x4f, + 2: 0x46, + 3: 0x32, + 4: 0x0, + 5: 0x1, + 6: 0x0, + 7: 0x0 + }, + extractor: null + }, + { + name: "Embedded OpenType font", + extension: "eot", + mime: "application/octet-stream", + description: "", + signature: [ + { + 8: 0x2, + 9: 0x0, + 10: 0x1, + 34: 0x4c, + 35: 0x50 + }, + { + 8: 0x1, + 9: 0x0, + 10: 0x0, + 34: 0x4c, + 35: 0x50 + }, + { + 8: 0x2, + 9: 0x0, + 10: 0x2, + 34: 0x4c, + 35: 0x50 + }, + ], + extractor: null + }, + { + name: "TrueType Font", + extension: "ttf", + mime: "application/font-sfnt", + description: "", + signature: { + 0: 0x0, + 1: 0x1, + 2: 0x0, + 3: 0x0, + 4: 0x0 + }, + extractor: null + }, + { + name: "OpenType Font", + extension: "otf", + mime: "application/font-sfnt", + description: "", + signature: { + 0: 0x4f, + 1: 0x54, + 2: 0x54, + 3: 0x4f, + 4: 0x0 + }, + extractor: null + }, + { + name: "SQLite", + extension: "sqlite", + mime: "application/x-sqlite3", + description: "", + signature: { + 0: 0x53, + 1: 0x51, + 2: 0x4c, + 3: 0x69 + }, + extractor: null + }, + ] +}; + + +/** + * JPEG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractJPEG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + while (stream.hasMore()) { + const marker = stream.getBytes(2); + if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + + let segmentSize = 0; + switch (marker[1]) { + // No length + case 0xd8: // Start of Image + case 0x01: // For temporary use in arithmetic coding + break; + case 0xd9: // End found + return stream.carve(); + + // Variable size segment + case 0xc0: // Start of frame (Baseline DCT) + case 0xc1: // Start of frame (Extended sequential DCT) + case 0xc2: // Start of frame (Progressive DCT) + case 0xc3: // Start of frame (Lossless sequential) + case 0xc4: // Define Huffman Table + case 0xc5: // Start of frame (Differential sequential DCT) + case 0xc6: // Start of frame (Differential progressive DCT) + case 0xc7: // Start of frame (Differential lossless) + case 0xc8: // Reserved for JPEG extensions + case 0xc9: // Start of frame (Extended sequential DCT) + case 0xca: // Start of frame (Progressive DCT) + case 0xcb: // Start of frame (Lossless sequential) + case 0xcc: // Define arithmetic conditioning table + case 0xcd: // Start of frame (Differential sequential DCT) + case 0xce: // Start of frame (Differential progressive DCT) + case 0xcf: // Start of frame (Differential lossless) + case 0xdb: // Define Quantization Table + case 0xde: // Define hierarchical progression + case 0xe0: // Application-specific + case 0xe1: // Application-specific + case 0xe2: // Application-specific + case 0xe3: // Application-specific + case 0xe4: // Application-specific + case 0xe5: // Application-specific + case 0xe6: // Application-specific + case 0xe7: // Application-specific + case 0xe8: // Application-specific + case 0xe9: // Application-specific + case 0xea: // Application-specific + case 0xeb: // Application-specific + case 0xec: // Application-specific + case 0xed: // Application-specific + case 0xee: // Application-specific + case 0xef: // Application-specific + case 0xfe: // Comment + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + break; + + // 1 byte + case 0xdf: // Expand reference image + stream.position++; + break; + + // 2 bytes + case 0xdc: // Define number of lines + case 0xdd: // Define restart interval + stream.position += 2; + break; + + // Start scan + case 0xda: // Start of scan + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + stream.continueUntil(0xff); + break; + + // Continue through encoded data + case 0x00: // Byte stuffing + case 0xd0: // Restart + case 0xd1: // Restart + case 0xd2: // Restart + case 0xd3: // Restart + case 0xd4: // Restart + case 0xd5: // Restart + case 0xd6: // Restart + case 0xd7: // Restart + stream.continueUntil(0xff); + break; + + default: + stream.continueUntil(0xff); + break; + } + } + + throw new Error("Unable to parse JPEG successfully"); +} + + +/** + * Portable executable extractor. + * Assumes that the offset refers to an MZ header. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMZPE(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to PE header pointer + stream.moveTo(0x3c); + const peAddress = stream.readInt(4, "le"); + + // Move to PE header + stream.moveTo(peAddress); + + // Get number of sections + stream.moveForwardsBy(6); + const numSections = stream.readInt(2, "le"); + + // Get optional header size + stream.moveForwardsBy(12); + const optionalHeaderSize = stream.readInt(2, "le"); + + // Move past optional header to section header + stream.moveForwardsBy(2 + optionalHeaderSize); + + // Move to final section header + stream.moveForwardsBy((numSections - 1) * 0x28); + + // Get raw data info + stream.moveForwardsBy(16); + const rawDataSize = stream.readInt(4, "le"); + const rawDataAddress = stream.readInt(4, "le"); + + // Move to end of final section + stream.moveTo(rawDataAddress + rawDataSize); + + return stream.carve(); +} + + +/** + * PDF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPDF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find end-of-file marker (%%EOF) + stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); + stream.moveForwardsBy(5); + stream.consumeIf(0x0d); + stream.consumeIf(0x0a); + + return stream.carve(); +} + + +/** + * ZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find End of central directory record + stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); + + // Get comment length and consume + stream.moveForwardsBy(20); + const commentLength = stream.readInt(2, "le"); + stream.moveForwardsBy(commentLength); + + return stream.carve(); +} diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index ef6cfb03..b96ea69e 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -6,256 +6,7 @@ * @license Apache-2.0 * */ -import Stream from "./Stream"; - -/** - * A categorised table of file types, including signatures to identifying them and functions - * to extract them where possible. - */ -const FILE_SIGNATURES = { - "Images": [ - { - name: "JPEG Image", - extension: "jpg", - mime: "image/jpeg", - description: "", - signature: { - 0: 0xff, - 1: 0xd8, - 2: 0xff - }, - extractor: extractJPEG - }, - { - name: "GIF Image", - extension: "gif", - mime: "image/gif", - description: "", - signature: { - 0: 0x47, - 1: 0x49, - 2: 0x46 - }, - extractor: null - }, - { - name: "PNG Image", - extension: "png", - mime: "image/png", - description: "", - signature: { - 0: 0x89, - 1: 0x50, - 2: 0x4e, - 3: 0x47 - }, - extractor: null - }, - { - name: "WEBP Image", - extension: "webp", - mime: "image/webp", - description: "", - signature: { - 8: 0x57, - 9: 0x45, - 10: 0x42, - 11: 0x50 - }, - extractor: null - }, - { - name: "TIFF Image", - extension: "tif", - mime: "image/tiff", - description: "", - signature: [ - { - 0: 0x49, - 1: 0x49, - 2: 0x2a, - 3: 0x0 - }, - { - 0: 0x4d, - 1: 0x4d, - 2: 0x0, - 3: 0x2a - } - ], - extractor: null - }, /* - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - },*/ - ], - "Video": [ - { - name: "WEBM", - extension: "webm", - mime: "video/webm", - description: "", - signature: { - 0: 0x1a, - 1: 0x45, - 2: 0xdf, - 3: 0xa3 - }, - extractor: null - }, - ], - "Audio": [ - { - name: "WAV", - extension: "wav", - mime: "audio/x-wav", - description: "", - signature: { - 0: 0x52, - 1: 0x49, - 2: 0x46, - 3: 0x46, - 8: 0x57, - 9: 0x41, - 10: 0x56, - 11: 0x45 - }, - extractor: null - }, - { - name: "OGG", - extension: "ogg", - mime: "audio/ogg", - description: "", - signature: { - 0: 0x4f, - 1: 0x67, - 2: 0x67, - 3: 0x53 - }, - extractor: null - }, - ], - "Documents": [ - { - name: "Portable Document Format", - extension: "pdf", - mime: "application/pdf", - description: "", - signature: { - 0: 0x25, - 1: 0x50, - 2: 0x44, - 3: 0x46 - }, - extractor: extractPDF - }, - ], - "Applications": [ - { - name: "Windows Portable Executable", - extension: "exe", - mime: "application/x-msdownload", - description: "", - signature: { - 0: 0x4d, - 1: 0x5a - }, - extractor: extractMZPE - }, - ], - "Archives": [ - { - name: "ZIP", - extension: "zip", - mime: "application/zip", - description: "", - signature: { - 0: 0x50, - 1: 0x4b, - 2: [0x3, 0x5, 0x7], - 3: [0x4, 0x6, 0x8] - }, - extractor: extractZIP - }, - - ], -}; +import {FILE_SIGNATURES} from "./FileSignatures"; /** @@ -313,7 +64,8 @@ function bytesMatch(sig, buf) { * extension and mime type. * * @param {Uint8Array} buf - * @returns {Object[]} type + * @returns {Object[]} types + * @returns {string} type.name - Name of file type * @returns {string} type.ext - File extension * @returns {string} type.mime - Mime type * @returns {string} [type.desc] - Description @@ -336,370 +88,6 @@ export function detectFileType(buf) { }); } return matchingFiles; - - // Delete all below this line once implemented in FILE_SIGNATURES above. - - - /* - // needs to be before `tif` check - if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { - return { - ext: "cr2", - mime: "image/x-canon-cr2" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) { - return { - ext: "tif", - mime: "image/tiff" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x4D) { - return { - ext: "bmp", - mime: "image/bmp" - }; - } - - if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) { - return { - ext: "jxr", - mime: "image/vnd.ms-photo" - }; - } - - if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) { - return { - ext: "psd", - mime: "image/vnd.adobe.photoshop" - }; - } - - // needs to be before `zip` check - if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) { - return { - ext: "epub", - mime: "application/epub+zip" - }; - } - - if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { - return { - ext: "tar", - mime: "application/x-tar" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) { - return { - ext: "rar", - mime: "application/x-rar-compressed" - }; - } - - if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) { - return { - ext: "gz", - mime: "application/gzip" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) { - return { - ext: "bz2", - mime: "application/x-bzip2" - }; - } - - if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) { - return { - ext: "7z", - mime: "application/x-7z-compressed" - }; - } - - if (buf[0] === 0x78 && buf[1] === 0x01) { - return { - ext: "dmg, zlib", - mime: "application/x-apple-diskimage, application/x-deflate" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) { - return { - ext: "mp4", - mime: "video/mp4" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) { - return { - ext: "m4v", - mime: "video/x-m4v" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) { - return { - ext: "mid", - mime: "audio/midi" - }; - } - - // needs to be before the `webm` check - if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) { - return { - ext: "mkv", - mime: "video/x-matroska" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { - return { - ext: "mov", - mime: "video/quicktime" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) { - return { - ext: "avi", - mime: "video/x-msvideo" - }; - } - - if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) { - return { - ext: "wmv", - mime: "video/x-ms-wmv" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") { - return { - ext: "mpg", - mime: "video/mpeg" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) { - return { - ext: "mp3", - mime: "audio/mpeg" - }; - } - - if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) { - return { - ext: "m4a", - mime: "audio/m4a" - }; - } - - if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { - return { - ext: "flac", - mime: "audio/x-flac" - }; - } - - if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { - return { - ext: "amr", - mime: "audio/amr" - }; - } - - if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { - return { - ext: "swf", - mime: "application/x-shockwave-flash" - }; - } - - if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) { - return { - ext: "rtf", - mime: "application/rtf" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff", - mime: "application/font-woff" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff2", - mime: "application/font-woff" - }; - } - - if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) { - return { - ext: "eot", - mime: "application/octet-stream" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) { - return { - ext: "ttf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) { - return { - ext: "otf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) { - return { - ext: "ico", - mime: "image/x-icon" - }; - } - - if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) { - return { - ext: "flv", - mime: "video/x-flv" - }; - } - - if (buf[0] === 0x25 && buf[1] === 0x21) { - return { - ext: "ps", - mime: "application/postscript" - }; - } - - if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) { - return { - ext: "xz", - mime: "application/x-xz" - }; - } - - if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) { - return { - ext: "sqlite", - mime: "application/x-sqlite3" - }; - } - */ - - /** - * - * Added by n1474335 [n1474335@gmail.com] from here on - * - */ - /* - if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) { - return { - ext: "z, tar.z", - mime: "application/x-gtar" - }; - } - - if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) { - return { - ext: "none, axf, bin, elf, o, prx, puff, so", - mime: "application/x-executable", - desc: "Executable and Linkable Format file. No standard file extension." - }; - } - - if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) { - return { - ext: "class", - mime: "application/java-vm" - }; - } - - if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) { - return { - ext: "txt", - mime: "text/plain", - desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files." - }; - } - - // Must be before Little-endian UTF-16 BOM - if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) { - return { - ext: "UTF32LE", - mime: "charset/utf32le", - desc: "Little-endian UTF-32 encoded Unicode byte order mark detected." - }; - } - - if (buf[0] === 0xFF && buf[1] === 0xFE) { - return { - ext: "UTF16LE", - mime: "charset/utf16le", - desc: "Little-endian UTF-16 encoded Unicode byte order mark detected." - }; - } - - if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) || - (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) || - (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) { - return { - ext: "iso", - mime: "application/octet-stream", - desc: "ISO 9660 CD/DVD image file" - }; - } - - if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) { - return { - ext: "doc, xls, ppt", - mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint", - desc: "Microsoft Office documents" - }; - } - - if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) { - return { - ext: "dex", - mime: "application/octet-stream", - desc: "Dalvik Executable (Android)" - }; - } - - if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) { - return { - ext: "vmdk", - mime: "application/vmdk, application/x-virtualbox-vmdk" - }; - } - - if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) { - return { - ext: "crx", - mime: "application/crx", - desc: "Google Chrome extension or packaged app" - }; - } - - if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) { - return { - ext: "zlib", - mime: "application/x-deflate" - }; - } - - return null; - */ } @@ -750,198 +138,9 @@ export function isImage(buf) { export function extractFile(bytes, fileDetail, offset) { if (fileDetail.extractor) { const fileData = fileDetail.extractor(bytes, offset); - return new File([fileData], `extracted_at_0x${offset.toString(16)}.${fileDetail.extension}`); + const ext = fileDetail.extension.split(",")[0]; + return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`); } throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); } - - -/** - * JPEG extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractJPEG(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - while (stream.hasMore()) { - const marker = stream.getBytes(2); - if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); - - let segmentSize = 0; - switch (marker[1]) { - // No length - case 0xd8: // Start of Image - case 0x01: // For temporary use in arithmetic coding - break; - case 0xd9: // End found - return stream.carve(); - - // Variable size segment - case 0xc0: // Start of frame (Baseline DCT) - case 0xc1: // Start of frame (Extended sequential DCT) - case 0xc2: // Start of frame (Progressive DCT) - case 0xc3: // Start of frame (Lossless sequential) - case 0xc4: // Define Huffman Table - case 0xc5: // Start of frame (Differential sequential DCT) - case 0xc6: // Start of frame (Differential progressive DCT) - case 0xc7: // Start of frame (Differential lossless) - case 0xc8: // Reserved for JPEG extensions - case 0xc9: // Start of frame (Extended sequential DCT) - case 0xca: // Start of frame (Progressive DCT) - case 0xcb: // Start of frame (Lossless sequential) - case 0xcc: // Define arithmetic conditioning table - case 0xcd: // Start of frame (Differential sequential DCT) - case 0xce: // Start of frame (Differential progressive DCT) - case 0xcf: // Start of frame (Differential lossless) - case 0xdb: // Define Quantization Table - case 0xde: // Define hierarchical progression - case 0xe0: // Application-specific - case 0xe1: // Application-specific - case 0xe2: // Application-specific - case 0xe3: // Application-specific - case 0xe4: // Application-specific - case 0xe5: // Application-specific - case 0xe6: // Application-specific - case 0xe7: // Application-specific - case 0xe8: // Application-specific - case 0xe9: // Application-specific - case 0xea: // Application-specific - case 0xeb: // Application-specific - case 0xec: // Application-specific - case 0xed: // Application-specific - case 0xee: // Application-specific - case 0xef: // Application-specific - case 0xfe: // Comment - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - break; - - // 1 byte - case 0xdf: // Expand reference image - stream.position++; - break; - - // 2 bytes - case 0xdc: // Define number of lines - case 0xdd: // Define restart interval - stream.position += 2; - break; - - // Start scan - case 0xda: // Start of scan - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - stream.continueUntil(0xff); - break; - - // Continue through encoded data - case 0x00: // Byte stuffing - case 0xd0: // Restart - case 0xd1: // Restart - case 0xd2: // Restart - case 0xd3: // Restart - case 0xd4: // Restart - case 0xd5: // Restart - case 0xd6: // Restart - case 0xd7: // Restart - stream.continueUntil(0xff); - break; - - default: - stream.continueUntil(0xff); - break; - } - } - - throw new Error("Unable to parse JPEG successfully"); -} - - -/** - * Portable executable extractor. - * Assumes that the offset refers to an MZ header. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractMZPE(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Move to PE header pointer - stream.moveTo(0x3c); - const peAddress = stream.readInt(4, "le"); - - // Move to PE header - stream.moveTo(peAddress); - - // Get number of sections - stream.moveForwardsBy(6); - const numSections = stream.readInt(2, "le"); - - // Get optional header size - stream.moveForwardsBy(12); - const optionalHeaderSize = stream.readInt(2, "le"); - - // Move past optional header to section header - stream.moveForwardsBy(2 + optionalHeaderSize); - - // Move to final section header - stream.moveForwardsBy((numSections - 1) * 0x28); - - // Get raw data info - stream.moveForwardsBy(16); - const rawDataSize = stream.readInt(4, "le"); - const rawDataAddress = stream.readInt(4, "le"); - - // Move to end of final section - stream.moveTo(rawDataAddress + rawDataSize); - - return stream.carve(); -} - - -/** - * PDF extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractPDF(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find end-of-file marker (%%EOF) - stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); - stream.moveForwardsBy(5); - stream.consumeIf(0x0d); - stream.consumeIf(0x0a); - - return stream.carve(); -} - - -/** - * ZIP extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractZIP(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find End of central directory record - stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); - - // Get comment length and consume - stream.moveForwardsBy(20); - const commentLength = stream.readInt(2, "le"); - stream.moveForwardsBy(commentLength); - - return stream.carve(); -} From 0198f05112cd0c5734ee5057be94aed23b654c8c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Dec 2018 00:03:41 +0000 Subject: [PATCH 006/110] Added and improved file signatures. --- src/core/lib/FileSignatures.mjs | 117 +++++++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 9 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 69699d7d..50a5e830 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -16,13 +16,14 @@ export const FILE_SIGNATURES = { "Images": [ { name: "Joint Photographic Experts Group image", - extension: "jpg", + extension: "jpg,jpeg,jpe,thm,mpo", mime: "image/jpeg", description: "", signature: { 0: 0xff, 1: 0xd8, - 2: 0xff + 2: 0xff, + 3: [0xc0, 0xc4, 0xdb, 0xdd, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe7, 0xe8, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xfe] }, extractor: extractJPEG }, @@ -32,9 +33,12 @@ export const FILE_SIGNATURES = { mime: "image/gif", description: "", signature: { - 0: 0x47, + 0: 0x47, // GIF 1: 0x49, - 2: 0x46 + 2: 0x46, + 3: 0x38, // 8 + 4: [0x37, 0x39], // 7|9 + 5: 0x61 // a }, extractor: null }, @@ -45,9 +49,13 @@ export const FILE_SIGNATURES = { description: "", signature: { 0: 0x89, - 1: 0x50, + 1: 0x50, // PNG 2: 0x4e, - 3: 0x47 + 3: 0x47, + 4: 0x0d, + 5: 0x0a, + 6: 0x1a, + 7: 0x0a }, extractor: null }, @@ -64,6 +72,23 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "Camera Image File Format", + extension: "crw", + mime: "image/x-canon-crw", + description: "", + signature: { + 6: 0x48, // HEAPCCDR + 7: 0x45, + 8: 0x41, + 9: 0x50, + 10: 0x43, + 11: 0x43, + 12: 0x44, + 13: 0x52 + }, + extractor: null + }, { // Place before tiff check name: "Canon CR2 raw image", extension: "cr2", @@ -117,7 +142,13 @@ export const FILE_SIGNATURES = { description: "", signature: { 0: 0x42, - 1: 0x4d + 1: 0x4d, + 7: 0x0, + 9: 0x0, + 14: [0x0c, 0x28, 0x38, 0x40, 0x6c, 0x7c], + 15: 0x0, + 16: 0x0, + 17: 0x0 }, extractor: null }, @@ -142,10 +173,52 @@ export const FILE_SIGNATURES = { 0: 0x38, 1: 0x42, 2: 0x50, - 3: 0x53 + 3: 0x53, + 4: 0x0, + 5: 0x1, + 6: 0x0, + 7: 0x0, + 8: 0x0, + 9: 0x0, + 10: 0x0, + 11: 0x0 }, extractor: null }, + { + name: "Paint Shop Pro image", + extension: "psp", + mime: "image/psp", + description: "", + signature: [ + { + 0: 0x50, // Paint Shop Pro Im + 1: 0x61, + 2: 0x69, + 3: 0x6e, + 4: 0x74, + 5: 0x20, + 6: 0x53, + 7: 0x68, + 8: 0x6f, + 9: 0x70, + 10: 0x20, + 11: 0x50, + 12: 0x72, + 13: 0x6f, + 14: 0x20, + 15: 0x49, + 16: 0x6d + }, + { + 0: 0x7e, + 1: 0x42, + 2: 0x4b, + 3: 0x0 + } + ], + extractor: null + }, { name: "Icon image", extension: "ico", @@ -155,7 +228,13 @@ export const FILE_SIGNATURES = { 0: 0x0, 1: 0x0, 2: 0x1, - 3: 0x0 + 3: 0x0, + 4: [0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15], + 5: 0x0, + 6: [0x10, 0x20, 0x30, 0x40, 0x80], + 7: [0x10, 0x20, 0x30, 0x40, 0x80], + 9: 0x00, + 10: [0x0, 0x1] }, extractor: null } @@ -512,6 +591,26 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "Microsoft Office 2007+ documents", + extension: "docx,xlsx,pptx", + mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.openxmlformats-officedocument.presentationml.presentation", + description: "", + signature: { + 38: 0x5f, // _Types].xml + 39: 0x54, + 40: 0x79, + 41: 0x70, + 42: 0x65, + 43: 0x73, + 44: 0x5d, + 45: 0x2e, + 46: 0x78, + 47: 0x6d, + 48: 0x6c + }, + extractor: null + }, { name: "EPUB e-book", extension: "epub", From 3ae225ac59c4843371ffa97a5cbd851350ca303c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sun, 30 Dec 2018 01:36:58 +0000 Subject: [PATCH 007/110] Untar operation now uses lib/Stream library --- src/core/operations/Untar.mjs | 35 ++--------------------------------- tests/browser/nightwatch.js | 2 +- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/src/core/operations/Untar.mjs b/src/core/operations/Untar.mjs index af029184..8655ba68 100644 --- a/src/core/operations/Untar.mjs +++ b/src/core/operations/Untar.mjs @@ -6,6 +6,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; +import Stream from "../lib/Stream"; /** * Untar operation @@ -41,38 +42,6 @@ class Untar extends Operation { * @returns {List} */ run(input, args) { - const Stream = function(input) { - this.bytes = input; - this.position = 0; - }; - - Stream.prototype.getBytes = function(bytesToGet) { - const newPosition = this.position + bytesToGet; - const bytes = this.bytes.slice(this.position, newPosition); - this.position = newPosition; - return bytes; - }; - - Stream.prototype.readString = function(numBytes) { - let result = ""; - for (let i = this.position; i < this.position + numBytes; i++) { - const currentByte = this.bytes[i]; - if (currentByte === 0) break; - result += String.fromCharCode(currentByte); - } - this.position += numBytes; - return result; - }; - - Stream.prototype.readInt = function(numBytes, base) { - const string = this.readString(numBytes); - return parseInt(string, base); - }; - - Stream.prototype.hasMore = function() { - return this.position < this.bytes.length; - }; - const stream = new Stream(input), files = []; @@ -85,7 +54,7 @@ class Untar extends Operation { ownerUID: stream.readString(8), ownerGID: stream.readString(8), size: parseInt(stream.readString(12), 8), // Octal - lastModTime: new Date(1000 * stream.readInt(12, 8)), // Octal + lastModTime: new Date(1000 * parseInt(stream.readString(12), 8)), // Octal checksum: stream.readString(8), type: stream.readString(1), linkedFileName: stream.readString(100), diff --git a/tests/browser/nightwatch.js b/tests/browser/nightwatch.js index 52587d2f..23100d8d 100644 --- a/tests/browser/nightwatch.js +++ b/tests/browser/nightwatch.js @@ -87,7 +87,7 @@ module.exports = { // Check output browser .useCss() - .waitForElementNotVisible("#stale-indicator", 500) + .waitForElementNotVisible("#stale-indicator", 1000) .expect.element("#output-text").to.have.value.that.equals("44 6f 6e 27 74 20 50 61 6e 69 63 2e"); // Clear recipe From ede75530d08506eb77e58557449982143c9567fe Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sun, 30 Dec 2018 02:21:45 +0000 Subject: [PATCH 008/110] Added PNG and BMP extractors --- src/core/lib/FileSignatures.mjs | 56 +++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 50a5e830..d4e19da9 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -57,7 +57,7 @@ export const FILE_SIGNATURES = { 6: 0x1a, 7: 0x0a }, - extractor: null + extractor: extractPNG }, { name: "WEBP Image", @@ -150,7 +150,7 @@ export const FILE_SIGNATURES = { 16: 0x0, 17: 0x0 }, - extractor: null + extractor: extractBMP }, { name: "JPEG Extended Range image", @@ -1233,3 +1233,55 @@ export function extractZIP(bytes, offset) { return stream.carve(); } + + +/** + * PNG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPNG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move past signature to first chunk + stream.moveForwardsBy(8); + + let chunkSize = 0, + chunkType = ""; + + while (chunkType !== "IEND") { + chunkSize = stream.readInt(4, "be"); + chunkType = stream.readString(4); + + // Chunk data size + CRC checksum + stream.moveForwardsBy(chunkSize + 4); + } + + + return stream.carve(); +} + + +/** + * BMP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractBMP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move past header + stream.moveForwardsBy(2); + + // Read full file size + const bmpSize = stream.readInt(4, "le"); + + // Move to end of file (file size minus header and size field) + stream.moveForwardsBy(bmpSize - 6); + + return stream.carve(); +} From 4c285bce57b582a2246b982e83dcc3ecf86fbed0 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Jan 2019 15:12:01 +0000 Subject: [PATCH 009/110] Refactored scanning for file types to be more than twice as fast. --- src/core/lib/FileType.mjs | 73 +++++++++++++++++--- src/core/operations/ExtractFiles.mjs | 26 +------ src/core/operations/ScanForEmbeddedFiles.mjs | 40 +++++------ 3 files changed, 83 insertions(+), 56 deletions(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index b96ea69e..5e1dd657 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -16,13 +16,22 @@ import {FILE_SIGNATURES} from "./FileSignatures"; * These values can be numbers for static checks, arrays of potential valid matches, * or bespoke functions to check the validity of the buffer value at that offset. * @param {Uint8Array} buf + * @param {number} [offset=0] Where in the buffer to start searching from * @returns {boolean} */ -function signatureMatches(sig, buf) { - if (sig instanceof Array) { - return sig.reduce((acc, s) => acc || bytesMatch(s, buf), false); +function signatureMatches(sig, buf, offset=0) { + // Using a length check seems to be more performant than `sig instanceof Array` + if (sig.length) { + // sig is an Array - return true if any of them match + // The following `reduce` method is nice, but performance matters here, so we + // opt for a faster, if less elegant, for loop. + // return sig.reduce((acc, s) => acc || bytesMatch(s, buf, offset), false); + for (let i = 0; i < sig.length; i++) { + if (bytesMatch(sig[i], buf, offset)) return true; + } + return false; } else { - return bytesMatch(sig, buf); + return bytesMatch(sig, buf, offset); } } @@ -34,25 +43,27 @@ function signatureMatches(sig, buf) { * These values can be numbers for static checks, arrays of potential valid matches, * or bespoke functions to check the validity of the buffer value at that offset. * @param {Uint8Array} buf + * @param {number} [offset=0] Where in the buffer to start searching from * @returns {boolean} */ -function bytesMatch(sig, buf) { - for (const offset in sig) { - switch (typeof sig[offset]) { +function bytesMatch(sig, buf, offset=0) { + for (const sigoffset in sig) { + const pos = parseInt(sigoffset, 10) + offset; + switch (typeof sig[sigoffset]) { case "number": // Static check - if (buf[offset] !== sig[offset]) + if (buf[pos] !== sig[sigoffset]) return false; break; case "object": // Array of options - if (sig[offset].indexOf(buf[offset]) < 0) + if (sig[sigoffset].indexOf(buf[pos]) < 0) return false; break; case "function": // More complex calculation - if (!sig[offset](buf[offset])) + if (!sig[sigoffset](buf[pos])) return false; break; default: - throw new Error(`Unrecognised signature type at offset ${offset}`); + throw new Error(`Unrecognised signature type at offset ${sigoffset}`); } } return true; @@ -91,6 +102,46 @@ export function detectFileType(buf) { } +/** + * Given a buffer, searches for magic byte sequences at all possible positions and returns + * the extensions and mime types. + * + * @param {Uint8Array} buf + * @returns {Object[]} foundFiles + * @returns {number} foundFiles.offset - The position in the buffer at which this file was found + * @returns {Object} foundFiles.fileDetails + * @returns {string} foundFiles.fileDetails.name - Name of file type + * @returns {string} foundFiles.fileDetails.ext - File extension + * @returns {string} foundFiles.fileDetails.mime - Mime type + * @returns {string} [foundFiles.fileDetails.desc] - Description + */ +export function scanForFileTypes(buf) { + if (!(buf && buf.length > 1)) { + return []; + } + + const foundFiles = []; + + // TODO allow user to select which categories to check + for (const cat in FILE_SIGNATURES) { + const category = FILE_SIGNATURES[cat]; + + for (let i = 0; i < category.length; i++) { + const filetype = category[i]; + for (let pos = 0; pos < buf.length; pos++) { + if (signatureMatches(filetype.signature, buf, pos)) { + foundFiles.push({ + offset: pos, + fileDetails: filetype + }); + } + } + } + } + return foundFiles; +} + + /** * Detects whether the given buffer is a file of the type specified. * diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index 3a87cd5e..da9d57a9 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -7,7 +7,7 @@ import Operation from "../Operation"; // import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import {detectFileType, extractFile} from "../lib/FileType"; +import {scanForFileTypes, extractFile} from "../lib/FileType"; /** * Extract Files operation @@ -39,7 +39,7 @@ class ExtractFiles extends Operation { const bytes = new Uint8Array(input); // Scan for embedded files - const detectedFiles = scanForEmbeddedFiles(bytes); + const detectedFiles = scanForFileTypes(bytes); // Extract each file that we support const files = []; @@ -64,26 +64,4 @@ class ExtractFiles extends Operation { } -/** - * TODO refactor - * @param data - */ -function scanForEmbeddedFiles(data) { - const detectedFiles = []; - - for (let i = 0; i < data.length; i++) { - const fileDetails = detectFileType(data.slice(i)); - if (fileDetails.length) { - fileDetails.forEach(match => { - detectedFiles.push({ - offset: i, - fileDetails: match, - }); - }); - } - } - - return detectedFiles; -} - export default ExtractFiles; diff --git a/src/core/operations/ScanForEmbeddedFiles.mjs b/src/core/operations/ScanForEmbeddedFiles.mjs index 41ea911b..a0465e83 100644 --- a/src/core/operations/ScanForEmbeddedFiles.mjs +++ b/src/core/operations/ScanForEmbeddedFiles.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; -import {detectFileType} from "../lib/FileType"; +import {scanForFileTypes} from "../lib/FileType"; /** * Scan for Embedded Files operation @@ -41,32 +41,30 @@ class ScanForEmbeddedFiles extends Operation { */ run(input, args) { let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n", - types, numFound = 0, numCommonFound = 0; const ignoreCommon = args[0], - commonExts = ["ico", "ttf", ""], - data = new Uint8Array(input); + commonExts = ["ttf", "utf16le", ""], + data = new Uint8Array(input), + types = scanForFileTypes(data); - for (let i = 0; i < data.length; i++) { - types = detectFileType(data.slice(i)); - if (types.length) { - types.forEach(type => { - if (ignoreCommon && commonExts.indexOf(type.extension) > -1) { - numCommonFound++; - return; - } - numFound++; - output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" + - " File extension: " + type.extension + "\n" + - " MIME type: " + type.mime + "\n"; + if (types.length) { + types.forEach(type => { + if (ignoreCommon && commonExts.indexOf(type.fileDetails.extension) > -1) { + numCommonFound++; + return; + } - if (type.description && type.description.length) { - output += " Description: " + type.description + "\n"; - } - }); - } + numFound++; + output += "\nOffset " + type.offset + " (0x" + Utils.hex(type.offset) + "):\n" + + " File extension: " + type.fileDetails.extension + "\n" + + " MIME type: " + type.fileDetails.mime + "\n"; + + if (type.fileDetails.description && type.fileDetails.description.length) { + output += " Description: " + type.fileDetails.description + "\n"; + } + }); } if (numFound === 0) { From a56f92cdee6ae9f44e20610a59dfdff02c94a185 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 2 Jan 2019 17:50:47 +0000 Subject: [PATCH 010/110] Significantly improved performance when scanning for embedded files by implementing a fastcheck algorithm. --- package-lock.json | 148 +++++++++++++++++++------------------- src/core/lib/FileType.mjs | 67 ++++++++++++++--- 2 files changed, 131 insertions(+), 84 deletions(-) diff --git a/package-lock.json b/package-lock.json index ac28bda1..fa857e05 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1497,7 +1497,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1615,7 +1615,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1700,7 +1700,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1800,7 +1800,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -2232,7 +2232,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -2269,7 +2269,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -2319,7 +2319,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2389,7 +2389,7 @@ }, "cacache": { "version": "10.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", + "resolved": "http://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==", "dev": true, "requires": { @@ -2466,7 +2466,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2515,7 +2515,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -3048,7 +3048,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -3061,7 +3061,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -3179,7 +3179,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3547,7 +3547,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3611,7 +3611,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3816,7 +3816,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -4240,7 +4240,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -4252,7 +4252,7 @@ }, "events": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4675,7 +4675,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -4911,7 +4911,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -5579,7 +5579,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5721,7 +5721,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -5798,7 +5798,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5846,7 +5846,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -5866,7 +5866,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5911,7 +5911,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -6010,7 +6010,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -6074,7 +6074,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6102,7 +6102,7 @@ }, "handle-thing": { "version": "1.2.5", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", + "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", "dev": true }, @@ -6335,7 +6335,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -6391,7 +6391,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -6410,7 +6410,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -6460,7 +6460,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -6938,7 +6938,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -7505,7 +7505,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -7616,7 +7616,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -7730,7 +7730,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -7743,7 +7743,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8102,7 +8102,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -8161,7 +8161,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -8346,7 +8346,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -8556,7 +8556,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -8655,7 +8655,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -8827,7 +8827,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -9121,13 +9121,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -9136,7 +9136,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -9351,7 +9351,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -9437,7 +9437,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -9478,7 +9478,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9661,7 +9661,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -10073,7 +10073,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -10098,13 +10098,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -10119,7 +10119,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -10342,7 +10342,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -10531,7 +10531,7 @@ "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", "dev": true } @@ -10582,7 +10582,7 @@ }, "htmlparser2": { "version": "3.3.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", "integrity": "sha1-zHDQWln2VC5D8OaFyYLhTJJKnv4=", "dev": true, "requires": { @@ -10594,7 +10594,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -10704,7 +10704,7 @@ }, "require-uncached": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", "dev": true, "requires": { @@ -10871,7 +10871,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -11191,7 +11191,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -11235,7 +11235,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -11913,7 +11913,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -11930,7 +11930,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -12009,7 +12009,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -12064,7 +12064,7 @@ }, "through": { "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -12758,7 +12758,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -12784,7 +12784,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -12800,7 +12800,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -13296,7 +13296,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -13450,14 +13450,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true @@ -13490,7 +13490,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 5e1dd657..93c7634a 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -128,17 +128,58 @@ export function scanForFileTypes(buf) { for (let i = 0; i < category.length; i++) { const filetype = category[i]; - for (let pos = 0; pos < buf.length; pos++) { - if (signatureMatches(filetype.signature, buf, pos)) { - foundFiles.push({ - offset: pos, - fileDetails: filetype - }); + const sigs = filetype.signature.length ? filetype.signature : [filetype.signature]; + + sigs.forEach(sig => { + let pos = 0; + while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) { + if (signatureMatches(sig, buf, pos)) { + foundFiles.push({ + offset: pos, + fileDetails: filetype + }); + } + pos++; } - } + }); } } - return foundFiles; + + // Return found files in order of increasing offset + return foundFiles.sort((a, b) => { + return a.offset - b.offset; + }); +} + + +/** + * Fastcheck function to quickly scan the buffer for the first byte in a signature. + * + * @param {Uint8Array} buf - The buffer to search + * @param {Object} sig - A single signature object (Not an array of signatures) + * @param {number} offset - Where to start search from + * @returs {number} The position of the match or -1 if one cannot be found. + */ +function locatePotentialSig(buf, sig, offset) { + // Find values for first key and value in sig + const k = parseInt(Object.keys(sig)[0], 10); + const v = Object.values(sig)[0]; + switch (typeof v) { + case "number": + return buf.indexOf(v, offset + k) - k; + case "object": + for (let i = offset + k; i < buf.length; i++) { + if (v.indexOf(buf[i]) >= 0) return i - k; + } + return -1; + case "function": + for (let i = offset + k; i < buf.length; i++) { + if (v(buf[i])) return i - k; + } + return -1; + default: + throw new Error("Unrecognised signature type"); + } } @@ -155,9 +196,15 @@ export function isType(type, buf) { if (!(types && types.length)) return false; if (typeof type === "string") { - return types[0].mime.startsWith(type) ? types[0].mime : false; + return types.reduce((acc, t) => { + const mime = t.mime.startsWith(type) ? t.mime : false; + return acc || mime; + }, false); } else if (type instanceof RegExp) { - return type.test(types[0].mime) ? types[0].mime : false; + return types.reduce((acc, t) => { + const mime = type.test(t.mime) ? t.mime : false; + return acc || mime; + }, false); } else { throw new Error("Invalid type input."); } From cd0c86e0d66ea8df0771282b0333b90607037188 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Jan 2019 13:03:41 +0000 Subject: [PATCH 011/110] File scan now uses bytesMatch() instead of signatureMatches(), reducing call stack size --- src/core/lib/FileType.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 93c7634a..202b54d9 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -133,7 +133,7 @@ export function scanForFileTypes(buf) { sigs.forEach(sig => { let pos = 0; while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) { - if (signatureMatches(sig, buf, pos)) { + if (bytesMatch(sig, buf, pos)) { foundFiles.push({ offset: pos, fileDetails: filetype From 088864fd9c45105e8e311e5f64adda751614652a Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 3 Jan 2019 16:36:56 +0000 Subject: [PATCH 012/110] Add Enigma operation --- .gitignore | 1 + src/core/config/Categories.json | 3 +- src/core/lib/Enigma.mjs | 349 ++++++++++++++++++++ src/core/operations/Enigma.mjs | 200 ++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/Enigma.mjs | 518 ++++++++++++++++++++++++++++++ 6 files changed, 1071 insertions(+), 1 deletion(-) create mode 100644 src/core/lib/Enigma.mjs create mode 100644 src/core/operations/Enigma.mjs create mode 100644 tests/operations/tests/Enigma.mjs diff --git a/.gitignore b/.gitignore index 3ca816f6..edbcf679 100755 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ docs/* !docs/*.conf.json !docs/*.ico .vscode +.*.swp src/core/config/modules/* src/core/config/OperationConfig.json src/core/operations/index.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 686c9842..3e0d108e 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -102,7 +102,8 @@ "JWT Decode", "Citrix CTX1 Encode", "Citrix CTX1 Decode", - "Pseudo-Random Number Generator" + "Pseudo-Random Number Generator", + "Enigma" ] }, { diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs new file mode 100644 index 00000000..845ce413 --- /dev/null +++ b/src/core/lib/Enigma.mjs @@ -0,0 +1,349 @@ +/** + * Emulation of the Enigma machine. + * + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import OperationError from "../errors/OperationError"; +import Utils from "../Utils"; + +/** + * Provided default Enigma rotor set. + * These are specified as a list of mappings from the letters A through Z in order, optionally + * followed by < and a list of letters at which the rotor steps. + */ +export const ROTORS = [ + {name: "I", value: "EKMFLGDQVZNTOWYHXUSPAIBRCJ= 65 && i <= 90) { + return i - 65; + } + if (permissive) { + // Allow case insensitivity + if (i >= 97 && i <= 122) { + return i - 97; + } + return -1; + } + throw new OperationError("a2i called on non-uppercase ASCII character"); +} + +/** + * Map a number in 0..25 to a letter. + * + * @param {number} i + * @returns {char} + */ +export function i2a(i) { + if (i >= 0 && i < 26) { + return Utils.chr(i+65); + } + throw new OperationError("i2a called on value outside 0..25"); +} + +/** + * A rotor in the Enigma machine. + */ +export class Rotor { + /** + * Rotor constructor. + * + * @param {string} wiring - A 26 character string of the wiring order. + * @param {string} steps - A 0..26 character string of stepping points. + * @param {char} ringSetting - The ring setting. + * @param {char} initialPosition - The initial position of the rotor. + */ + constructor(wiring, steps, ringSetting, initialPosition) { + if (!/^[A-Z]{26}$/.test(wiring)) { + throw new OperationError("Rotor wiring must be 26 unique uppercase letters"); + } + if (!/^[A-Z]{0,26}$/.test(steps)) { + throw new OperationError("Rotor steps must be 0-26 unique uppercase letters"); + } + if (!/^[A-Z]$/.test(ringSetting)) { + throw new OperationError("Rotor ring setting must be exactly one uppercase letter"); + } + if (!/^[A-Z]$/.test(initialPosition)) { + throw new OperationError("Rotor initial position must be exactly one uppercase letter"); + } + this.map = {}; + this.revMap = {}; + for (let i=0; i { + if (!/^[A-Z]{2}$/.test(pair)) { + throw new OperationError(name + " must be a whitespace-separated list of uppercase letter pairs"); + } + const a = a2i(pair[0]), b = a2i(pair[1]); + if (a === b) { + throw new OperationError(`${name}: cannot connect ${pair[0]} to itself`); + } + if (this.map.hasOwnProperty(a)) { + throw new OperationError(`${name} connects ${pair[0]} more than once`); + } + if (this.map.hasOwnProperty(b)) { + throw new OperationError(`${name} connects ${pair[1]} more than once`); + } + this.map[a] = b; + this.map[b] = a; + }); + } + + /** + * Transform a character through this object. + * Returns other characters unchanged. + * + * @param {number} c - The character. + * @returns {number} + */ + transform(c) { + if (!this.map.hasOwnProperty(c)) { + return c; + } + return this.map[c]; + } + + /** + * Alias for transform, to allow interchangeable use with rotors. + * + * @param {number} c - The character. + * @returns {number} + */ + revTransform(c) { + return this.transform(c); + } +} + +/** + * Reflector. PairMapBase but requires that all characters are accounted for. + */ +export class Reflector extends PairMapBase { + /** + * Reflector constructor. See PairMapBase. + * Additional restriction: every character must be accounted for. + */ + constructor(pairs) { + super(pairs, "Reflector"); + const s = Object.keys(this.map).length; + if (s !== 26) { + throw new OperationError("Reflector must have exactly 13 pairs covering every letter"); + } + } +} + +/** + * Plugboard. Unmodified PairMapBase. + */ +export class Plugboard extends PairMapBase { + /** + * Plugboard constructor. See PairMapbase. + */ + constructor(pairs) { + super(pairs, "Plugboard"); + } +} + +/** + * Base class for the Enigma machine itself. Holds rotors, a reflector, and a plugboard. + */ +export class EnigmaBase { + /** + * EnigmaBase constructor. + * + * @param {Object[]} rotors - List of Rotors. + * @param {Object} reflector - A Reflector. + * @param {Plugboard} plugboard - A Plugboard. + */ + constructor(rotors, reflector, plugboard) { + this.rotors = rotors; + this.rotorsRev = [].concat(rotors).reverse(); + this.reflector = reflector; + this.plugboard = plugboard; + } + + /** + * Step the rotors forward by one. + * + * This happens before the output character is generated. + * + * Note that rotor 4, if it's there, never steps. + * + * Why is all the logic in EnigmaBase and not a nice neat method on + * Rotor that knows when it should advance the next item? + * Because the double stepping anomaly is a thing. tl;dr if the left rotor + * should step the next time the middle rotor steps, the middle rotor will + * immediately step. + */ + step() { + const r0 = this.rotors[0]; + const r1 = this.rotors[1]; + r0.step(); + // The second test here is the double-stepping anomaly + if (r0.steps.has(r0.pos) || r1.steps.has(Utils.mod(r1.pos + 1, 26))) { + r1.step(); + if (r1.steps.has(r1.pos)) { + const r2 = this.rotors[2]; + r2.step(); + } + } + } + + /** + * Encrypt (or decrypt) some data. + * Takes an arbitrary string and runs the Engima machine on that data from + * *its current state*, and outputs the result. Non-alphabetic characters + * are returned unchanged. + * + * @param {string} input - Data to encrypt. + * @returns {string} + */ + crypt(input) { + let result = ""; + for (const c of input) { + let letter = a2i(c, true); + if (letter === -1) { + result += c; + continue; + } + // First, step the rotors forward. + this.step(); + // Now, run through the plugboard. + letter = this.plugboard.transform(letter); + // Then through each wheel in sequence, through the reflector, and + // backwards through the wheels again. + for (const rotor of this.rotors) { + letter = rotor.transform(letter); + } + letter = this.reflector.transform(letter); + for (const rotor of this.rotorsRev) { + letter = rotor.revTransform(letter); + } + // Finally, back through the plugboard. + letter = this.plugboard.revTransform(letter); + result += i2a(letter); + } + return result; + } +} + +/** + * The Enigma machine itself. Holds 3-4 rotors, a reflector, and a plugboard. + */ +export class EnigmaMachine extends EnigmaBase { + /** + * EnigmaMachine constructor. + * + * @param {Object[]} rotors - List of Rotors. + * @param {Object} reflector - A Reflector. + * @param {Plugboard} plugboard - A Plugboard. + */ + constructor(rotors, reflector, plugboard) { + super(rotors, reflector, plugboard); + if (rotors.length !== 3 && rotors.length !== 4) { + throw new OperationError("Enigma must have 3 or 4 rotors"); + } + } +} diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs new file mode 100644 index 00000000..d43d780d --- /dev/null +++ b/src/core/operations/Enigma.mjs @@ -0,0 +1,200 @@ +/** + * Emulation of the Enigma machine. + * + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import * as Enigma from "../lib/Enigma"; + +/** + * Enigma operation + */ +class EnigmaOp extends Operation { + /** + * Enigma constructor + */ + constructor() { + super(); + + this.name = "Enigma"; + this.module = "Default"; + this.description = "Encipher/decipher with the WW2 Enigma machine.

The standard set of German military rotors and reflectors are provided. To configure the plugboard, enter a string of connected pairs of letters, e.g. AB CD EF connects A to B, C to D, and E to F. This is also used to create your own reflectors. To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points.
This is deliberately fairly permissive with rotor placements etc compared to a real Enigma (on which, for example, a four-rotor Enigma uses the thin reflectors and the beta or gamma rotor in the 4th slot)."; + this.infoURL = "https://wikipedia.org/wiki/Enigma_machine"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "1st (right-hand) rotor", + type: "editableOption", + value: Enigma.ROTORS, + // Default config is the rotors I-III *left to right* + defaultIndex: 2 + }, + { + name: "1st rotor ring setting", + type: "option", + value: Enigma.LETTERS + }, + { + name: "1st rotor initial value", + type: "option", + value: Enigma.LETTERS + }, + { + name: "2nd rotor", + type: "editableOption", + value: Enigma.ROTORS, + defaultIndex: 1 + }, + { + name: "2nd rotor ring setting", + type: "option", + value: Enigma.LETTERS + }, + { + name: "2nd rotor initial value", + type: "option", + value: Enigma.LETTERS + }, + { + name: "3rd rotor", + type: "editableOption", + value: Enigma.ROTORS, + defaultIndex: 0 + }, + { + name: "3rd rotor ring setting", + type: "option", + value: Enigma.LETTERS + }, + { + name: "3rd rotor initial value", + type: "option", + value: Enigma.LETTERS + }, + { + name: "4th rotor", + type: "editableOption", + value: Enigma.ROTORS_OPTIONAL, + defaultIndex: 10 + }, + { + name: "4th rotor initial value", + type: "option", + value: Enigma.LETTERS + }, + { + name: "Reflector", + type: "editableOption", + value: Enigma.REFLECTORS + }, + { + name: "Plugboard", + type: "string", + value: "" + }, + { + name: "Strict output", + hint: "Remove non-alphabet letters and group output", + type: "boolean", + value: true + }, + ]; + } + + /** + * Helper - for ease of use rotors are specified as a single string; this + * method breaks the spec string into wiring and steps parts. + * + * @param {string} rotor - Rotor specification string. + * @param {number} i - For error messages, the number of this rotor. + * @returns {string[]} + */ + parseRotorStr(rotor, i) { + if (rotor === "") { + throw new OperationError(`Rotor ${i} must be provided.`); + } + if (!rotor.includes("<")) { + return [rotor, ""]; + } + return rotor.split("<", 2); + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const [ + rotor1str, rotor1ring, rotor1pos, + rotor2str, rotor2ring, rotor2pos, + rotor3str, rotor3ring, rotor3pos, + rotor4str, rotor4pos, + reflectorstr, plugboardstr, + removeOther + ] = args; + const rotors = []; + const [rotor1wiring, rotor1steps] = this.parseRotorStr(rotor1str, 1); + rotors.push(new Enigma.Rotor(rotor1wiring, rotor1steps, rotor1ring, rotor1pos)); + const [rotor2wiring, rotor2steps] = this.parseRotorStr(rotor2str, 2); + rotors.push(new Enigma.Rotor(rotor2wiring, rotor2steps, rotor2ring, rotor2pos)); + const [rotor3wiring, rotor3steps] = this.parseRotorStr(rotor3str, 3); + rotors.push(new Enigma.Rotor(rotor3wiring, rotor3steps, rotor3ring, rotor3pos)); + if (rotor4str !== "") { + // Fourth rotor doesn't have a ring setting - A is equivalent to no setting + const [rotor4wiring, rotor4steps] = this.parseRotorStr(rotor4str, 4); + rotors.push(new Enigma.Rotor(rotor4wiring, rotor4steps, "A", rotor4pos)); + } + const reflector = new Enigma.Reflector(reflectorstr); + const plugboard = new Enigma.Plugboard(plugboardstr); + if (removeOther) { + input = input.replace(/[^A-Za-z]/g, ""); + } + const enigma = new Enigma.EnigmaMachine(rotors, reflector, plugboard); + let result = enigma.crypt(input); + if (removeOther) { + // Five character cipher groups is traditional + result = result.replace(/([A-Z]{5})(?!$)/g, "$1 "); + } + return result; + } + + /** + * Highlight Enigma + * This is only possible if we're passing through non-alphabet characters. + * + * @param {Object[]} pos + * @param {number} pos[].start + * @param {number} pos[].end + * @param {Object[]} args + * @returns {Object[]} pos + */ + highlight(pos, args) { + if (args[13] === false) { + return pos; + } + } + + /** + * Highlight Enigma in reverse + * + * @param {Object[]} pos + * @param {number} pos[].start + * @param {number} pos[].end + * @param {Object[]} args + * @returns {Object[]} pos + */ + highlightReverse(pos, args) { + if (args[13] === false) { + return pos; + } + } + +} + +export default EnigmaOp; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index da9d41be..119f3ac4 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -82,6 +82,7 @@ import "./tests/TranslateDateTimeFormat"; import "./tests/Magic"; import "./tests/ParseTLV"; import "./tests/Media"; +import "./tests/Enigma"; // Cannot test operations that use the File type yet //import "./tests/SplitColourChannels"; diff --git a/tests/operations/tests/Enigma.mjs b/tests/operations/tests/Enigma.mjs new file mode 100644 index 00000000..a1acaf33 --- /dev/null +++ b/tests/operations/tests/Enigma.mjs @@ -0,0 +1,518 @@ +/** + * Enigma machine tests. + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + // Simplest test: A single keypress in the default position on a basic + // Enigma. + name: "Enigma: basic wiring", + input: "G", + expectedOutput: "P", + recipeConfig: [ + { + "op": "Enigma", + "args": [ + // Note: start on Z because it steps when the key is pressed + "BDFHJLCPRTXVZNYEIWGAKMUSQO Date: Thu, 3 Jan 2019 16:40:29 +0000 Subject: [PATCH 013/110] Add Typex operation WIP --- src/core/config/Categories.json | 3 +- src/core/operations/Typex.mjs | 396 ++++++++++++++++++++++++++++++++ 2 files changed, 398 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/Typex.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3e0d108e..f3b73921 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -103,7 +103,8 @@ "Citrix CTX1 Encode", "Citrix CTX1 Decode", "Pseudo-Random Number Generator", - "Enigma" + "Enigma", + "Typex" ] }, { diff --git a/src/core/operations/Typex.mjs b/src/core/operations/Typex.mjs new file mode 100644 index 00000000..89114863 --- /dev/null +++ b/src/core/operations/Typex.mjs @@ -0,0 +1,396 @@ +/** + * Emulation of the Typex machine. + * + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Utils from "../Utils"; +import * as Enigma from "../lib/Enigma"; + +const ROTORS = [ + {name: "1", value: "QWECYJIBFKMLTVZPOHUDGNRSXA Date: Thu, 3 Jan 2019 17:51:20 +0000 Subject: [PATCH 014/110] Enigma: fix 4th rotor ringstellung --- src/core/operations/Enigma.mjs | 33 +++++++------- tests/operations/tests/Enigma.mjs | 73 +++++++++++++++++++------------ 2 files changed, 61 insertions(+), 45 deletions(-) diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs index d43d780d..38456ebf 100644 --- a/src/core/operations/Enigma.mjs +++ b/src/core/operations/Enigma.mjs @@ -82,6 +82,11 @@ class EnigmaOp extends Operation { value: Enigma.ROTORS_OPTIONAL, defaultIndex: 10 }, + { + name: "4th rotor ring setting", + type: "option", + value: Enigma.LETTERS + }, { name: "4th rotor initial value", type: "option", @@ -130,25 +135,17 @@ class EnigmaOp extends Operation { * @returns {string} */ run(input, args) { - const [ - rotor1str, rotor1ring, rotor1pos, - rotor2str, rotor2ring, rotor2pos, - rotor3str, rotor3ring, rotor3pos, - rotor4str, rotor4pos, - reflectorstr, plugboardstr, - removeOther - ] = args; + const reflectorstr = args[12]; + const plugboardstr = args[13]; + const removeOther = args[14]; const rotors = []; - const [rotor1wiring, rotor1steps] = this.parseRotorStr(rotor1str, 1); - rotors.push(new Enigma.Rotor(rotor1wiring, rotor1steps, rotor1ring, rotor1pos)); - const [rotor2wiring, rotor2steps] = this.parseRotorStr(rotor2str, 2); - rotors.push(new Enigma.Rotor(rotor2wiring, rotor2steps, rotor2ring, rotor2pos)); - const [rotor3wiring, rotor3steps] = this.parseRotorStr(rotor3str, 3); - rotors.push(new Enigma.Rotor(rotor3wiring, rotor3steps, rotor3ring, rotor3pos)); - if (rotor4str !== "") { - // Fourth rotor doesn't have a ring setting - A is equivalent to no setting - const [rotor4wiring, rotor4steps] = this.parseRotorStr(rotor4str, 4); - rotors.push(new Enigma.Rotor(rotor4wiring, rotor4steps, "A", rotor4pos)); + for (let i=0; i<4; i++) { + if (i === 3 && args[i*3] === "") { + // No fourth rotor + break; + } + const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*3], 1); + rotors.push(new Enigma.Rotor(rotorwiring, rotorsteps, args[i*3 + 1], args[i*3 + 2])); } const reflector = new Enigma.Reflector(reflectorstr); const plugboard = new Enigma.Plugboard(plugboardstr); diff --git a/tests/operations/tests/Enigma.mjs b/tests/operations/tests/Enigma.mjs index a1acaf33..f8776b42 100644 --- a/tests/operations/tests/Enigma.mjs +++ b/tests/operations/tests/Enigma.mjs @@ -21,7 +21,7 @@ TestRegister.addTests([ "BDFHJLCPRTXVZNYEIWGAKMUSQO Date: Thu, 3 Jan 2019 18:40:22 +0000 Subject: [PATCH 015/110] Added FLV extractor. --- src/core/lib/FileSignatures.mjs | 48 ++++++++++++++++++++++++++++++++- src/core/lib/Stream.mjs | 39 ++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index d4e19da9..80ea77d1 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -417,7 +417,7 @@ export const FILE_SIGNATURES = { 2: 0x56, 3: 0x1 }, - extractor: null + extractor: extractFLV }, ], "Audio": [ @@ -1285,3 +1285,49 @@ export function extractBMP(bytes, offset) { return stream.carve(); } + + +/** + * FLV extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractFLV(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move past signature, version and flags + stream.moveForwardsBy(5); + + // Read header size + const headerSize = stream.readInt(4, "be"); + + // Skip through the rest of the header + stream.moveForwardsBy(headerSize - 9); + + let tagSize = -11; // Fake size of previous tag header + while (stream.position < stream.length) { + const prevTagSize = stream.readInt(4, "be"); + const tagType = stream.readInt(1, "be"); + + if ([8, 9, 18].indexOf(tagType) < 0) { + // This tag is not valid + stream.moveBackwardsBy(1); + break; + } + + if (prevTagSize !== tagSize + 11) { + // Previous tag was not valid + stream.moveBackwardsBy(tagSize + 11); + break; + } + + tagSize = stream.readInt(3, "be"); + + // Move past the rest of the tag header and payload + stream.moveForwardsBy(7 + tagSize); + } + + return stream.carve(); +} diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 19903117..0a29b5a6 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -22,6 +22,7 @@ export default class Stream { constructor(input) { this.bytes = input; this.position = 0; + this.length = this.bytes.length; } /** @@ -31,6 +32,8 @@ export default class Stream { * @returns {Uint8Array} */ getBytes(numBytes) { + if (this.position > this.length) return undefined; + const newPosition = this.position + numBytes; const bytes = this.bytes.slice(this.position, newPosition); this.position = newPosition; @@ -45,6 +48,8 @@ export default class Stream { * @returns {string} */ readString(numBytes) { + if (this.position > this.length) return undefined; + let result = ""; for (let i = this.position; i < this.position + numBytes; i++) { const currentByte = this.bytes[i]; @@ -63,6 +68,8 @@ export default class Stream { * @returns {number} */ readInt(numBytes, endianness="be") { + if (this.position > this.length) return undefined; + let val = 0; if (endianness === "be") { for (let i = this.position; i < this.position + numBytes; i++) { @@ -85,8 +92,10 @@ export default class Stream { * @param {number|List} val */ continueUntil(val) { + if (this.position > this.length) return; + if (typeof val === "number") { - while (++this.position < this.bytes.length && this.bytes[this.position] !== val) { + while (++this.position < this.length && this.bytes[this.position] !== val) { continue; } return; @@ -94,13 +103,13 @@ export default class Stream { // val is an array let found = false; - while (!found && this.position < this.bytes.length) { - while (++this.position < this.bytes.length && this.bytes[this.position] !== val[0]) { + while (!found && this.position < this.length) { + while (++this.position < this.length && this.bytes[this.position] !== val[0]) { continue; } found = true; for (let i = 1; i < val.length; i++) { - if (this.position + i > this.bytes.length || this.bytes[this.position + i] !== val[i]) + if (this.position + i > this.length || this.bytes[this.position + i] !== val[i]) found = false; } } @@ -122,7 +131,23 @@ export default class Stream { * @param {number} numBytes */ moveForwardsBy(numBytes) { - this.position += numBytes; + const pos = this.position + numBytes; + if (pos < 0 || pos > this.length) + throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); + this.position = pos; + } + + + /** + * Move backwards through the stream by the specified number of bytes. + * + * @param {number} numBytes + */ + moveBackwardsBy(numBytes) { + const pos = this.position - numBytes; + if (pos < 0 || pos > this.length) + throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); + this.position = pos; } /** @@ -131,7 +156,7 @@ export default class Stream { * @param {number} pos */ moveTo(pos) { - if (pos < 0 || pos > this.bytes.length - 1) + if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; } @@ -142,7 +167,7 @@ export default class Stream { * @returns {boolean} */ hasMore() { - return this.position < this.bytes.length; + return this.position < this.length; } /** From 53cc413eaee00d1de585abfafbdc5cec5ae2413c Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 3 Jan 2019 18:48:50 +0000 Subject: [PATCH 016/110] Typex: move machine implementation to lib/ --- src/core/lib/Typex.mjs | 183 ++++++++++++++++++++++++++++++++ src/core/operations/Typex.mjs | 193 ++-------------------------------- 2 files changed, 193 insertions(+), 183 deletions(-) create mode 100644 src/core/lib/Typex.mjs diff --git a/src/core/lib/Typex.mjs b/src/core/lib/Typex.mjs new file mode 100644 index 00000000..a99f3b6e --- /dev/null +++ b/src/core/lib/Typex.mjs @@ -0,0 +1,183 @@ +/** + * Emulation of the Typex machine. + * + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import OperationError from "../errors/OperationError"; +import * as Enigma from "../lib/Enigma"; +import Utils from "../Utils"; + +export const ROTORS = [ + {name: "1", value: "QWECYJIBFKMLTVZPOHUDGNRSXA Date: Thu, 3 Jan 2019 18:51:39 +0000 Subject: [PATCH 017/110] Enigma: make sure op class is called Enigma --- src/core/operations/Enigma.mjs | 40 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs index 38456ebf..c45f59ae 100644 --- a/src/core/operations/Enigma.mjs +++ b/src/core/operations/Enigma.mjs @@ -8,12 +8,12 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; -import * as Enigma from "../lib/Enigma"; +import {ROTORS, LETTERS, ROTORS_OPTIONAL, REFLECTORS, Rotor, Reflector, Plugboard, EnigmaMachine} from "../lib/Enigma"; /** * Enigma operation */ -class EnigmaOp extends Operation { +class Enigma extends Operation { /** * Enigma constructor */ @@ -30,72 +30,72 @@ class EnigmaOp extends Operation { { name: "1st (right-hand) rotor", type: "editableOption", - value: Enigma.ROTORS, + value: ROTORS, // Default config is the rotors I-III *left to right* defaultIndex: 2 }, { name: "1st rotor ring setting", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "1st rotor initial value", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "2nd rotor", type: "editableOption", - value: Enigma.ROTORS, + value: ROTORS, defaultIndex: 1 }, { name: "2nd rotor ring setting", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "2nd rotor initial value", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "3rd rotor", type: "editableOption", - value: Enigma.ROTORS, + value: ROTORS, defaultIndex: 0 }, { name: "3rd rotor ring setting", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "3rd rotor initial value", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "4th rotor", type: "editableOption", - value: Enigma.ROTORS_OPTIONAL, + value: ROTORS_OPTIONAL, defaultIndex: 10 }, { name: "4th rotor ring setting", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "4th rotor initial value", type: "option", - value: Enigma.LETTERS + value: LETTERS }, { name: "Reflector", type: "editableOption", - value: Enigma.REFLECTORS + value: REFLECTORS }, { name: "Plugboard", @@ -145,14 +145,14 @@ class EnigmaOp extends Operation { break; } const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*3], 1); - rotors.push(new Enigma.Rotor(rotorwiring, rotorsteps, args[i*3 + 1], args[i*3 + 2])); + rotors.push(new Rotor(rotorwiring, rotorsteps, args[i*3 + 1], args[i*3 + 2])); } - const reflector = new Enigma.Reflector(reflectorstr); - const plugboard = new Enigma.Plugboard(plugboardstr); + const reflector = new Reflector(reflectorstr); + const plugboard = new Plugboard(plugboardstr); if (removeOther) { input = input.replace(/[^A-Za-z]/g, ""); } - const enigma = new Enigma.EnigmaMachine(rotors, reflector, plugboard); + const enigma = new EnigmaMachine(rotors, reflector, plugboard); let result = enigma.crypt(input); if (removeOther) { // Five character cipher groups is traditional @@ -194,4 +194,4 @@ class EnigmaOp extends Operation { } -export default EnigmaOp; +export default Enigma; From 7d8d80ca2c4b89f5b62e74e3b503ed285eabe059 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Jan 2019 19:01:12 +0000 Subject: [PATCH 018/110] Added extractor for MS Office 2007+ files --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 80ea77d1..e879fcf7 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -609,7 +609,7 @@ export const FILE_SIGNATURES = { 47: 0x6d, 48: 0x6c }, - extractor: null + extractor: extractZIP }, { name: "EPUB e-book", From 0d2cb02f975c9a3186cf16576ee243281f780819 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Jan 2019 11:49:12 +0000 Subject: [PATCH 019/110] Fixed FLV previous tag size error --- src/core/lib/FileSignatures.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index e879fcf7..e55bd566 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1317,9 +1317,10 @@ export function extractFLV(bytes, offset) { break; } - if (prevTagSize !== tagSize + 11) { - // Previous tag was not valid - stream.moveBackwardsBy(tagSize + 11); + if (prevTagSize !== (tagSize + 11)) { + // Previous tag was not valid, reverse back over this header + // and the previous tag body and header + stream.moveBackwardsBy(tagSize + 11 + 5); break; } From 1b1a3c261dc62c0e62e6c71a6b38b5fe4cacbe36 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Fri, 4 Jan 2019 13:21:15 +0000 Subject: [PATCH 020/110] Typex: random rotors --- src/core/lib/Typex.mjs | 25 ++++++++++++++++--------- src/core/operations/Typex.mjs | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/core/lib/Typex.mjs b/src/core/lib/Typex.mjs index a99f3b6e..df6e646b 100644 --- a/src/core/lib/Typex.mjs +++ b/src/core/lib/Typex.mjs @@ -9,19 +9,26 @@ import OperationError from "../errors/OperationError"; import * as Enigma from "../lib/Enigma"; import Utils from "../Utils"; +/** + * A set of example Typex rotors. No Typex rotor wirings are publicly available, so these are + * randomised. + */ export const ROTORS = [ - {name: "1", value: "QWECYJIBFKMLTVZPOHUDGNRSXA Date: Fri, 4 Jan 2019 14:57:31 +0000 Subject: [PATCH 021/110] Added RTF extractor --- src/core/lib/FileSignatures.mjs | 44 +++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index e55bd566..4409c96c 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -572,7 +572,7 @@ export const FILE_SIGNATURES = { 3: 0x74, 4: 0x66 }, - extractor: null + extractor: extractRTF }, { name: "Microsoft Office documents/OLE2", @@ -1307,7 +1307,7 @@ export function extractFLV(bytes, offset) { stream.moveForwardsBy(headerSize - 9); let tagSize = -11; // Fake size of previous tag header - while (stream.position < stream.length) { + while (stream.hasMore()) { const prevTagSize = stream.readInt(4, "be"); const tagType = stream.readInt(1, "be"); @@ -1332,3 +1332,43 @@ export function extractFLV(bytes, offset) { return stream.carve(); } + + +/** + * RTF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractRTF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + let openTags = 0; + + if (stream.readInt(1, "be") !== 0x7b) { // { + throw new Error("Not a valid RTF file"); + } else { + openTags++; + } + + while (openTags > 0 && stream.hasMore()) { + switch (stream.readInt(1, "be")) { + case 0x7b: // { + openTags++; + break; + case 0x7d: // } + openTags--; + break; + case 0x5c: // \ + // Consume any more escapes and then skip over the next character + stream.consumeIf(0x5c); + stream.position++; + break; + default: + break; + } + } + + return stream.carve(); +} From 2a6db47aeb1add115bbf859c12a1be468812bb03 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Jan 2019 18:12:49 +0000 Subject: [PATCH 022/110] Began implementing GZIP/DEFLATE extraction. Unfinished. --- src/core/lib/FileSignatures.mjs | 109 ++++++++++++++++++++++++++++++-- src/core/lib/Stream.mjs | 65 ++++++++++++++++++- 2 files changed, 167 insertions(+), 7 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 4409c96c..b0eb5884 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -650,7 +650,7 @@ export const FILE_SIGNATURES = { 56: 0x69, 57: 0x70 }, - extractor: null + extractor: extractZIP }, ], "Applications": [ @@ -790,7 +790,7 @@ export const FILE_SIGNATURES = { 1: 0x8b, 2: 0x8 }, - extractor: null + extractor: extractGZIP }, { name: "Bzip2", @@ -1309,7 +1309,7 @@ export function extractFLV(bytes, offset) { let tagSize = -11; // Fake size of previous tag header while (stream.hasMore()) { const prevTagSize = stream.readInt(4, "be"); - const tagType = stream.readInt(1, "be"); + const tagType = stream.readInt(1); if ([8, 9, 18].indexOf(tagType) < 0) { // This tag is not valid @@ -1346,14 +1346,14 @@ export function extractRTF(bytes, offset) { let openTags = 0; - if (stream.readInt(1, "be") !== 0x7b) { // { + if (stream.readInt(1) !== 0x7b) { // { throw new Error("Not a valid RTF file"); } else { openTags++; } while (openTags > 0 && stream.hasMore()) { - switch (stream.readInt(1, "be")) { + switch (stream.readInt(1)) { case 0x7b: // { openTags++; break; @@ -1372,3 +1372,102 @@ export function extractRTF(bytes, offset) { return stream.carve(); } + + +/** + * GZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractGZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + /* HEADER */ + + // Skip over signature and compression method + stream.moveForwardsBy(3); + + // Read flags + const flags = stream.readInt(1); + + // Skip over last modification time + stream.moveForwardsBy(4); + + // Read compression flags + const compressionFlags = stream.readInt(1); + + // Skip over OS + stream.moveForwardsBy(1); + + + /* OPTIONAL HEADERS */ + + // Extra fields + if (flags & 0x4) { + console.log("Extra fields"); + const extraFieldsSize = stream.readInt(2, "le"); + stream.moveForwardsby(extraFieldsSize); + } + + // Original filename + if (flags & 0x8) { + console.log("Filename"); + stream.continueUntil(0x00); + stream.moveForwardsBy(1); + } + + // Comment + if (flags & 0x10) { + console.log("Comment"); + stream.continueUntil(0x00); + stream.moveForwardsBy(1); + } + + // Checksum + if (flags & 0x2) { + console.log("Checksum"); + stream.moveForwardsBy(2); + } + + + /* DEFLATE DATA */ + + let finalBlock = 0; + + while (!finalBlock) { + // Read header + const blockHeader = stream.readBits(3); + + finalBlock = blockHeader & 0x1; + const blockType = blockHeader & 0x6; + + if (blockType === 0) { + // No compression + stream.moveForwardsBy(1); + const blockLength = stream.readInt(2, "le"); + console.log("No compression. Length: " + blockLength); + stream.moveForwardsBy(2 + blockLength); + } else if (blockType === 1) { + // Fixed Huffman + + } else if (blockType === 2) { + // Dynamic Huffman + + } else { + throw new Error("Invalid block type"); + break; + } + } + + + /* FOOTER */ + + // Skip over checksum and size of original uncompressed input + stream.moveForwardsBy(8); + + console.log(stream.position); + + return stream.carve(); +} diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 0a29b5a6..2b5d8d09 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -21,8 +21,9 @@ export default class Stream { */ constructor(input) { this.bytes = input; - this.position = 0; this.length = this.bytes.length; + this.position = 0; + this.bitPos = 0; } /** @@ -37,6 +38,7 @@ export default class Stream { const newPosition = this.position + numBytes; const bytes = this.bytes.slice(this.position, newPosition); this.position = newPosition; + this.bitPos = 0; return bytes; } @@ -57,6 +59,7 @@ export default class Stream { result += String.fromCharCode(currentByte); } this.position += numBytes; + this.bitPos = 0; return result; } @@ -83,9 +86,59 @@ export default class Stream { } } this.position += numBytes; + this.bitPos = 0; return val; } + + /** + * Reads a number of bits from the buffer. + * + * @TODO Add endianness + * + * @param {number} numBits + * @returns {number} + */ + readBits(numBits) { + if (this.position > this.length) return undefined; + + let bitBuf = 0, + bitBufLen = 0; + + // Add remaining bits from current byte + bitBuf = this.bytes[this.position++] & bitMask(this.bitPos); + bitBufLen = 8 - this.bitPos; + this.bitPos = 0; + + // Not enough bits yet + while (bitBufLen < numBits) { + bitBuf |= this.bytes[this.position++] << bitBufLen; + bitBufLen += 8; + } + + // Reverse back to numBits + if (bitBufLen > numBits) { + const excess = bitBufLen - numBits; + bitBuf >>>= excess; + bitBufLen -= excess; + this.position--; + this.bitPos = 8 - excess; + } + + return bitBuf; + + /** + * Calculates the bit mask based on the current bit position. + * + * @param {number} bitPos + * @returns {number} The bit mask + */ + function bitMask(bitPos) { + return (1 << (8 - bitPos)) - 1; + } + } + + /** * Consume the stream until we reach the specified byte or sequence of bytes. * @@ -94,6 +147,8 @@ export default class Stream { continueUntil(val) { if (this.position > this.length) return; + this.bitPos = 0; + if (typeof val === "number") { while (++this.position < this.length && this.bytes[this.position] !== val) { continue; @@ -121,8 +176,10 @@ export default class Stream { * @param {number} val */ consumeIf(val) { - if (this.bytes[this.position] === val) + if (this.bytes[this.position] === val) { this.position++; + this.bitPos = 0; + } } /** @@ -135,6 +192,7 @@ export default class Stream { if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; + this.bitPos = 0; } @@ -148,6 +206,7 @@ export default class Stream { if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; + this.bitPos = 0; } /** @@ -159,6 +218,7 @@ export default class Stream { if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; + this.bitPos = 0; } /** @@ -176,6 +236,7 @@ export default class Stream { * @returns {Uint8Array} */ carve() { + if (this.bitPos > 0) this.position++; return this.bytes.slice(0, this.position); } From b6eb4e84e4ea9da996c0a3c08d9f9c4da78ad4f5 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Fri, 4 Jan 2019 13:33:31 +0000 Subject: [PATCH 023/110] Add Bombe operation Still needs some work, but functional --- src/core/config/Categories.json | 3 +- src/core/lib/Bombe.mjs | 472 +++++++++++++++++++++++++++++++ src/core/lib/Enigma.mjs | 25 +- src/core/operations/Bombe.mjs | 118 ++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/Bombe.mjs | 67 +++++ 6 files changed, 682 insertions(+), 4 deletions(-) create mode 100644 src/core/lib/Bombe.mjs create mode 100644 src/core/operations/Bombe.mjs create mode 100644 tests/operations/tests/Bombe.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3e0d108e..5a40846c 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -103,7 +103,8 @@ "Citrix CTX1 Encode", "Citrix CTX1 Decode", "Pseudo-Random Number Generator", - "Enigma" + "Enigma", + "Bombe" ] }, { diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs new file mode 100644 index 00000000..70a2d2cb --- /dev/null +++ b/src/core/lib/Bombe.mjs @@ -0,0 +1,472 @@ +/** + * Emulation of the Bombe machine. + * + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import OperationError from "../errors/OperationError"; +import Utils from "../Utils"; +import {Rotor, a2i, i2a} from "./Enigma"; + +/** + * Convenience/optimisation subclass of Rotor + * + * This allows creating multiple Rotors which share backing maps, to avoid repeatedly parsing the + * rotor spec strings and duplicating the maps in memory. + */ +class CopyRotor extends Rotor { + /** + * Return a copy of this Rotor. + */ + copy() { + const clone = { + map: this.map, + revMap: this.revMap, + pos: this.pos, + step: this.step, + transform: this.transform, + revTransform: this.revTransform, + }; + return clone; + } +} + +/** + * Node in the menu graph + * + * A node represents a cipher/plaintext letter. + */ +class Node { + /** + * Node constructor. + * @param {number} letter - The plain/ciphertext letter this node represents (as a number). + */ + constructor(letter) { + this.letter = letter; + this.edges = new Set(); + this.visited = false; + } +} + +/** + * Edge in the menu graph + * + * An edge represents an Enigma machine transformation between two letters. + */ +class Edge { + /** + * Edge constructor - an Enigma machine mapping between letters + * @param {number} pos - The rotor position, relative to the beginning of the crib, at this edge + * @param {number} node1 - Letter at one end (as a number) + * @param {number} node2 - Letter at the other end + */ + constructor(pos, node1, node2) { + this.pos = pos; + this.node1 = node1; + this.node2 = node2; + node1.edges.add(this); + node2.edges.add(this); + this.visited = false; + } + + /** + * Given the node at one end of this edge, return the other end. + * @param node {number} - The node we have + * @returns {number} + */ + getOther(node) { + if (this.node1 === node) { + return this.node2; + } + return this.node1; + } +} + +/** + * Scrambler. + * + * This is effectively just an Enigma machine, but it only operates on one character at a time and + * the stepping mechanism is different. + */ +class Scrambler { + /** Scrambler constructor. + * @param {Object[]} rotors - List of rotors in this scrambler + * @param {Object} reflector - This scrambler's reflector + * @param {number} pos - Position offset from start of crib + * @param {number} end1 - Letter in menu this scrambler is attached to + * @param {number} end2 - Other letter in menu this scrambler is attached to + */ + constructor(rotors, reflector, pos, end1, end2) { + this.reflector = reflector; + this.rotors = rotors; + this.rotorsRev = [].concat(rotors).reverse(); + this.initialPos = pos; + this.rotors[0].pos += pos; + this.end1 = end1; + this.end2 = end2; + } + + /** + * Step the rotors forward. + * + * All nodes in the Bombe step in sync. + * @param {number} n - How many rotors to step + */ + step(n) { + // The Bombe steps the slowest rotor on an actual Enigma first. + for (let i=this.rotors.length - 1; i>=this.rotors.length-n; i--) { + this.rotors[i].step(); + } + } + + /** + * Run a letter through the scrambler. + * @param {number} i - The letter to transform (as a number) + * @returns {number} + */ + transform(i) { + let letter = i; + for (const rotor of this.rotors) { + letter = rotor.transform(letter); + } + letter = this.reflector.transform(letter); + for (const rotor of this.rotorsRev) { + letter = rotor.revTransform(letter); + } + return letter; + } + + /** + * Given one letter in the menu this scrambler maps to, return the other. + * @param end {number} - The node we have + * @returns {number} + */ + getOtherEnd(end) { + if (this.end1 === end) { + return this.end2; + } + return this.end1; + } + + /** + * Read the position this scrambler is set to. + * Note that because of Enigma's stepping, you need to set an actual Enigma to the previous + * position in order to get it to make a certain set of electrical connections when a button + * is pressed - this function *does* take this into account. + * However, as with the rest of the Bombe, it does not take stepping into account - the middle + * and slow rotors are treated as static. + * @return {string} + */ + getPos() { + let result = ""; + for (let i=0; i 25) { + // A crib longer than this will definitely cause the middle rotor to step somewhere + // A shorter crib is preferable to reduce this chance, of course + throw new OperationError("Crib is too long"); + } + for (let i=0; i nConnections) { + mostConnected = oMostConnected; + nConnections = oNConnections; + } + } + return [loops, nNodes, mostConnected, nConnections, edges]; + } + + /** + * Build a menu from the ciphertext and crib. + * A menu is just a graph where letters in either the ciphertext or crib (Enigma is symmetric, + * so there's no difference mathematically) are nodes and states of the Enigma machine itself + * are the edges. + * Additionally, we want a single connected graph, and of the subgraphs available, we want the + * one with the most loops (since these generate feedback cycles which efficiently close off + * disallowed states). + * Finally, we want to identify the most connected node in that graph (as it's the best choice + * of measurement point). + * @returns [Object, Object[]] - the most connected node, and the list of edges in the subgraph + */ + makeMenu() { + // First, we make a graph of all of the mappings given by the crib + // Make all nodes first + const nodes = new Map(); + for (const c of this.ciphertext + this.crib) { + if (!nodes.has(c)) { + const node = new Node(c); + nodes.set(c, node); + } + } + // Then all edges + for (let i=0; i { + let result = b[0] - a[0]; + if (result === 0) { + result = b[1] - a[1]; + } + return result; + }); + this.nLoops = graphs[0][0]; + return [graphs[0][2], graphs[0][4]]; + } + + /** + * Implement Welchman's diagonal board: If A steckers to B, that implies B steckers to A, and + * so forth. This function just gets the paired wire. + * @param {number[2]} i - Bombe state wire + * @returns {number[2]} + */ + getDiagonal(i) { + return [i[1], i[0]]; + } + + /** + * Bombe electrical simulation. Energise a wire. For all connected wires (both via the diagonal + * board and via the scramblers), energise them too, recursively. + * @param {number[2]} i - Bombe state wire + */ + energise(i) { + const idx = 26*i[0] + i[1]; + if (this.wires[idx]) { + return; + } + this.energiseCount ++; + this.wires[idx] = true; + this.energise(this.getDiagonal(i)); + + for (const scrambler of this.scramblers[i[0]]) { + const out = scrambler.transform(i[1]); + const other = scrambler.getOtherEnd(i[0]); + this.energise([other, out]); + } + } + + /** + * Having set up the Bombe, do the actual attack run. This tries every possible rotor setting + * and attempts to logically invalidate them. If it can't, it's added to the list of candidate + * solutions. + * @returns {string[][2]} - list of pairs of candidate rotor setting, and calculated stecker pair + */ + run() { + let stops = 0; + const result = []; + // For each possible rotor setting + const nChecks = Math.pow(26, this.baseRotors.length); + for (let i=1; i<=nChecks; i++) { + this.wires.fill(false); + // Energise the test input, follow the current through each scrambler + // (and the diagonal board) + this.energiseCount = 0; + this.energise(this.testInput); + // Count the energised outputs + let count = 0; + for (let j=26*this.testRegister; j<26*(1+this.testRegister); j++) { + if (this.wires[j]) { + count++; + } + } + // If it's not all of them, we have a stop + if (count < 26) { + stops += 1; + let stecker; + // The Bombe tells us one stecker pair as well. The input wire and test register we + // started with are hypothesised to be a stecker pair. + if (count === 25) { + // Our steckering hypothesis is wrong. Correct value is the un-energised wire. + for (let j=0; j<26; j++) { + if (!this.wires[26*this.testRegister + j]) { + stecker = `${i2a(this.testRegister)} <-> ${i2a(j)}`; + break; + } + } + } else if (count === 1) { + // This means our hypothesis for the steckering is correct. + stecker = `${i2a(this.testRegister)} <-> ${i2a(this.testInput[1])}`; + } else { + // Unusual, probably indicative of a poor menu. I'm a little unclear on how + // this was really handled, but we'll return it for the moment. + stecker = `? (wire count: ${count})`; + } + result.push([this.indicator.getPos(), stecker]); + } + // Step all the scramblers + // This loop counts how many rotors have reached their starting position (meaning the + // next one needs to step as well) + let n = 1; + for (let j=1; j 2) { + const msg = `Bombe run with ${this.nLoops} loops in menu (2+ desirable): ${stops} stops, ${Math.floor(100 * i / nChecks)}% done`; + this.update(msg); + } + } + return result; + } +} diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs index 845ce413..9fc0f7d0 100644 --- a/src/core/lib/Enigma.mjs +++ b/src/core/lib/Enigma.mjs @@ -103,15 +103,17 @@ export class Rotor { if (!/^[A-Z]$/.test(initialPosition)) { throw new OperationError("Rotor initial position must be exactly one uppercase letter"); } - this.map = {}; - this.revMap = {}; + this.map = new Array(26).fill(); + this.revMap = new Array(26).fill(); + const uniq = {}; for (let i=0; i S\)/, + recipeConfig: [ + { + "op": "Bombe", + "args": [ + "BDFHJLCPRTXVZNYEIWGAKMUSQO G\)/, + recipeConfig: [ + { + "op": "Bombe", + "args": [ + "BDFHJLCPRTXVZNYEIWGAKMUSQO S\)/, + recipeConfig: [ + { + "op": "Bombe", + "args": [ + "BDFHJLCPRTXVZNYEIWGAKMUSQO Date: Tue, 8 Jan 2019 18:25:42 +0000 Subject: [PATCH 024/110] Bombe: review, tests, validation --- src/core/lib/Bombe.mjs | 34 ++++----- src/core/lib/Enigma.mjs | 6 +- src/core/operations/Bombe.mjs | 31 ++++++-- src/core/operations/Enigma.mjs | 6 +- tests/operations/tests/Bombe.mjs | 127 +++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 31 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 70a2d2cb..53766560 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -77,10 +77,7 @@ class Edge { * @returns {number} */ getOther(node) { - if (this.node1 === node) { - return this.node2; - } - return this.node1; + return this.node1 === node ? this.node2 : this.node1; } } @@ -144,10 +141,7 @@ class Scrambler { * @returns {number} */ getOtherEnd(end) { - if (this.end1 === end) { - return this.end2; - } - return this.end1; + return this.end1 === end ? this.end2 : this.end1; } /** @@ -194,8 +188,11 @@ export class BombeMachine { * @param {function} update - Function to call to send status updates (optional) */ constructor(rotors, reflector, ciphertext, crib, update=undefined) { - if (ciphertext.length !== crib.length) { - throw new OperationError("Ciphertext and crib length differ"); + if (ciphertext.length < crib.length) { + throw new OperationError("Crib overruns supplied ciphertext"); + } + if (ciphertext.length > crib.length) { + throw new OperationError("Ciphertext is longer than crib"); } if (crib.length < 2) { // This is the absolute bare minimum to be sane, and even then it's likely too short to @@ -226,7 +223,7 @@ export class BombeMachine { // This is the bundle of wires corresponding to the 26 letters within each of the 26 // possible nodes in the menu - this.wires = new Array(26*26).fill(false); + this.wires = new Array(26*26); // These are the pseudo-Engima devices corresponding to each edge in the menu, and the // nodes in the menu they each connect to @@ -271,9 +268,9 @@ export class BombeMachine { * If we have a way of sending status messages, do so. * @param {string} msg - Message to send. */ - update(msg) { + update(...msg) { if (this.updateFn !== undefined) { - this.updateFn(msg); + this.updateFn(...msg); } } @@ -411,7 +408,10 @@ export class BombeMachine { // For each possible rotor setting const nChecks = Math.pow(26, this.baseRotors.length); for (let i=1; i<=nChecks; i++) { - this.wires.fill(false); + // Benchmarking suggests this is faster than using .fill() + for (let i=0; i 2) { - const msg = `Bombe run with ${this.nLoops} loops in menu (2+ desirable): ${stops} stops, ${Math.floor(100 * i / nChecks)}% done`; - this.update(msg); + // (note this won't be triggered on 3-rotor runs - they run fast enough it doesn't seem necessary) + if (n > 3) { + this.update(this.nLoops, stops, i/nChecks); } } return result; diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs index 9fc0f7d0..cfc93933 100644 --- a/src/core/lib/Enigma.mjs +++ b/src/core/lib/Enigma.mjs @@ -103,8 +103,8 @@ export class Rotor { if (!/^[A-Z]$/.test(initialPosition)) { throw new OperationError("Rotor initial position must be exactly one uppercase letter"); } - this.map = new Array(26).fill(); - this.revMap = new Array(26).fill(); + this.map = new Array(26); + this.revMap = new Array(26); const uniq = {}; for (let i=0; i S\)/, + recipeConfig: [ + { + "op": "Bombe", + "args": [ + "BDFHJLCPRTXVZNYEIWGAKMUSQO Date: Tue, 8 Jan 2019 19:37:34 +0000 Subject: [PATCH 025/110] Bombe: add trial decryption preview --- src/core/lib/Bombe.mjs | 60 +++++++++++++++++++++++++++----- src/core/operations/Bombe.mjs | 8 ++--- tests/operations/tests/Bombe.mjs | 27 +++++++++++--- 3 files changed, 79 insertions(+), 16 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 53766560..1e6c3d2d 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -191,9 +191,6 @@ export class BombeMachine { if (ciphertext.length < crib.length) { throw new OperationError("Crib overruns supplied ciphertext"); } - if (ciphertext.length > crib.length) { - throw new OperationError("Ciphertext is longer than crib"); - } if (crib.length < 2) { // This is the absolute bare minimum to be sane, and even then it's likely too short to // be useful @@ -204,7 +201,7 @@ export class BombeMachine { // A shorter crib is preferable to reduce this chance, of course throw new OperationError("Crib is too long"); } - for (let i=0; i ${i2a(j)}`; + stecker = [this.testRegister, j]; break; } } } else if (count === 1) { // This means our hypothesis for the steckering is correct. - stecker = `${i2a(this.testRegister)} <-> ${i2a(this.testInput[1])}`; + stecker = [this.testRegister, this.testInput[1]]; } else { // Unusual, probably indicative of a poor menu. I'm a little unclear on how // this was really handled, but we'll return it for the moment. - stecker = `? (wire count: ${count})`; + stecker = undefined; } - result.push([this.indicator.getPos(), stecker]); + const testDecrypt = this.tryDecrypt(stecker); + let steckerStr; + if (stecker !== undefined) { + steckerStr = `${i2a(stecker[0])}${i2a(stecker[1])}`; + } else { + steckerStr = `?? (wire count: ${count})`; + } + result.push([this.indicator.getPos(), steckerStr, testDecrypt]); } // Step all the scramblers // This loop counts how many rotors have reached their starting position (meaning the diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index e875e50d..9ddd4b7b 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -112,7 +112,7 @@ class Bombe extends Operation { // For symmetry with the Enigma op, for the input we'll just remove all invalid characters input = input.replace(/[^A-Za-z]/g, "").toUpperCase(); crib = crib.replace(/[^A-Za-z]/g, "").toUpperCase(); - const ciphertext = input.slice(offset, offset+crib.length); + const ciphertext = input.slice(offset); const reflector = new Reflector(reflectorstr); let update; if (ENVIRONMENT_IS_WORKER()) { @@ -122,9 +122,9 @@ class Bombe extends Operation { } const bombe = new BombeMachine(rotors, reflector, ciphertext, crib, update); const result = bombe.run(); - let msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotor positions are listed left to right and start at the beginning of the crib, and ignore stepping and the ring setting. One stecker pair is determined. Results:\n`; - for (const [setting, wires] of result) { - msg += `Stop: ${setting} (${wires})\n`; + let msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotor positions are listed left to right and start at the beginning of the crib, and ignore stepping and the ring setting. One stecker pair is determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; + for (const [setting, stecker, decrypt] of result) { + msg += `Stop: ${setting} (plugboard: ${stecker}): ${decrypt}\n`; } return msg; } diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index 8742cfc0..65f4b701 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -8,9 +8,10 @@ import TestRegister from "../TestRegister"; TestRegister.addTests([ { + // Plugboard for this test is BO LC KE GA name: "Bombe: 3 rotor (self-stecker)", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA \(S <-> S\)/, + expectedMatch: /LGA \(plugboard: SS\): VFISUSGTKSTMPSUNAK/, recipeConfig: [ { "op": "Bombe", @@ -28,7 +29,7 @@ TestRegister.addTests([ { name: "Bombe: 3 rotor (other stecker)", input: "JBYALIHDYNUAAVKBYM", - expectedMatch: /LGA \(A <-> G\)/, + expectedMatch: /LGA \(plugboard: AG\): QFIMUMAFKMQSKMYNGW/, recipeConfig: [ { "op": "Bombe", @@ -46,7 +47,7 @@ TestRegister.addTests([ { name: "Bombe: crib offset", input: "AAABBYFLTHHYIJQAYBBYS", // first three chars here are faked - expectedMatch: /LGA \(S <-> S\)/, + expectedMatch: /LGA \(plugboard: SS\): VFISUSGTKSTMPSUNAK/, recipeConfig: [ { "op": "Bombe", @@ -61,12 +62,30 @@ TestRegister.addTests([ } ] }, + { + name: "Bombe: multiple stops", + input: "BBYFLTHHYIJQAYBBYS", + expectedMatch: /LGA \(plugboard: TT\): VFISUSGTKSTMPSUNAK/, + recipeConfig: [ + { + "op": "Bombe", + "args": [ + "BDFHJLCPRTXVZNYEIWGAKMUSQO S\)/, + expectedMatch: /LHSC \(plugboard: SS\)/, recipeConfig: [ { "op": "Bombe", From 8c757d1e03918ef2f3fb9098e0642719260b6690 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Wed, 9 Jan 2019 20:44:14 +0000 Subject: [PATCH 026/110] Bombe: optimise This cuts about 85% off the execution time. --- src/core/lib/Bombe.mjs | 209 +++++++++++++++++++++++-------- tests/operations/tests/Bombe.mjs | 5 +- 2 files changed, 157 insertions(+), 57 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 1e6c3d2d..3103f56a 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -81,6 +81,104 @@ class Edge { } } +/** + * As all the Bombe's rotors move in step, at any given point the vast majority of the scramblers + * in the machine share the majority of their state, which is hosted in this class. + */ +class SharedScrambler { + /** + * SharedScrambler constructor. + * @param {Object[]} rotors - List of rotors in the shared state _only_. + * @param {Object} reflector - The reflector in use. + */ + constructor(rotors, reflector) { + this.reflector = reflector; + this.rotors = rotors; + this.rotorsRev = [].concat(rotors).reverse(); + this.lowerCache = new Array(26); + this.higherCache = new Array(26); + for (let i=0; i<26; i++) { + this.higherCache[i] = new Array(26); + } + this.cacheGen(); + } + + /** + * Step the rotors forward. + * @param {number} n - How many rotors to step. This includes the rotors which are not part of + * the shared state, so should be 2 or more. + */ + step(n) { + for (let i=0; i=this.rotors.length-n; i--) { - this.rotors[i].step(); - } + step() { + // The Bombe steps the slowest rotor on an actual Enigma fastest, for reasons. + // ...but for optimisation reasons I'm going to cheat and not do that, as this vastly + // simplifies caching the state of the majority of the scramblers. The results are the + // same, just in a slightly different order. + this.rotor.step(); } + /** * Run a letter through the scrambler. * @param {number} i - The letter to transform (as a number) @@ -125,13 +224,14 @@ class Scrambler { */ transform(i) { let letter = i; - for (const rotor of this.rotors) { - letter = rotor.transform(letter); - } - letter = this.reflector.transform(letter); - for (const rotor of this.rotorsRev) { - letter = rotor.revTransform(letter); + const cached = this.baseScrambler.fullTransform(this.rotor.pos, i); + if (cached !== undefined) { + return cached; } + letter = this.rotor.transform(letter); + letter = this.baseScrambler.transform(letter); + letter = this.rotor.revTransform(letter); + this.baseScrambler.addCache(this.rotor.pos, i, letter); return letter; } @@ -155,15 +255,11 @@ class Scrambler { */ getPos() { let result = ""; - for (let i=0; i 1) { + this.sharedScrambler.step(n); + } for (const scrambler of this.allScramblers) { - scrambler.step(n); + scrambler.step(); } // Send status messages at what seems to be a reasonably sensible frequency // (note this won't be triggered on 3-rotor runs - they run fast enough it doesn't seem necessary) diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index 65f4b701..6a96884c 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -27,6 +27,7 @@ TestRegister.addTests([ ] }, { + // This test produces a menu that doesn't use the first letter, which is also a good test name: "Bombe: 3 rotor (other stecker)", input: "JBYALIHDYNUAAVKBYM", expectedMatch: /LGA \(plugboard: AG\): QFIMUMAFKMQSKMYNGW/, @@ -80,8 +81,7 @@ TestRegister.addTests([ } ] }, - /* - * Long test is long + // This test is a bit slow - it takes about 12s on my test hardware { name: "Bombe: 4 rotor", input: "LUOXGJSHGEDSRDOQQX", @@ -100,7 +100,6 @@ TestRegister.addTests([ } ] }, - */ { name: "Bombe: no crib", input: "JBYALIHDYNUAAVKBYM", From c077b22410037559d0e8af9fedc6f0971a67b9ab Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 17:30:52 +0000 Subject: [PATCH 027/110] Stream.readBits() method implemented. Unfinished. --- src/core/lib/FileSignatures.mjs | 12 +++++------- src/core/lib/Stream.mjs | 7 +++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index b0eb5884..d8d97a2b 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1438,10 +1438,8 @@ export function extractGZIP(bytes, offset) { while (!finalBlock) { // Read header - const blockHeader = stream.readBits(3); - - finalBlock = blockHeader & 0x1; - const blockType = blockHeader & 0x6; + finalBlock = stream.readBits(1); + const blockType = stream.readBits(2); if (blockType === 0) { // No compression @@ -1451,23 +1449,23 @@ export function extractGZIP(bytes, offset) { stream.moveForwardsBy(2 + blockLength); } else if (blockType === 1) { // Fixed Huffman + console.log("Fixed Huffman"); } else if (blockType === 2) { // Dynamic Huffman - + console.log("Dynamic Huffman"); } else { throw new Error("Invalid block type"); break; } } - /* FOOTER */ // Skip over checksum and size of original uncompressed input stream.moveForwardsBy(8); - console.log(stream.position); + console.log("Ending at " + stream.position); return stream.carve(); } diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 2b5d8d09..886d4589 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -106,7 +106,7 @@ export default class Stream { bitBufLen = 0; // Add remaining bits from current byte - bitBuf = this.bytes[this.position++] & bitMask(this.bitPos); + bitBuf = (this.bytes[this.position++] & bitMask(this.bitPos)) >>> this.bitPos; bitBufLen = 8 - this.bitPos; this.bitPos = 0; @@ -119,7 +119,7 @@ export default class Stream { // Reverse back to numBits if (bitBufLen > numBits) { const excess = bitBufLen - numBits; - bitBuf >>>= excess; + bitBuf &= (1 << numBits) - 1; bitBufLen -= excess; this.position--; this.bitPos = 8 - excess; @@ -134,11 +134,10 @@ export default class Stream { * @returns {number} The bit mask */ function bitMask(bitPos) { - return (1 << (8 - bitPos)) - 1; + return 256 - (1 << bitPos); } } - /** * Consume the stream until we reach the specified byte or sequence of bytes. * From 3eb44708e503e42e3429ca9488b004efeef5b5e5 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 10 Jan 2019 18:04:02 +0000 Subject: [PATCH 028/110] Add MultiBombe Runs the Bombe multiple times with different rotor specs. Edits the core BombeMachine a little to add the ability to switch rotors without rewiring everything --- src/core/config/Categories.json | 3 +- src/core/lib/Bombe.mjs | 63 ++++- src/core/lib/Enigma.mjs | 1 + src/core/operations/MultipleBombe.mjs | 307 +++++++++++++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/Bombe.mjs | 2 +- tests/operations/tests/MultipleBombe.mjs | 47 ++++ 7 files changed, 411 insertions(+), 13 deletions(-) create mode 100644 src/core/operations/MultipleBombe.mjs create mode 100644 tests/operations/tests/MultipleBombe.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 5a40846c..986606c9 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -104,7 +104,8 @@ "Citrix CTX1 Decode", "Pseudo-Random Number Generator", "Enigma", - "Bombe" + "Bombe", + "Multiple Bombe" ] }, { diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 3103f56a..8b781b68 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -92,14 +92,24 @@ class SharedScrambler { * @param {Object} reflector - The reflector in use. */ constructor(rotors, reflector) { - this.reflector = reflector; - this.rotors = rotors; - this.rotorsRev = [].concat(rotors).reverse(); this.lowerCache = new Array(26); this.higherCache = new Array(26); for (let i=0; i<26; i++) { this.higherCache[i] = new Array(26); } + this.changeRotors(rotors, reflector); + } + + /** + * Replace the rotors and reflector in this SharedScrambler. + * This takes care of flushing caches as well. + * @param {Object[]} rotors - List of rotors in the shared state _only_. + * @param {Object} reflector - The reflector in use. + */ + changeRotors(rotors, reflector) { + this.reflector = reflector; + this.rotors = rotors; + this.rotorsRev = [].concat(rotors).reverse(); this.cacheGen(); } @@ -195,13 +205,22 @@ class Scrambler { */ constructor(base, rotor, pos, end1, end2) { this.baseScrambler = base; - this.rotor = rotor; this.initialPos = pos; - this.rotor.pos += pos; + this.changeRotor(rotor); this.end1 = end1; this.end2 = end2; } + /** + * Replace the rotor in this scrambler. + * The position is reset automatically. + * @param {Object} rotor - New rotor + */ + changeRotor(rotor) { + this.rotor = rotor; + this.rotor.pos += this.initialPos; + } + /** * Step the rotors forward. * @@ -304,12 +323,7 @@ export class BombeMachine { } this.ciphertext = ciphertext; this.crib = crib; - // This is ordered from the Enigma fast rotor to the slow, so bottom to top for the Bombe - this.baseRotors = []; - for (const rstr of rotors) { - const rotor = new CopyRotor(rstr, "", "A", "A"); - this.baseRotors.push(rotor); - } + this.initRotors(rotors); this.updateFn = update; const [mostConnected, edges] = this.makeMenu(); @@ -355,6 +369,33 @@ export class BombeMachine { } } + /** + * Build Rotor objects from list of rotor wiring strings. + * @param {string[]} rotors - List of rotor wiring strings + */ + initRotors(rotors) { + // This is ordered from the Enigma fast rotor to the slow, so bottom to top for the Bombe + this.baseRotors = []; + for (const rstr of rotors) { + const rotor = new CopyRotor(rstr, "", "A", "A"); + this.baseRotors.push(rotor); + } + } + + /** + * Replace the rotors and reflector in all components of this Bombe. + * @param {string[]} rotors - List of rotor wiring strings + * @param {Object} reflector - Reflector object + */ + changeRotors(rotors, reflector) { + // At the end of the run, the rotors are all back in the same position they started + this.initRotors(rotors); + this.sharedScrambler.changeRotors(this.baseRotors.slice(1), reflector); + for (const scrambler of this.allScramblers) { + scrambler.changeRotor(this.baseRotors[0].copy()); + } + } + /** * If we have a way of sending status messages, do so. * @param {string} msg - Message to send. diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs index cfc93933..0a083bce 100644 --- a/src/core/lib/Enigma.mjs +++ b/src/core/lib/Enigma.mjs @@ -171,6 +171,7 @@ class PairMapBase { constructor(pairs, name="PairMapBase") { // I've chosen to make whitespace significant here to make a) code and // b) inputs easier to read + this.pairs = pairs; this.map = {}; if (pairs === "") { return; diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs new file mode 100644 index 00000000..6bcd1051 --- /dev/null +++ b/src/core/operations/MultipleBombe.mjs @@ -0,0 +1,307 @@ +/** + * Emulation of the Bombe machine. + * This version carries out multiple Bombe runs to handle unknown rotor configurations. + * + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import {BombeMachine} from "../lib/Bombe"; +import {ROTORS, REFLECTORS, Reflector} from "../lib/Enigma"; + +/** + * Convenience method for flattening the preset ROTORS object into a newline-separated string. + * @param {Object[]} - Preset rotors object + * @param {number} s - Start index + * @param {number} n - End index + * @returns {string} + */ +function rotorsFormat(rotors, s, n) { + const res = []; + for (const i of rotors.slice(s, n)) { + res.push(i.value); + } + return res.join("\n"); +} + +/** + * Combinatorics choose function + * @param {number} n + * @param {number} k + * @returns number + */ +function choose(n, k) { + let res = 1; + for (let i=1; i<=k; i++) { + res *= (n + 1 - i) / i; + } + return res; +} + +/** + * Bombe operation + */ +class MultipleBombe extends Operation { + /** + * Bombe constructor + */ + constructor() { + super(); + + this.name = "Multiple Bombe"; + this.module = "Default"; + this.description = "Emulation of the Bombe machine used to attack Enigma. This version carries out multiple Bombe runs to handle unknown rotor configurations.

You should test your menu on the single Bombe operation before running it here. See the description of the Bombe operation for instructions on choosing a crib."; + this.infoURL = "https://wikipedia.org/wiki/Bombe"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + "name": "Standard Enigmas", + "type": "populateOption", + "value": [ + { + name: "German Service Enigma (First - 3 rotor)", + value: rotorsFormat(ROTORS, 0, 5) + }, + { + name: "German Service Enigma (Second - 3 rotor)", + value: rotorsFormat(ROTORS, 0, 8) + }, + { + name: "German Service Enigma (Third - 4 rotor)", + value: rotorsFormat(ROTORS, 0, 8) + }, + { + name: "German Service Enigma (Fourth - 4 rotor)", + value: rotorsFormat(ROTORS, 0, 8) + }, + { + name: "User defined", + value: "" + }, + ], + "target": 1 + }, + { + name: "Main rotors", + type: "text", + value: "" + }, + { + "name": "Standard Enigmas", + "type": "populateOption", + "value": [ + { + name: "German Service Enigma (First - 3 rotor)", + value: "" + }, + { + name: "German Service Enigma (Second - 3 rotor)", + value: "" + }, + { + name: "German Service Enigma (Third - 4 rotor)", + value: rotorsFormat(ROTORS, 8, 9) + }, + { + name: "German Service Enigma (Fourth - 4 rotor)", + value: rotorsFormat(ROTORS, 8, 10) + }, + { + name: "User defined", + value: "" + }, + ], + "target": 3 + }, + { + name: "4th rotor", + type: "text", + value: "" + }, + { + "name": "Standard Enigmas", + "type": "populateOption", + "value": [ + { + name: "German Service Enigma (First - 3 rotor)", + value: rotorsFormat(REFLECTORS, 0, 1) + }, + { + name: "German Service Enigma (Second - 3 rotor)", + value: rotorsFormat(REFLECTORS, 0, 2) + }, + { + name: "German Service Enigma (Third - 4 rotor)", + value: rotorsFormat(REFLECTORS, 2, 3) + }, + { + name: "German Service Enigma (Fourth - 4 rotor)", + value: rotorsFormat(REFLECTORS, 2, 4) + }, + { + name: "User defined", + value: "" + }, + ], + "target": 5 + }, + { + name: "Reflectors", + type: "text", + value: "" + }, + { + name: "Crib", + type: "string", + value: "" + }, + { + name: "Crib offset", + type: "number", + value: 0 + } + ]; + } + + /** + * Format and send a status update message. + * @param {number} nLoops - Number of loops in the menu + * @param {number} nStops - How many stops so far + * @param {number} progress - Progress (as a float in the range 0..1) + */ + updateStatus(nLoops, nStops, progress) { + const msg = `Bombe run with ${nLoops} loops in menu (2+ desirable): ${nStops} stops, ${Math.floor(100 * progress)}% done`; + self.sendStatusMessage(msg); + } + + /** + * Early rotor description string validation. + * Drops stepping information. + * @param {string} rstr - The rotor description string + * @returns {string} - Rotor description with stepping stripped, if any + */ + validateRotor(rstr) { + // The Bombe doesn't take stepping into account so we'll just ignore it here + if (rstr.includes("<")) { + rstr = rstr.split("<", 2)[0]; + } + // Duplicate the validation of the rotor strings here, otherwise you might get an error + // thrown halfway into a big Bombe run + if (!/^[A-Z]{26}$/.test(rstr)) { + throw new OperationError("Rotor wiring must be 26 unique uppercase letters"); + } + if (new Set(rstr).size !== 26) { + throw new OperationError("Rotor wiring must be 26 unique uppercase letters"); + } + return rstr; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const mainRotorsStr = args[1]; + const fourthRotorsStr = args[3]; + const reflectorsStr = args[5]; + let crib = args[6]; + const offset = args[7]; + // TODO got this far + const rotors = []; + const fourthRotors = []; + const reflectors = []; + for (let rstr of mainRotorsStr.split("\n")) { + rstr = this.validateRotor(rstr); + rotors.push(rstr); + } + if (rotors.length < 3) { + throw new OperationError("A minimum of three rotors must be supplied"); + } + if (fourthRotorsStr !== "") { + for (let rstr of fourthRotorsStr.split("\n")) { + rstr = this.validateRotor(rstr); + fourthRotors.push(rstr); + } + } + if (fourthRotors.length === 0) { + fourthRotors.push(""); + } + for (const rstr of reflectorsStr.split("\n")) { + const reflector = new Reflector(rstr); + reflectors.push(reflector); + } + if (reflectors.length === 0) { + throw new OperationError("A minimum of one reflector must be supplied"); + } + if (crib.length === 0) { + throw new OperationError("Crib cannot be empty"); + } + if (offset < 0) { + throw new OperationError("Offset cannot be negative"); + } + // For symmetry with the Enigma op, for the input we'll just remove all invalid characters + input = input.replace(/[^A-Za-z]/g, "").toUpperCase(); + crib = crib.replace(/[^A-Za-z]/g, "").toUpperCase(); + const ciphertext = input.slice(offset); + let update; + if (ENVIRONMENT_IS_WORKER()) { + update = this.updateStatus; + } else { + update = undefined; + } + let bombe = undefined; + let msg; + // I could use a proper combinatorics algorithm here... but it would be more code to + // write one, and we don't seem to have one in our existing libraries, so massively nested + // for loop it is + const totalRuns = choose(rotors.length, 3) * 6 * fourthRotors.length * reflectors.length; + let nRuns = 0; + let nStops = 0; + for (const rotor1 of rotors) { + for (const rotor2 of rotors) { + if (rotor2 === rotor1) { + continue; + } + for (const rotor3 of rotors) { + if (rotor3 === rotor2 || rotor3 === rotor1) { + continue; + } + for (const rotor4 of fourthRotors) { + for (const reflector of reflectors) { + nRuns++; + const runRotors = [rotor1, rotor2, rotor3]; + if (rotor4 !== "") { + runRotors.push(rotor4); + } + if (bombe === undefined) { + bombe = new BombeMachine(runRotors, reflector, ciphertext, crib); + msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotors and rotor positions are listed left to right, ignore stepping and the ring setting, and positions start at the beginning of the crib. One stecker pair is determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; + } else { + bombe.changeRotors(runRotors, reflector); + } + const result = bombe.run(); + nStops += result.length; + if (update !== undefined) { + update(bombe.nLoops, nStops, nRuns / totalRuns); + } + if (result.length > 0) { + msg += `Rotors: ${runRotors.join(", ")}\nReflector: ${reflector.pairs}\n`; + for (const [setting, stecker, decrypt] of result) { + msg += `Stop: ${setting} (plugboard: ${stecker}): ${decrypt}\n`; + } + } + } + } + } + } + } + return msg; + } +} + +export default MultipleBombe; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index dfd5fb1d..b5b25c0c 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -84,6 +84,7 @@ import "./tests/ParseTLV"; import "./tests/Media"; import "./tests/Enigma"; import "./tests/Bombe"; +import "./tests/MultipleBombe"; // Cannot test operations that use the File type yet //import "./tests/SplitColourChannels"; diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index 6a96884c..0a8af93f 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -85,7 +85,7 @@ TestRegister.addTests([ { name: "Bombe: 4 rotor", input: "LUOXGJSHGEDSRDOQQX", - expectedMatch: /LHSC \(plugboard: SS\)/, + expectedMatch: /LHSC \(plugboard: SS\): HHHSSSGQUUQPKSEKWK/, recipeConfig: [ { "op": "Bombe", diff --git a/tests/operations/tests/MultipleBombe.mjs b/tests/operations/tests/MultipleBombe.mjs new file mode 100644 index 00000000..5f7f43c4 --- /dev/null +++ b/tests/operations/tests/MultipleBombe.mjs @@ -0,0 +1,47 @@ +/** + * Bombe machine tests. + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + name: "Multi-Bombe: 3 rotor", + input: "BBYFLTHHYIJQAYBBYS", + expectedMatch: /LGA \(plugboard: SS\): VFISUSGTKSTMPSUNAK/, + recipeConfig: [ + { + "op": "Multiple Bombe", + "args": [ + // I, II and III + "User defined", "EKMFLGDQVZNTOWYHXUSPAIBRCJ Date: Thu, 10 Jan 2019 18:44:50 +0000 Subject: [PATCH 029/110] Bombe: Firefox optimisation Switch a couple of for of loops in the critical path for classic fors. This loses about 10% performance in Chrome, but it brings Firefox performance in line with Chrome's, rather than 2.5 times slower. --- src/core/lib/Bombe.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 8b781b68..7400c98a 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -514,7 +514,8 @@ export class BombeMachine { const idxPair = 26*j + i; this.wires[idxPair] = true; - for (const scrambler of this.scramblers[i]) { + for (let k=0; k Date: Fri, 11 Jan 2019 13:18:25 +0000 Subject: [PATCH 030/110] Bombe: Add checking machine --- src/core/lib/Bombe.mjs | 168 +++++++++++++++++------ src/core/lib/Enigma.mjs | 3 +- src/core/operations/Bombe.mjs | 12 +- src/core/operations/MultipleBombe.mjs | 13 +- tests/operations/tests/Bombe.mjs | 40 ++++-- tests/operations/tests/Enigma.mjs | 4 +- tests/operations/tests/MultipleBombe.mjs | 4 +- 7 files changed, 178 insertions(+), 66 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 7400c98a..4ae0ff7f 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -8,7 +8,7 @@ import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import {Rotor, a2i, i2a} from "./Enigma"; +import {Rotor, Plugboard, a2i, i2a} from "./Enigma"; /** * Convenience/optimisation subclass of Rotor @@ -302,7 +302,7 @@ export class BombeMachine { * @param {string} crib - Known plaintext for this ciphertext * @param {function} update - Function to call to send status updates (optional) */ - constructor(rotors, reflector, ciphertext, crib, update=undefined) { + constructor(rotors, reflector, ciphertext, crib, check, update=undefined) { if (ciphertext.length < crib.length) { throw new OperationError("Crib overruns supplied ciphertext"); } @@ -324,6 +324,7 @@ export class BombeMachine { this.ciphertext = ciphertext; this.crib = crib; this.initRotors(rotors); + this.check = check; this.updateFn = update; const [mostConnected, edges] = this.makeMenu(); @@ -507,7 +508,6 @@ export class BombeMachine { if (this.wires[idx]) { return; } - this.energiseCount ++; this.wires[idx] = true; // Welchman's diagonal board: if A steckers to B, that implies B steckers to A. Handle // both. @@ -564,16 +564,131 @@ export class BombeMachine { const fastRotor = this.indicator.rotor; const initialPos = fastRotor.pos; const res = []; + const plugboard = new Plugboard(stecker); // The indicator scrambler starts in the right place for the beginning of the ciphertext. for (let i=0; i 1) { + // This is an invalid stop. + return ""; + } else if (count === 0) { + // No information about steckering from this wire + continue; + } + results.add(this.formatPair(i, other)); + } + return [...results].join(" "); + } + + /** + * Check to see if the Bombe has stopped. If so, process the stop. + * @returns {(undefined|string[3])} - Undefined for no stop, or [rotor settings, plugboard settings, decryption preview] + */ + checkStop() { + // Count the energised outputs + let count = 0; + for (let j=26*this.testRegister; j<26*(1+this.testRegister); j++) { + if (this.wires[j]) { + count++; + } + } + if (count === 26) { + return undefined; + } + // If it's not all of them, we have a stop + let steckerPair; + // The Bombe tells us one stecker pair as well. The input wire and test register we + // started with are hypothesised to be a stecker pair. + if (count === 25) { + // Our steckering hypothesis is wrong. Correct value is the un-energised wire. + for (let j=0; j<26; j++) { + if (!this.wires[26*this.testRegister + j]) { + steckerPair = j; + break; + } + } + } else if (count === 1) { + // This means our hypothesis for the steckering is correct. + steckerPair = this.testInput[1]; + } else { + // If this happens a lot it implies the menu isn't good enough. We can't do + // anything useful with it as we don't have a stecker partner, so we'll just drop it + // and move on. This does risk eating the actual stop occasionally, but I've only seen + // this happen when the menu is bad enough we have thousands of stops, so I'm not sure + // it matters. + return undefined; + } + let stecker; + if (this.check) { + stecker = this.checkingMachine(steckerPair); + if (stecker === "") { + // Invalid stop - don't count it, don't return it + return undefined; + } + } else { + stecker = `${i2a(this.testRegister)}${i2a(steckerPair)}`; + } + const testDecrypt = this.tryDecrypt(stecker); + return [this.indicator.getPos(), stecker, testDecrypt]; + } + /** * Having set up the Bombe, do the actual attack run. This tries every possible rotor setting * and attempts to logically invalidate them. If it can't, it's added to the list of candidate @@ -592,45 +707,12 @@ export class BombeMachine { } // Energise the test input, follow the current through each scrambler // (and the diagonal board) - this.energiseCount = 0; this.energise(...this.testInput); - // Count the energised outputs - let count = 0; - for (let j=26*this.testRegister; j<26*(1+this.testRegister); j++) { - if (this.wires[j]) { - count++; - } - } - // If it's not all of them, we have a stop - if (count < 26) { - stops += 1; - let stecker; - // The Bombe tells us one stecker pair as well. The input wire and test register we - // started with are hypothesised to be a stecker pair. - if (count === 25) { - // Our steckering hypothesis is wrong. Correct value is the un-energised wire. - for (let j=0; j<26; j++) { - if (!this.wires[26*this.testRegister + j]) { - stecker = [this.testRegister, j]; - break; - } - } - } else if (count === 1) { - // This means our hypothesis for the steckering is correct. - stecker = [this.testRegister, this.testInput[1]]; - } else { - // Unusual, probably indicative of a poor menu. I'm a little unclear on how - // this was really handled, but we'll return it for the moment. - stecker = undefined; - } - const testDecrypt = this.tryDecrypt(stecker); - let steckerStr; - if (stecker !== undefined) { - steckerStr = `${i2a(stecker[0])}${i2a(stecker[1])}`; - } else { - steckerStr = `?? (wire count: ${count})`; - } - result.push([this.indicator.getPos(), steckerStr, testDecrypt]); + + const stop = this.checkStop(); + if (stop !== undefined) { + stops++; + result.push(stop); } // Step all the scramblers // This loop counts how many rotors have reached their starting position (meaning the diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs index 0a083bce..6b6c4d63 100644 --- a/src/core/lib/Enigma.mjs +++ b/src/core/lib/Enigma.mjs @@ -182,7 +182,8 @@ class PairMapBase { } const a = a2i(pair[0]), b = a2i(pair[1]); if (a === b) { - throw new OperationError(`${name}: cannot connect ${pair[0]} to itself`); + // self-stecker + return; } if (this.map.hasOwnProperty(a)) { throw new OperationError(`${name} connects ${pair[0]} more than once`); diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 9ddd4b7b..292017e8 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -23,7 +23,7 @@ class Bombe extends Operation { this.name = "Bombe"; this.module = "Default"; - this.description = "Emulation of the Bombe machine used to attack Enigma.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and one plugboard pair.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops, a large number of incorrect outputs will be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor."; + this.description = "Emulation of the Bombe machine used to attack Enigma.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and known plugboard pairs.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops, a large number of incorrect outputs will be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor.

By default this operation runs the checking machine, a manual process to verify the quality of Bombe stops, on each stop, discarding stops which fail. If you want to see how many times the hardware actually stops for a given input, disable the checking machine."; this.infoURL = "https://wikipedia.org/wiki/Bombe"; this.inputType = "string"; this.outputType = "string"; @@ -66,6 +66,11 @@ class Bombe extends Operation { name: "Crib offset", type: "number", value: 0 + }, + { + name: "Use checking machine", + type: "boolean", + value: true } ]; } @@ -90,6 +95,7 @@ class Bombe extends Operation { const reflectorstr = args[4]; let crib = args[5]; const offset = args[6]; + const check = args[7]; const rotors = []; for (let i=0; i<4; i++) { if (i === 3 && args[i] === "") { @@ -120,9 +126,9 @@ class Bombe extends Operation { } else { update = undefined; } - const bombe = new BombeMachine(rotors, reflector, ciphertext, crib, update); + const bombe = new BombeMachine(rotors, reflector, ciphertext, crib, check, update); const result = bombe.run(); - let msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotor positions are listed left to right and start at the beginning of the crib, and ignore stepping and the ring setting. One stecker pair is determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; + let msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotor positions are listed left to right and start at the beginning of the crib, and ignore stepping and the ring setting. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; for (const [setting, stecker, decrypt] of result) { msg += `Stop: ${setting} (plugboard: ${stecker}): ${decrypt}\n`; } diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs index 6bcd1051..4b3123a4 100644 --- a/src/core/operations/MultipleBombe.mjs +++ b/src/core/operations/MultipleBombe.mjs @@ -163,6 +163,11 @@ class MultipleBombe extends Operation { name: "Crib offset", type: "number", value: 0 + }, + { + name: "Use checking machine", + type: "boolean", + value: true } ]; } @@ -211,7 +216,7 @@ class MultipleBombe extends Operation { const reflectorsStr = args[5]; let crib = args[6]; const offset = args[7]; - // TODO got this far + const check = args[8]; const rotors = []; const fourthRotors = []; const reflectors = []; @@ -279,8 +284,8 @@ class MultipleBombe extends Operation { runRotors.push(rotor4); } if (bombe === undefined) { - bombe = new BombeMachine(runRotors, reflector, ciphertext, crib); - msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotors and rotor positions are listed left to right, ignore stepping and the ring setting, and positions start at the beginning of the crib. One stecker pair is determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; + bombe = new BombeMachine(runRotors, reflector, ciphertext, crib, check); + msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotors and rotor positions are listed left to right, ignore stepping and the ring setting, and positions start at the beginning of the crib. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; } else { bombe.changeRotors(runRotors, reflector); } @@ -290,7 +295,7 @@ class MultipleBombe extends Operation { update(bombe.nLoops, nStops, nRuns / totalRuns); } if (result.length > 0) { - msg += `Rotors: ${runRotors.join(", ")}\nReflector: ${reflector.pairs}\n`; + msg += `\nRotors: ${runRotors.join(", ")}\nReflector: ${reflector.pairs}\n`; for (const [setting, stecker, decrypt] of result) { msg += `Stop: ${setting} (plugboard: ${stecker}): ${decrypt}\n`; } diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index 0a8af93f..fca420d3 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -21,7 +21,7 @@ TestRegister.addTests([ "EKMFLGDQVZNTOWYHXUSPAIBRCJ Date: Fri, 11 Jan 2019 17:44:13 +0000 Subject: [PATCH 031/110] Completed GZIP extraction --- src/core/lib/FileSignatures.mjs | 215 ++++++++++++++++++++++++--- src/core/lib/Stream.mjs | 25 +++- src/core/operations/ExtractFiles.mjs | 5 +- 3 files changed, 222 insertions(+), 23 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index d8d97a2b..b2b1ff06 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1384,6 +1384,7 @@ export function extractRTF(bytes, offset) { export function extractGZIP(bytes, offset) { const stream = new Stream(bytes.slice(offset)); + /* HEADER */ // Skip over signature and compression method @@ -1396,7 +1397,7 @@ export function extractGZIP(bytes, offset) { stream.moveForwardsBy(4); // Read compression flags - const compressionFlags = stream.readInt(1); + stream.readInt(1); // Skip over OS stream.moveForwardsBy(1); @@ -1406,34 +1407,63 @@ export function extractGZIP(bytes, offset) { // Extra fields if (flags & 0x4) { - console.log("Extra fields"); const extraFieldsSize = stream.readInt(2, "le"); stream.moveForwardsby(extraFieldsSize); } // Original filename if (flags & 0x8) { - console.log("Filename"); stream.continueUntil(0x00); stream.moveForwardsBy(1); } // Comment if (flags & 0x10) { - console.log("Comment"); stream.continueUntil(0x00); stream.moveForwardsBy(1); } // Checksum if (flags & 0x2) { - console.log("Checksum"); stream.moveForwardsBy(2); } /* DEFLATE DATA */ + parseDEFLATE(stream); + + + /* FOOTER */ + + // Skip over checksum and size of original uncompressed input + stream.moveForwardsBy(8); + + return stream.carve(); +} + + +/** + * Steps through a DEFLATE stream + * + * @param {Stream} stream + */ +function parseDEFLATE(stream) { + // Construct required Huffman Tables + const fixedLiteralTableLengths = new Uint8Array(288); + for (let i = 0; i < fixedLiteralTableLengths.length; i++) { + fixedLiteralTableLengths[i] = + (i <= 143) ? 8 : + (i <= 255) ? 9 : + (i <= 279) ? 7 : + 8; + } + const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths); + const fixedDistanceTableLengths = new Uint8Array(30).fill(5); + const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths); + const huffmanOrder = new Uint8Array([16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]); + + // Parse DEFLATE data let finalBlock = 0; while (!finalBlock) { @@ -1442,30 +1472,175 @@ export function extractGZIP(bytes, offset) { const blockType = stream.readBits(2); if (blockType === 0) { - // No compression + /* No compression */ + + // Consume the rest of the current byte stream.moveForwardsBy(1); + // Read the block length value const blockLength = stream.readInt(2, "le"); - console.log("No compression. Length: " + blockLength); + // Move to the end of this block stream.moveForwardsBy(2 + blockLength); } else if (blockType === 1) { - // Fixed Huffman - console.log("Fixed Huffman"); + /* Fixed Huffman */ + parseHuffmanBlock(stream, fixedLiteralTable, fixedDistanceTable); } else if (blockType === 2) { - // Dynamic Huffman - console.log("Dynamic Huffman"); + /* Dynamic Huffman */ + + // Read the number of liternal and length codes + const hlit = stream.readBits(5) + 257; + // Read the number of distance codes + const hdist = stream.readBits(5) + 1; + // Read the number of code lengths + const hclen = stream.readBits(4) + 4; + + // Parse code lengths + const codeLengths = new Uint8Array(huffmanOrder.length); + for (let i = 0; i < hclen; i++) { + codeLengths[huffmanOrder[i]] = stream.readBits(3); + } + + // Parse length table + const codeLengthsTable = buildHuffmanTable(codeLengths); + const lengthTable = new Uint8Array(hlit + hdist); + + let code, repeat, prev; + for (let i = 0; i < hlit + hdist;) { + code = readHuffmanCode(stream, codeLengthsTable); + switch (code) { + case 16: + repeat = 3 + stream.readBits(2); + while (repeat--) lengthTable[i++] = prev; + break; + case 17: + repeat = 3 + stream.readBits(3); + while (repeat--) lengthTable[i++] = 0; + prev = 0; + break; + case 18: + repeat = 11 + stream.readBits(7); + while (repeat--) lengthTable[i++] = 0; + prev = 0; + break; + default: + lengthTable[i++] = code; + prev = code; + break; + } + } + + const dynamicLiteralTable = buildHuffmanTable(lengthTable.subarray(0, hlit)); + const dynamicDistanceTable = buildHuffmanTable(lengthTable.subarray(hlit)); + + parseHuffmanBlock(stream, dynamicLiteralTable, dynamicDistanceTable); } else { throw new Error("Invalid block type"); - break; } } - /* FOOTER */ - - // Skip over checksum and size of original uncompressed input - stream.moveForwardsBy(8); - - console.log("Ending at " + stream.position); - - return stream.carve(); + // Consume final byte if it has not been fully consumed yet + if (stream.bitPos > 0) + stream.moveForwardsBy(1); +} + + +/** + * Parses a Huffman Block given the literal and distance tables + * + * @param {Stream} stream + * @param {Uint32Array} litTab + * @param {Uint32Array} distTab + */ +function parseHuffmanBlock(stream, litTab, distTab) { + const lengthExtraTable = new Uint8Array([ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 + ]); + const distanceExtraTable = new Uint8Array([ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 + ]); + + let code; + while ((code = readHuffmanCode(stream, litTab))) { + // console.log("Code: " + code + " (" + Utils.chr(code) + ") " + Utils.bin(code)); + + // End of block + if (code === 256) break; + + // Literal + if (code < 256) continue; + + // Length code + stream.readBits(lengthExtraTable[code - 257]); + + // Dist code + code = readHuffmanCode(stream, distTab); + stream.readBits(distanceExtraTable[code]); + } +} + + +/** + * Builds a Huffman table given the relevant code lengths + * + * @param {Uint8Array} lengths + * @returns {Array} result + * @returns {Uint32Array} result.table + * @returns {number} result.maxCodeLength + * @returns {number} result.minCodeLength + */ +function buildHuffmanTable(lengths) { + const maxCodeLength = Math.max.apply(Math, lengths); + const minCodeLength = Math.min.apply(Math, lengths); + const size = 1 << maxCodeLength; + const table = new Uint32Array(size); + + for (let bitLength = 1, code = 0, skip = 2; bitLength <= maxCodeLength;) { + for (let i = 0; i < lengths.length; i++) { + if (lengths[i] === bitLength) { + let reversed, rtemp, j; + for (reversed = 0, rtemp = code, j = 0; j < bitLength; j++) { + reversed = (reversed << 1) | (rtemp & 1); + rtemp >>= 1; + } + + const value = (bitLength << 16) | i; + for (let j = reversed; j < size; j += skip) { + table[j] = value; + } + + code++; + } + } + + bitLength++; + code <<= 1; + skip <<= 1; + } + + return [table, maxCodeLength, minCodeLength]; +} + + +/** + * Reads the next Huffman code from the stream, given the relevant code table + * + * @param {Stream} stream + * @param {Uint32Array} table + * @returns {number} + */ +function readHuffmanCode(stream, table) { + const [codeTable, maxCodeLength] = table; + + // Read max length + const bitsBuf = stream.readBits(maxCodeLength); + const codeWithLength = codeTable[bitsBuf & ((1 << maxCodeLength) - 1)]; + const codeLength = codeWithLength >>> 16; + + if (codeLength > maxCodeLength) { + throw new Error("Invalid code length: " + codeLength); + } + + stream.moveBackwardsByBits(maxCodeLength - codeLength); + + return codeWithLength & 0xffff; } diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 886d4589..7e82a5eb 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -90,7 +90,6 @@ export default class Stream { return val; } - /** * Reads a number of bits from the buffer. * @@ -194,7 +193,6 @@ export default class Stream { this.bitPos = 0; } - /** * Move backwards through the stream by the specified number of bytes. * @@ -208,6 +206,29 @@ export default class Stream { this.bitPos = 0; } + /** + * Move backwards through the strem by the specified number of bits. + * + * @param {number} numBits + */ + moveBackwardsByBits(numBits) { + if (numBits <= this.bitPos) { + this.bitPos -= numBits; + } else { + if (this.bitPos > 0) { + numBits -= this.bitPos; + this.bitPos = 0; + } + + while (numBits > 0) { + this.moveBackwardsBy(1); + this.bitPos = 8; + this.moveBackwardsByBits(numBits); + numBits -= 8; + } + } + } + /** * Move to a specified position in the stream. * diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index da9d57a9..0f861276 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -46,7 +46,10 @@ class ExtractFiles extends Operation { detectedFiles.forEach(detectedFile => { try { files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); - } catch (err) {} + } catch (err) { + if (err.message.indexOf("No extraction algorithm available") < 0) + throw err; + } }); return files; From 2307325af8ff20523f8a4e5c46503685d9184941 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 11 Jan 2019 17:58:25 +0000 Subject: [PATCH 032/110] Added Zlib extraction --- src/core/lib/FileSignatures.mjs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index b2b1ff06..f19688d1 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -828,7 +828,7 @@ export const FILE_SIGNATURES = { 0: 0x78, 1: [0x1, 0x9c, 0xda, 0x5e] }, - extractor: null + extractor: extractZlib }, { name: "xz compression", @@ -1443,6 +1443,37 @@ export function extractGZIP(bytes, offset) { } +/** + * Zlib extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZlib(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Skip over CMF + stream.moveForwardsBy(1); + + // Read flags + const flags = stream.readInt(1); + + // Skip over preset dictionary checksum + if (flags & 0x20) { + stream.moveForwardsBy(4); + } + + // Parse DEFLATE stream + parseDEFLATE(stream); + + // Skip over final checksum + stream.moveForwardsBy(4); + + return stream.carve(); +} + + /** * Steps through a DEFLATE stream * From d94e8c81879cc9b109bb11d41fd2425dc9de9c2f Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Fri, 11 Jan 2019 18:24:16 +0000 Subject: [PATCH 033/110] Bombe: wording/docs tweaks --- src/core/lib/Bombe.mjs | 2 +- src/core/operations/Bombe.mjs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 4ae0ff7f..81581d67 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -693,7 +693,7 @@ export class BombeMachine { * Having set up the Bombe, do the actual attack run. This tries every possible rotor setting * and attempts to logically invalidate them. If it can't, it's added to the list of candidate * solutions. - * @returns {string[][2]} - list of pairs of candidate rotor setting, and calculated stecker pair + * @returns {string[][3]} - list of 3-tuples of candidate rotor setting, plugboard settings, and decryption preview */ run() { let stops = 0; diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 292017e8..3c29cd8c 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -35,19 +35,19 @@ class Bombe extends Operation { defaultIndex: 2 }, { - name: "2nd rotor", + name: "2nd (middle) rotor", type: "editableOption", value: ROTORS, defaultIndex: 1 }, { - name: "3rd rotor", + name: "3rd (left-hand) rotor", type: "editableOption", value: ROTORS, defaultIndex: 0 }, { - name: "4th rotor", + name: "4th (left-most, only some models) rotor", type: "editableOption", value: ROTORS_OPTIONAL, defaultIndex: 10 From 49f5c94a750c8a11c7a19f29efa3265549df3d8f Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Sat, 12 Jan 2019 01:10:47 +0000 Subject: [PATCH 034/110] Bombe: further optimisation --- src/core/lib/Bombe.mjs | 63 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 81581d67..1e3c5b3e 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -158,27 +158,6 @@ class SharedScrambler { } } - /** - * Get the fully cached result, if present. - * @param {number} pos - Position of the fast rotor - * @param {number} i - Letter - * @returns {number|undefined} - undefined if not cached - */ - fullTransform(pos, i) { - return this.higherCache[pos][i]; - } - - /** - * Add a value to the full result cache. - * @param {number} pos - Position of the fast rotor - * @param {number} i - Letter - * @param {number} val - Transformed letter - */ - addCache(pos, i, val) { - this.higherCache[pos][i] = val; - this.higherCache[pos][val] = i; - } - /** * Map a letter through this (partial) scrambler. * @param {number} i - The letter @@ -209,6 +188,9 @@ class Scrambler { this.changeRotor(rotor); this.end1 = end1; this.end2 = end2; + // For efficiency reasons, we pull the relevant shared cache from the baseScrambler into + // this object - this saves us a few pointer dereferences + this.cache = this.baseScrambler.higherCache[pos]; } /** @@ -233,6 +215,7 @@ class Scrambler { // simplifies caching the state of the majority of the scramblers. The results are the // same, just in a slightly different order. this.rotor.step(); + this.cache = this.baseScrambler.higherCache[this.rotor.pos]; } @@ -243,14 +226,15 @@ class Scrambler { */ transform(i) { let letter = i; - const cached = this.baseScrambler.fullTransform(this.rotor.pos, i); + const cached = this.cache[i]; if (cached !== undefined) { return cached; } letter = this.rotor.transform(letter); letter = this.baseScrambler.transform(letter); letter = this.rotor.revTransform(letter); - this.baseScrambler.addCache(this.rotor.pos, i, letter); + this.cache[i] = letter; + this.cache[letter] = i; return letter; } @@ -513,12 +497,26 @@ export class BombeMachine { // both. const idxPair = 26*j + i; this.wires[idxPair] = true; + if (i === this.testRegister || j === this.testRegister) { + this.energiseCount++; + if (this.energiseCount === 26) { + // no point continuing, bail out + return; + } + } for (let k=0; k Date: Sat, 12 Jan 2019 01:35:24 +0000 Subject: [PATCH 035/110] Bombe: tweaks Twiddle the default rotor sets a bit. Add a time remaining estimate for the multibombe. --- src/core/lib/Enigma.mjs | 8 ++++---- src/core/operations/Bombe.mjs | 4 ++-- src/core/operations/Enigma.mjs | 6 +++--- src/core/operations/MultipleBombe.mjs | 18 ++++++++++++------ 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs index 6b6c4d63..1ed0ea2b 100644 --- a/src/core/lib/Enigma.mjs +++ b/src/core/lib/Enigma.mjs @@ -22,14 +22,14 @@ export const ROTORS = [ {name: "VI", value: "JPGVOUMFYQBENHZRDKASXLICTW 0) { msg += `\nRotors: ${runRotors.join(", ")}\nReflector: ${reflector.pairs}\n`; From eee92aa1aaf2156ba188e450a73efa4704048dde Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Sat, 12 Jan 2019 12:56:21 +0000 Subject: [PATCH 036/110] Bombe: fix some outdated docs --- src/core/lib/Bombe.mjs | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 1e3c5b3e..03413350 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -19,6 +19,7 @@ import {Rotor, Plugboard, a2i, i2a} from "./Enigma"; class CopyRotor extends Rotor { /** * Return a copy of this Rotor. + * @returns {Object} */ copy() { const clone = { @@ -204,10 +205,9 @@ class Scrambler { } /** - * Step the rotors forward. + * Step the rotor forward. * - * All nodes in the Bombe step in sync. - * @param {number} n - How many rotors to step + * The base SharedScrambler needs to be instructed to step separately. */ step() { // The Bombe steps the slowest rotor on an actual Enigma fastest, for reasons. @@ -284,6 +284,7 @@ export class BombeMachine { * @param {Object} reflector - Reflector object * @param {string} ciphertext - The ciphertext to attack * @param {string} crib - Known plaintext for this ciphertext + * @param {boolean} check - Whether to use the checking machine * @param {function} update - Function to call to send status updates (optional) */ constructor(rotors, reflector, ciphertext, crib, check, update=undefined) { @@ -383,7 +384,7 @@ export class BombeMachine { /** * If we have a way of sending status messages, do so. - * @param {string} msg - Message to send. + * @param {...*} msg - Message to send. */ update(...msg) { if (this.updateFn !== undefined) { @@ -485,7 +486,8 @@ export class BombeMachine { /** * Bombe electrical simulation. Energise a wire. For all connected wires (both via the diagonal * board and via the scramblers), energise them too, recursively. - * @param {number[2]} i - Bombe state wire + * @param {number} i - Bombe wire bundle + * @param {number} j - Bombe stecker hypothesis wire within bundle */ energise(i, j) { const idx = 26*i + j; @@ -535,33 +537,13 @@ export class BombeMachine { } } - /** - * Single-pair steckering. Used for trial decryption rather than building a whole plugboard - * object for one pair - * @param {number[2]} stecker - Known stecker pair. - * @param {number} x - Letter to transform. - * @result number - */ - singleStecker(stecker, x) { - if (stecker === undefined) { - return x; - } - if (x === stecker[0]) { - return stecker[1]; - } - if (x === stecker[1]) { - return stecker[0]; - } - return x; - } - /** * Trial decryption at the current setting. * Used after we get a stop. * This applies the detected stecker pair if we have one. It does not handle the other * steckering or stepping (which is why we limit it to 26 characters, since it's guaranteed to * be wrong after that anyway). - * @param {number[2]} stecker - Known stecker pair. + * @param {string} stecker - Known stecker spec string. * @returns {string} */ tryDecrypt(stecker) { From ffc4b0a0a8ef1576bf0b292e184aed5dace1994c Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Mon, 14 Jan 2019 17:15:54 +0000 Subject: [PATCH 037/110] Bombe: lol --- src/core/operations/Bombe.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 9014fd3a..34a94e53 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -50,7 +50,7 @@ class Bombe extends Operation { name: "4th (left-most, only some models) rotor", type: "editableOption", value: ROTORS_FOURTH, - defaultIndex: 10 + defaultIndex: 0 }, { name: "Reflector", From cd2c8078c8f42365eb4f0723831c1e62c21d9914 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Mon, 14 Jan 2019 18:55:10 +0000 Subject: [PATCH 038/110] Added ELF extractor. You can now specific which categories to search for in file type operations. --- src/core/lib/FileSignatures.mjs | 46 +++++++++++++++++++- src/core/lib/FileType.mjs | 26 ++++++++--- src/core/operations/DetectFileType.mjs | 17 +++++++- src/core/operations/ExtractFiles.mjs | 18 ++++++-- src/core/operations/ScanForEmbeddedFiles.mjs | 42 +++++++----------- 5 files changed, 111 insertions(+), 38 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index f19688d1..93247413 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -678,7 +678,7 @@ export const FILE_SIGNATURES = { 2: 0x4c, 3: 0x46 }, - extractor: null + extractor: extractELF }, { name: "Adobe Flash", @@ -1474,6 +1474,50 @@ export function extractZlib(bytes, offset) { } +/** + * ELF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractELF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Skip over magic number + stream.moveForwardsBy(4); + + // Read architecture (x86 == 1, x64 == 2) + const x86 = stream.readInt(1) === 1; + + // Read endianness (1 == little, 2 == big) + const endian = stream.readInt(1) === 1 ? "le" : "be"; + + // Skip over header values + stream.moveForwardsBy(x86 ? 26 : 34); + + // Read section header table offset + const shoff = x86 ? stream.readInt(4, endian) : stream.readInt(8, endian); + + // Skip over flags, header size and program header size and entries + stream.moveForwardsBy(10); + + // Read section header table entry size + const shentsize = stream.readInt(2, endian); + + // Read number of entries in the section header table + const shnum = stream.readInt(2, endian); + + // Jump to section header table + stream.moveTo(shoff); + + // Move past each section header + stream.moveForwardsBy(shentsize * shnum); + + return stream.carve(); +} + + /** * Steps through a DEFLATE stream * diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 202b54d9..e5d990d9 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -75,22 +75,29 @@ function bytesMatch(sig, buf, offset=0) { * extension and mime type. * * @param {Uint8Array} buf + * @param {string[]} [categories=All] - Which categories of file to look for * @returns {Object[]} types * @returns {string} type.name - Name of file type * @returns {string} type.ext - File extension * @returns {string} type.mime - Mime type * @returns {string} [type.desc] - Description */ -export function detectFileType(buf) { +export function detectFileType(buf, categories=Object.keys(FILE_SIGNATURES)) { if (!(buf && buf.length > 1)) { return []; } const matchingFiles = []; + const signatures = {}; - // TODO allow user to select which categories to check for (const cat in FILE_SIGNATURES) { - const category = FILE_SIGNATURES[cat]; + if (categories.includes(cat)) { + signatures[cat] = FILE_SIGNATURES[cat]; + } + } + + for (const cat in signatures) { + const category = signatures[cat]; category.forEach(filetype => { if (signatureMatches(filetype.signature, buf)) { @@ -107,6 +114,7 @@ export function detectFileType(buf) { * the extensions and mime types. * * @param {Uint8Array} buf + * @param {string[]} [categories=All] - Which categories of file to look for * @returns {Object[]} foundFiles * @returns {number} foundFiles.offset - The position in the buffer at which this file was found * @returns {Object} foundFiles.fileDetails @@ -115,16 +123,22 @@ export function detectFileType(buf) { * @returns {string} foundFiles.fileDetails.mime - Mime type * @returns {string} [foundFiles.fileDetails.desc] - Description */ -export function scanForFileTypes(buf) { +export function scanForFileTypes(buf, categories=Object.keys(FILE_SIGNATURES)) { if (!(buf && buf.length > 1)) { return []; } const foundFiles = []; + const signatures = {}; - // TODO allow user to select which categories to check for (const cat in FILE_SIGNATURES) { - const category = FILE_SIGNATURES[cat]; + if (categories.includes(cat)) { + signatures[cat] = FILE_SIGNATURES[cat]; + } + } + + for (const cat in signatures) { + const category = signatures[cat]; for (let i = 0; i < category.length; i++) { const filetype = category[i]; diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index 55db5edf..2321cee8 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -6,6 +6,7 @@ import Operation from "../Operation"; import {detectFileType} from "../lib/FileType"; +import {FILE_SIGNATURES} from "../lib/FileSignatures"; /** * Detect File Type operation @@ -24,7 +25,13 @@ class DetectFileType extends Operation { this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures"; this.inputType = "ArrayBuffer"; this.outputType = "string"; - this.args = []; + this.args = Object.keys(FILE_SIGNATURES).map(cat => { + return { + name: cat, + type: "boolean", + value: true + }; + }); } /** @@ -34,7 +41,13 @@ class DetectFileType extends Operation { */ run(input, args) { const data = new Uint8Array(input), - types = detectFileType(data); + categories = []; + + args.forEach((cat, i) => { + if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); + }); + + const types = detectFileType(data, categories); if (!types.length) { return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?"; diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index 0f861276..f172d926 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -8,6 +8,7 @@ import Operation from "../Operation"; // import OperationError from "../errors/OperationError"; import Utils from "../Utils"; import {scanForFileTypes, extractFile} from "../lib/FileType"; +import {FILE_SIGNATURES} from "../lib/FileSignatures"; /** * Extract Files operation @@ -27,7 +28,13 @@ class ExtractFiles extends Operation { this.inputType = "ArrayBuffer"; this.outputType = "List"; this.presentType = "html"; - this.args = []; + this.args = Object.keys(FILE_SIGNATURES).map(cat => { + return { + name: cat, + type: "boolean", + value: cat === "Miscellaneous" ? false : true + }; + }); } /** @@ -36,10 +43,15 @@ class ExtractFiles extends Operation { * @returns {List} */ run(input, args) { - const bytes = new Uint8Array(input); + const bytes = new Uint8Array(input), + categories = []; + + args.forEach((cat, i) => { + if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); + }); // Scan for embedded files - const detectedFiles = scanForFileTypes(bytes); + const detectedFiles = scanForFileTypes(bytes, categories); // Extract each file that we support const files = []; diff --git a/src/core/operations/ScanForEmbeddedFiles.mjs b/src/core/operations/ScanForEmbeddedFiles.mjs index a0465e83..ae88134f 100644 --- a/src/core/operations/ScanForEmbeddedFiles.mjs +++ b/src/core/operations/ScanForEmbeddedFiles.mjs @@ -7,6 +7,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; import {scanForFileTypes} from "../lib/FileType"; +import {FILE_SIGNATURES} from "../lib/FileSignatures"; /** * Scan for Embedded Files operation @@ -25,13 +26,13 @@ class ScanForEmbeddedFiles extends Operation { this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures"; this.inputType = "ArrayBuffer"; this.outputType = "string"; - this.args = [ - { - "name": "Ignore common byte sequences", - "type": "boolean", - "value": true - } - ]; + this.args = Object.keys(FILE_SIGNATURES).map(cat => { + return { + name: cat, + type: "boolean", + value: cat === "Miscellaneous" ? false : true + }; + }); } /** @@ -41,21 +42,18 @@ class ScanForEmbeddedFiles extends Operation { */ run(input, args) { let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n", - numFound = 0, - numCommonFound = 0; - const ignoreCommon = args[0], - commonExts = ["ttf", "utf16le", ""], - data = new Uint8Array(input), - types = scanForFileTypes(data); + numFound = 0; + const categories = [], + data = new Uint8Array(input); + args.forEach((cat, i) => { + if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); + }); + + const types = scanForFileTypes(data, categories); if (types.length) { types.forEach(type => { - if (ignoreCommon && commonExts.indexOf(type.fileDetails.extension) > -1) { - numCommonFound++; - return; - } - numFound++; output += "\nOffset " + type.offset + " (0x" + Utils.hex(type.offset) + "):\n" + " File extension: " + type.fileDetails.extension + "\n" + @@ -71,14 +69,6 @@ class ScanForEmbeddedFiles extends Operation { output += "\nNo embedded files were found."; } - if (numCommonFound > 0) { - output += "\n\n" + numCommonFound; - output += numCommonFound === 1 ? - " file type was detected that has a common byte sequence. This is likely to be a false positive." : - " file types were detected that have common byte sequences. These are likely to be false positives."; - output += " Run this operation with the 'Ignore common byte sequences' option unchecked to see details."; - } - return output; } From 02b9dbdee962dfff2df1a6c4c61a62e9414415d9 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 15 Jan 2019 19:03:17 +0000 Subject: [PATCH 039/110] Replaced loading animation with animated Bombe SVG --- src/web/App.mjs | 2 + src/web/OutputWaiter.mjs | 41 ++++- src/web/html/index.html | 10 +- src/web/static/images/bombe.svg | 261 +++++++++++++++++++++++++++++ src/web/stylesheets/layout/_io.css | 32 ++-- src/web/stylesheets/preloader.css | 69 +++----- 6 files changed, 349 insertions(+), 66 deletions(-) create mode 100644 src/web/static/images/bombe.svg diff --git a/src/web/App.mjs b/src/web/App.mjs index 1dab16e6..453fba22 100755 --- a/src/web/App.mjs +++ b/src/web/App.mjs @@ -51,10 +51,12 @@ class App { */ setup() { document.dispatchEvent(this.manager.appstart); + this.initialiseSplitter(); this.loadLocalStorage(); this.populateOperationsList(); this.manager.setup(); + this.manager.output.saveBombe(); this.resetLayout(); this.setCompileMessage(); diff --git a/src/web/OutputWaiter.mjs b/src/web/OutputWaiter.mjs index 7203a16f..39d6e51b 100755 --- a/src/web/OutputWaiter.mjs +++ b/src/web/OutputWaiter.mjs @@ -334,24 +334,55 @@ class OutputWaiter { /** - * Shows or hides the loading icon. + * Save bombe object before it is removed so that it can be used later + */ + saveBombe() { + this.bombeEl = document.getElementById("bombe").cloneNode(); + this.bombeEl.setAttribute("width", "100%"); + this.bombeEl.setAttribute("height", "100%"); + } + + + /** + * Shows or hides the output loading screen. + * The animated Bombe SVG, whilst quite aesthetically pleasing, is reasonably CPU + * intensive, so we remove it from the DOM when not in use. We only show it if the + * recipe is taking longer than 200ms. We add it to the DOM just before that so that + * it is ready to fade in without stuttering. * - * @param {boolean} value + * @param {boolean} value - true == show loader */ toggleLoader(value) { + clearTimeout(this.appendBombeTimeout); + clearTimeout(this.outputLoaderTimeout); + const outputLoader = document.getElementById("output-loader"), - outputElement = document.getElementById("output-text"); + outputElement = document.getElementById("output-text"), + loader = outputLoader.querySelector(".loader"); if (value) { this.manager.controls.hideStaleIndicator(); - this.bakingStatusTimeout = setTimeout(function() { + + // Start a timer to add the Bombe to the DOM just before we make it + // visible so that there is no stuttering + this.appendBombeTimeout = setTimeout(function() { + loader.appendChild(this.bombeEl); + }.bind(this), 150); + + // Show the loading screen + this.outputLoaderTimeout = setTimeout(function() { outputElement.disabled = true; outputLoader.style.visibility = "visible"; outputLoader.style.opacity = 1; this.manager.controls.toggleBakeButtonFunction(true); }.bind(this), 200); } else { - clearTimeout(this.bakingStatusTimeout); + // Remove the Bombe from the DOM to save resources + this.outputLoaderTimeout = setTimeout(function () { + try { + loader.removeChild(this.bombeEl); + } catch (err) {} + }.bind(this), 500); outputElement.disabled = false; outputLoader.style.opacity = 0; outputLoader.style.visibility = "hidden"; diff --git a/src/web/html/index.html b/src/web/html/index.html index f03590ab..a1f915b6 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -81,7 +81,11 @@ if (!el.classList.contains("loading")) el.classList.add("loading"); // Causes CSS transition on first message el.innerHTML = msg; - } catch (err) {} // Ignore errors if DOM not yet ready + } catch (err) { + // This error was likely caused by the DOM not being ready yet, + // so we wait another second and then try again. + setTimeout(changeLoadingMsg, 1000); + } } changeLoadingMsg(); @@ -138,7 +142,9 @@
-
+
+ +
diff --git a/src/web/static/images/bombe.svg b/src/web/static/images/bombe.svg new file mode 100644 index 00000000..40857bdf --- /dev/null +++ b/src/web/static/images/bombe.svg @@ -0,0 +1,261 @@ + + + + + + + + + + + + + + Z + Y + X + W + V + U + T + S + R + Q + P + O + N + M + L + K + J + I + H + G + F + E + D + C + B + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/web/stylesheets/layout/_io.css b/src/web/stylesheets/layout/_io.css index 5b0433f6..0a1e4ec4 100755 --- a/src/web/stylesheets/layout/_io.css +++ b/src/web/stylesheets/layout/_io.css @@ -73,6 +73,28 @@ background-color: var(--primary-background-colour); visibility: hidden; opacity: 0; + display: flex; + justify-content: center; + align-items: center; + + transition: all 0.5s ease; +} + +#output-loader .loader { + width: 60%; + height: 60%; + left: unset; + top: 10%; +} + +#output-loader .loading-msg { + opacity: 1; + font-family: var(--primary-font-family); + line-height: var(--primary-line-height); + color: var(--primary-font-colour); + left: unset; + top: 30%; + position: relative; transition: all 0.5s ease; } @@ -138,16 +160,6 @@ margin-bottom: 5px; } -#output-loader .loading-msg { - opacity: 1; - font-family: var(--primary-font-family); - line-height: var(--primary-line-height); - color: var(--primary-font-colour); - top: 50%; - - transition: all 0.5s ease; -} - #magic { opacity: 1; visibility: visibile; diff --git a/src/web/stylesheets/preloader.css b/src/web/stylesheets/preloader.css index 702d04a6..bce0cd03 100755 --- a/src/web/stylesheets/preloader.css +++ b/src/web/stylesheets/preloader.css @@ -16,57 +16,28 @@ background-color: var(--secondary-border-colour); } +#loader-wrapper div { + animation: fadeIn 1s ease-in 0s; +} + .loader { display: block; - position: relative; - left: 50%; - top: 50%; - width: 150px; - height: 150px; - margin: -75px 0 0 -75px; - - border: 3px solid transparent; - border-top-color: #3498db; - border-radius: 50%; - - animation: spin 2s linear infinite; -} - -.loader:before, -.loader:after { - content: ""; position: absolute; - border: 3px solid transparent; - border-radius: 50%; -} - -.loader:before { - top: 5px; - left: 5px; - right: 5px; - bottom: 5px; - border-top-color: #e74c3c; - animation: spin 3s linear infinite; -} - -.loader:after { - top: 13px; - left: 13px; - right: 13px; - bottom: 13px; - border-top-color: #f9c922; - animation: spin 1.5s linear infinite; + left: calc(50% - 200px); + top: calc(50% - 160px); + width: 400px; + height: 260px; } .loading-msg { display: block; - position: relative; + position: absolute; width: 400px; left: calc(50% - 200px); - top: calc(50% + 50px); + top: calc(50% + 110px); text-align: center; - margin-top: 50px; opacity: 0; + font-size: 18px; } .loading-msg.loading { @@ -145,18 +116,18 @@ /* Animations */ -@keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(360deg); - } -} - @keyframes bump { from { opacity: 0; transform: translate3d(0, 200px, 0); } } + +@keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} From 87e956fe7df28bb82c0c57e01044154d0eea9996 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 16 Jan 2019 12:29:34 +0000 Subject: [PATCH 040/110] Added old loading icon back for initial page load. --- src/web/App.mjs | 3 ++ src/web/OutputWaiter.mjs | 13 ++++--- src/web/RecipeWaiter.mjs | 3 +- src/web/html/index.html | 8 ++--- src/web/static/images/bombe.svg | 4 +-- src/web/stylesheets/layout/_io.css | 6 ++-- src/web/stylesheets/preloader.css | 58 ++++++++++++++++++++++++------ 7 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/web/App.mjs b/src/web/App.mjs index 453fba22..e203b85c 100755 --- a/src/web/App.mjs +++ b/src/web/App.mjs @@ -124,6 +124,9 @@ class App { // Reset attemptHighlight flag this.options.attemptHighlight = true; + // Remove all current indicators + this.manager.recipe.updateBreakpointIndicator(false); + this.manager.worker.bake( this.getInput(), // The user's input this.getRecipeConfig(), // The configuration of the recipe diff --git a/src/web/OutputWaiter.mjs b/src/web/OutputWaiter.mjs index 39d6e51b..5e7ae7e8 100755 --- a/src/web/OutputWaiter.mjs +++ b/src/web/OutputWaiter.mjs @@ -334,12 +334,11 @@ class OutputWaiter { /** - * Save bombe object before it is removed so that it can be used later + * Save bombe object then remove it from the DOM so that it does not cause performance issues. */ saveBombe() { - this.bombeEl = document.getElementById("bombe").cloneNode(); - this.bombeEl.setAttribute("width", "100%"); - this.bombeEl.setAttribute("height", "100%"); + this.bombeEl = document.getElementById("bombe"); + this.bombeEl.parentNode.removeChild(this.bombeEl); } @@ -358,7 +357,7 @@ class OutputWaiter { const outputLoader = document.getElementById("output-loader"), outputElement = document.getElementById("output-text"), - loader = outputLoader.querySelector(".loader"); + animation = document.getElementById("output-loader-animation"); if (value) { this.manager.controls.hideStaleIndicator(); @@ -366,7 +365,7 @@ class OutputWaiter { // Start a timer to add the Bombe to the DOM just before we make it // visible so that there is no stuttering this.appendBombeTimeout = setTimeout(function() { - loader.appendChild(this.bombeEl); + animation.appendChild(this.bombeEl); }.bind(this), 150); // Show the loading screen @@ -380,7 +379,7 @@ class OutputWaiter { // Remove the Bombe from the DOM to save resources this.outputLoaderTimeout = setTimeout(function () { try { - loader.removeChild(this.bombeEl); + animation.removeChild(this.bombeEl); } catch (err) {} }.bind(this), 500); outputElement.disabled = false; diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index b913fede..a8326b27 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -340,10 +340,11 @@ class RecipeWaiter { /** * Moves or removes the breakpoint indicator in the recipe based on the position. * - * @param {number} position + * @param {number|boolean} position - If boolean, turn off all indicators */ updateBreakpointIndicator(position) { const operations = document.querySelectorAll("#rec-list li.operation"); + if (typeof position === "boolean") position = operations.length; for (let i = 0; i < operations.length; i++) { if (i === position) { operations[i].classList.add("break"); diff --git a/src/web/html/index.html b/src/web/html/index.html index a1f915b6..478d2bb3 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -142,9 +142,7 @@
-
- -
+
@@ -321,7 +319,9 @@
-
+
+ +
diff --git a/src/web/static/images/bombe.svg b/src/web/static/images/bombe.svg index 40857bdf..1fdca842 100644 --- a/src/web/static/images/bombe.svg +++ b/src/web/static/images/bombe.svg @@ -56,13 +56,13 @@ - + diff --git a/src/web/stylesheets/layout/_io.css b/src/web/stylesheets/layout/_io.css index 0a1e4ec4..d4af353f 100755 --- a/src/web/stylesheets/layout/_io.css +++ b/src/web/stylesheets/layout/_io.css @@ -80,11 +80,13 @@ transition: all 0.5s ease; } -#output-loader .loader { +#output-loader-animation { + display: block; + position: absolute; width: 60%; height: 60%; - left: unset; top: 10%; + transition: all 0.5s ease; } #output-loader .loading-msg { diff --git a/src/web/stylesheets/preloader.css b/src/web/stylesheets/preloader.css index bce0cd03..690fe5c1 100755 --- a/src/web/stylesheets/preloader.css +++ b/src/web/stylesheets/preloader.css @@ -16,25 +16,54 @@ background-color: var(--secondary-border-colour); } -#loader-wrapper div { - animation: fadeIn 1s ease-in 0s; -} - .loader { display: block; + position: relative; + left: 50%; + top: 50%; + width: 150px; + height: 150px; + margin: -75px 0 0 -75px; + + border: 3px solid transparent; + border-top-color: #3498db; + border-radius: 50%; + + animation: spin 2s linear infinite; +} + +.loader:before, +.loader:after { + content: ""; position: absolute; - left: calc(50% - 200px); - top: calc(50% - 160px); - width: 400px; - height: 260px; + border: 3px solid transparent; + border-radius: 50%; +} + +.loader:before { + top: 5px; + left: 5px; + right: 5px; + bottom: 5px; + border-top-color: #e74c3c; + animation: spin 3s linear infinite; +} + +.loader:after { + top: 13px; + left: 13px; + right: 13px; + bottom: 13px; + border-top-color: #f9c922; + animation: spin 1.5s linear infinite; } .loading-msg { display: block; - position: absolute; + position: relative; width: 400px; left: calc(50% - 200px); - top: calc(50% + 110px); + top: calc(50% + 50px); text-align: center; opacity: 0; font-size: 18px; @@ -116,6 +145,15 @@ /* Animations */ +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + @keyframes bump { from { opacity: 0; From 220053c0444c6dfbcc785e61fe6188a5f937566a Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 7 Feb 2019 18:10:16 +0000 Subject: [PATCH 041/110] Typex: add ring setting --- src/core/lib/Typex.mjs | 9 ++++----- src/core/operations/Typex.mjs | 37 +++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/core/lib/Typex.mjs b/src/core/lib/Typex.mjs index df6e646b..b4cf297c 100644 --- a/src/core/lib/Typex.mjs +++ b/src/core/lib/Typex.mjs @@ -148,14 +148,13 @@ export class Rotor extends Enigma.Rotor { * @param {string} wiring - A 26 character string of the wiring order. * @param {string} steps - A 0..26 character string of stepping points. * @param {bool} reversed - Whether to reverse the rotor. + * @param {char} ringSetting - Ring setting of the rotor. * @param {char} initialPosition - The initial position of the rotor. */ - constructor(wiring, steps, reversed, initialPos) { - let initialPosMod = initialPos; + constructor(wiring, steps, reversed, ringSetting, initialPos) { let wiringMod = wiring; if (reversed) { - initialPosMod = Enigma.i2a(Utils.mod(26 - Enigma.a2i(initialPos), 26)); - const outMap = new Array(26).fill(); + const outMap = new Array(26); for (let i=0; i<26; i++) { // wiring[i] is the original output // Enigma.LETTERS[i] is the original input @@ -165,7 +164,7 @@ export class Rotor extends Enigma.Rotor { } wiringMod = outMap.join(""); } - super(wiringMod, steps, "A", initialPosMod); + super(wiringMod, steps, ringSetting, initialPos); } } diff --git a/src/core/operations/Typex.mjs b/src/core/operations/Typex.mjs index 79468645..504cb891 100644 --- a/src/core/operations/Typex.mjs +++ b/src/core/operations/Typex.mjs @@ -39,6 +39,11 @@ class Typex extends Operation { type: "boolean", value: false }, + { + name: "1st rotor ring setting", + type: "option", + value: LETTERS + }, { name: "1st rotor initial value", type: "option", @@ -55,6 +60,11 @@ class Typex extends Operation { type: "boolean", value: false }, + { + name: "2nd rotor ring setting", + type: "option", + value: LETTERS + }, { name: "2nd rotor initial value", type: "option", @@ -71,6 +81,11 @@ class Typex extends Operation { type: "boolean", value: false }, + { + name: "3rd rotor ring setting", + type: "option", + value: LETTERS + }, { name: "3rd rotor initial value", type: "option", @@ -87,6 +102,11 @@ class Typex extends Operation { type: "boolean", value: false }, + { + name: "4th rotor ring setting", + type: "option", + value: LETTERS + }, { name: "4th rotor initial value", type: "option", @@ -103,6 +123,11 @@ class Typex extends Operation { type: "boolean", value: false }, + { + name: "5th rotor ring setting", + type: "option", + value: LETTERS + }, { name: "5th rotor initial value", type: "option", @@ -156,14 +181,14 @@ class Typex extends Operation { * @returns {string} */ run(input, args) { - const reflectorstr = args[15]; - const plugboardstr = args[16]; - const typexKeyboard = args[17]; - const removeOther = args[18]; + const reflectorstr = args[20]; + const plugboardstr = args[21]; + const typexKeyboard = args[22]; + const removeOther = args[23]; const rotors = []; for (let i=0; i<5; i++) { - const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*3]); - rotors.push(new Rotor(rotorwiring, rotorsteps, args[i*3 + 1], args[i*3+2])); + const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*4]); + rotors.push(new Rotor(rotorwiring, rotorsteps, args[i*4 + 1], args[i*4+2], args[i*4+3])); } const reflector = new Reflector(reflectorstr); let plugboardstrMod = plugboardstr; From 53226c105069b97369e9a81ef23da2c88e0c9441 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 8 Feb 2019 00:59:56 +0000 Subject: [PATCH 042/110] Added populateMultiOption ingredient type --- .gitignore | 1 + src/core/operations/MultipleBombe.mjs | 94 +++++++----------------- src/web/HTMLIngredient.mjs | 49 +++++++++++- src/web/RecipeWaiter.mjs | 10 +++ tests/operations/tests/MultipleBombe.mjs | 14 ++-- 5 files changed, 92 insertions(+), 76 deletions(-) diff --git a/.gitignore b/.gitignore index edbcf679..b5aad5d0 100755 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/* !docs/*.ico .vscode .*.swp +.DS_Store src/core/config/modules/* src/core/config/OperationConfig.json src/core/operations/index.mjs diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs index dac7a334..a453ca34 100644 --- a/src/core/operations/MultipleBombe.mjs +++ b/src/core/operations/MultipleBombe.mjs @@ -60,95 +60,57 @@ class MultipleBombe extends Operation { this.args = [ { "name": "Standard Enigmas", - "type": "populateOption", + "type": "populateMultiOption", "value": [ { name: "German Service Enigma (First - 3 rotor)", - value: rotorsFormat(ROTORS, 0, 5) + value: [ + rotorsFormat(ROTORS, 0, 5), + "", + rotorsFormat(REFLECTORS, 0, 1) + ] }, { name: "German Service Enigma (Second - 3 rotor)", - value: rotorsFormat(ROTORS, 0, 8) + value: [ + rotorsFormat(ROTORS, 0, 8), + "", + rotorsFormat(REFLECTORS, 0, 2) + ] }, { name: "German Service Enigma (Third - 4 rotor)", - value: rotorsFormat(ROTORS, 0, 8) + value: [ + rotorsFormat(ROTORS, 0, 8), + rotorsFormat(ROTORS_FOURTH, 1, 2), + rotorsFormat(REFLECTORS, 2, 3) + ] }, { name: "German Service Enigma (Fourth - 4 rotor)", - value: rotorsFormat(ROTORS, 0, 8) + value: [ + rotorsFormat(ROTORS, 0, 8), + rotorsFormat(ROTORS_FOURTH, 1, 3), + rotorsFormat(REFLECTORS, 2, 4) + ] }, { name: "User defined", - value: "" + value: ["", "", ""] }, ], - "target": 1 + "target": [1, 2, 3] }, { name: "Main rotors", type: "text", value: "" }, - { - "name": "Standard Enigmas", - "type": "populateOption", - "value": [ - { - name: "German Service Enigma (First - 3 rotor)", - value: "" - }, - { - name: "German Service Enigma (Second - 3 rotor)", - value: "" - }, - { - name: "German Service Enigma (Third - 4 rotor)", - value: rotorsFormat(ROTORS_FOURTH, 1, 2) - }, - { - name: "German Service Enigma (Fourth - 4 rotor)", - value: rotorsFormat(ROTORS_FOURTH, 1, 3) - }, - { - name: "User defined", - value: "" - }, - ], - "target": 3 - }, { name: "4th rotor", type: "text", value: "" }, - { - "name": "Standard Enigmas", - "type": "populateOption", - "value": [ - { - name: "German Service Enigma (First - 3 rotor)", - value: rotorsFormat(REFLECTORS, 0, 1) - }, - { - name: "German Service Enigma (Second - 3 rotor)", - value: rotorsFormat(REFLECTORS, 0, 2) - }, - { - name: "German Service Enigma (Third - 4 rotor)", - value: rotorsFormat(REFLECTORS, 2, 3) - }, - { - name: "German Service Enigma (Fourth - 4 rotor)", - value: rotorsFormat(REFLECTORS, 2, 4) - }, - { - name: "User defined", - value: "" - }, - ], - "target": 5 - }, { name: "Reflectors", type: "text", @@ -217,11 +179,11 @@ class MultipleBombe extends Operation { */ run(input, args) { const mainRotorsStr = args[1]; - const fourthRotorsStr = args[3]; - const reflectorsStr = args[5]; - let crib = args[6]; - const offset = args[7]; - const check = args[8]; + const fourthRotorsStr = args[2]; + const reflectorsStr = args[3]; + let crib = args[4]; + const offset = args[5]; + const check = args[6]; const rotors = []; const fourthRotors = []; const reflectors = []; diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index bb01d7de..c7c024fb 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -39,7 +39,7 @@ class HTMLIngredient { */ toHtml() { let html = "", - i, m; + i, m, eventFn; switch (this.type) { case "string": @@ -142,10 +142,11 @@ class HTMLIngredient { `; break; case "populateOption": + case "populateMultiOption": html += `
${this.hint ? "" + this.hint + "" : ""}
`; - this.manager.addDynamicListener("#" + this.id, "change", this.populateOptionChange, this); + eventFn = this.type === "populateMultiOption" ? + this.populateMultiOptionChange : + this.populateOptionChange; + this.manager.addDynamicListener("#" + this.id, "change", eventFn, this); break; case "editableOption": html += `
@@ -248,6 +255,9 @@ class HTMLIngredient { * @param {event} e */ populateOptionChange(e) { + e.preventDefault(); + e.stopPropagation(); + const el = e.target; const op = el.parentNode.parentNode; const target = op.querySelectorAll(".arg")[this.target]; @@ -260,6 +270,37 @@ class HTMLIngredient { } + /** + * Handler for populate multi option changes. + * Populates the relevant arguments with the specified values. + * + * @param {event} e + */ + populateMultiOptionChange(e) { + e.preventDefault(); + e.stopPropagation(); + + const el = e.target; + const op = el.parentNode.parentNode; + const args = op.querySelectorAll(".arg"); + const targets = this.target.map(i => args[i]); + const vals = JSON.parse(el.childNodes[el.selectedIndex].getAttribute("populate-value")); + const evt = new Event("change"); + + for (let i = 0; i < targets.length; i++) { + targets[i].value = vals[i]; + } + + // Fire change event after all targets have been assigned + this.manager.recipe.ingChange(); + + // Send change event for each target once all have been assigned, to update the label placement. + for (const target of targets) { + target.dispatchEvent(evt); + } + } + + /** * Handler for editable option clicks. * Populates the input box with the selected value. diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index a8326b27..4c568c8b 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -205,6 +205,7 @@ class RecipeWaiter { * @fires Manager#statechange */ ingChange(e) { + if (e && e.target && e.target.classList.contains("no-state-change")) return; window.dispatchEvent(this.manager.statechange); } @@ -392,6 +393,15 @@ class RecipeWaiter { this.buildRecipeOperation(item); document.getElementById("rec-list").appendChild(item); + // Trigger populateOption events + const populateOptions = item.querySelectorAll(".populate-option"); + const evt = new Event("change", {bubbles: true}); + if (populateOptions.length) { + for (const el of populateOptions) { + el.dispatchEvent(evt); + } + } + item.dispatchEvent(this.manager.operationadd); return item; } diff --git a/tests/operations/tests/MultipleBombe.mjs b/tests/operations/tests/MultipleBombe.mjs index f809859c..5c06ece4 100644 --- a/tests/operations/tests/MultipleBombe.mjs +++ b/tests/operations/tests/MultipleBombe.mjs @@ -16,9 +16,10 @@ TestRegister.addTests([ "op": "Multiple Bombe", "args": [ // I, II and III - "User defined", "EKMFLGDQVZNTOWYHXUSPAIBRCJ Date: Fri, 8 Feb 2019 11:53:58 +0000 Subject: [PATCH 043/110] Fixed Bombe svg animation in standalone version --- package-lock.json | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/package-lock.json b/package-lock.json index 57175720..14b14f1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12074,6 +12074,28 @@ "resolved": "http://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, + "svg-url-loader": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/svg-url-loader/-/svg-url-loader-2.3.2.tgz", + "integrity": "sha1-3YaybBn+O5FPBOoQ7zlZTq3gRGQ=", + "dev": true, + "requires": { + "file-loader": "1.1.11", + "loader-utils": "1.1.0" + }, + "dependencies": { + "file-loader": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-1.1.11.tgz", + "integrity": "sha512-TGR4HU7HUsGg6GCOPJnFk06RhWgEWFLAGWiT6rcD+GRC2keU3s9RGJ+b3Z6/U73jwwNb2gKLJ7YCrp+jvU4ALg==", + "dev": true, + "requires": { + "loader-utils": "^1.0.2", + "schema-utils": "^0.4.5" + } + } + } + }, "symbol-tree": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz", From 1079080f5c024b13fca415ed85cdfb92ecf3e506 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 8 Feb 2019 15:21:14 +0000 Subject: [PATCH 044/110] Bombe results are now presented in a table --- src/core/operations/Bombe.mjs | 31 +++++++++++++++---- src/core/operations/MultipleBombe.mjs | 43 +++++++++++++++++++++------ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 34a94e53..6b277a03 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -26,7 +26,8 @@ class Bombe extends Operation { this.description = "Emulation of the Bombe machine used to attack Enigma.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and known plugboard pairs.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops, a large number of incorrect outputs will be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor.

By default this operation runs the checking machine, a manual process to verify the quality of Bombe stops, on each stop, discarding stops which fail. If you want to see how many times the hardware actually stops for a given input, disable the checking machine."; this.infoURL = "https://wikipedia.org/wiki/Bombe"; this.inputType = "string"; - this.outputType = "string"; + this.outputType = "JSON"; + this.presentType = "html"; this.args = [ { name: "1st (right-hand) rotor", @@ -82,7 +83,7 @@ class Bombe extends Operation { * @param {number} progress - Progress (as a float in the range 0..1) */ updateStatus(nLoops, nStops, progress) { - const msg = `Bombe run with ${nLoops} loops in menu (2+ desirable): ${nStops} stops, ${Math.floor(100 * progress)}% done`; + const msg = `Bombe run with ${nLoops} loop${nLoops === 1 ? "" : "s"} in menu (2+ desirable): ${nStops} stops, ${Math.floor(100 * progress)}% done`; self.sendStatusMessage(msg); } @@ -128,11 +129,29 @@ class Bombe extends Operation { } const bombe = new BombeMachine(rotors, reflector, ciphertext, crib, check, update); const result = bombe.run(); - let msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotor positions are listed left to right and start at the beginning of the crib, and ignore stepping and the ring setting. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; - for (const [setting, stecker, decrypt] of result) { - msg += `Stop: ${setting} (plugboard: ${stecker}): ${decrypt}\n`; + return { + nLoops: bombe.nLoops, + result: result + }; + } + + + /** + * Displays the Bombe results in an HTML table + * + * @param {Object} output + * @param {number} output.nLoops + * @param {Array[]} output.result + * @returns {html} + */ + present(output) { + let html = `Bombe run on menu with ${output.nLoops} loop${output.nLoops === 1 ? "" : "s"} (2+ desirable). Note: Rotor positions are listed left to right and start at the beginning of the crib, and ignore stepping and the ring setting. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided.\n\n`; + html += ""; + for (const [setting, stecker, decrypt] of output.result) { + html += `\n`; } - return msg; + html += "
Rotor stopsPartial plugboardDecryption preview
${setting}${stecker}${decrypt}
"; + return html; } } diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs index a453ca34..7a0ae2fd 100644 --- a/src/core/operations/MultipleBombe.mjs +++ b/src/core/operations/MultipleBombe.mjs @@ -56,7 +56,8 @@ class MultipleBombe extends Operation { this.description = "Emulation of the Bombe machine used to attack Enigma. This version carries out multiple Bombe runs to handle unknown rotor configurations.

You should test your menu on the single Bombe operation before running it here. See the description of the Bombe operation for instructions on choosing a crib."; this.infoURL = "https://wikipedia.org/wiki/Bombe"; this.inputType = "string"; - this.outputType = "string"; + this.outputType = "JSON"; + this.presentType = "html"; this.args = [ { "name": "Standard Enigmas", @@ -146,7 +147,7 @@ class MultipleBombe extends Operation { const hours = Math.floor(remaining / 3600); const minutes = `0${Math.floor((remaining % 3600) / 60)}`.slice(-2); const seconds = `0${Math.floor(remaining % 60)}`.slice(-2); - const msg = `Bombe run with ${nLoops} loops in menu (2+ desirable): ${nStops} stops, ${Math.floor(100 * progress)}% done, ${hours}:${minutes}:${seconds} remaining`; + const msg = `Bombe run with ${nLoops} loop${nLoops === 1 ? "" : "s"} in menu (2+ desirable): ${nStops} stops, ${Math.floor(100 * progress)}% done, ${hours}:${minutes}:${seconds} remaining`; self.sendStatusMessage(msg); } @@ -227,7 +228,7 @@ class MultipleBombe extends Operation { update = undefined; } let bombe = undefined; - let msg; + const output = {bombeRuns: []}; // I could use a proper combinatorics algorithm here... but it would be more code to // write one, and we don't seem to have one in our existing libraries, so massively nested // for loop it is @@ -253,7 +254,7 @@ class MultipleBombe extends Operation { } if (bombe === undefined) { bombe = new BombeMachine(runRotors, reflector, ciphertext, crib, check); - msg = `Bombe run on menu with ${bombe.nLoops} loops (2+ desirable). Note: Rotors and rotor positions are listed left to right, ignore stepping and the ring setting, and positions start at the beginning of the crib. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided. Results:\n`; + output.nLoops = bombe.nLoops; } else { bombe.changeRotors(runRotors, reflector); } @@ -263,17 +264,41 @@ class MultipleBombe extends Operation { update(bombe.nLoops, nStops, nRuns / totalRuns, start); } if (result.length > 0) { - msg += `\nRotors: ${runRotors.join(", ")}\nReflector: ${reflector.pairs}\n`; - for (const [setting, stecker, decrypt] of result) { - msg += `Stop: ${setting} (plugboard: ${stecker}): ${decrypt}\n`; - } + output.bombeRuns.push({ + rotors: runRotors, + reflector: reflector.pairs, + result: result + }); } } } } } } - return msg; + return output; + } + + + /** + * Displays the MultiBombe results in an HTML table + * + * @param {Object} output + * @param {number} output.nLoops + * @param {Array[]} output.result + * @returns {html} + */ + present(output) { + let html = `Bombe run on menu with ${output.nLoops} loop${output.nLoops === 1 ? "" : "s"} (2+ desirable). Note: Rotors and rotor positions are listed left to right, ignore stepping and the ring setting, and positions start at the beginning of the crib. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided.\n`; + + for (const run of output.bombeRuns) { + html += `\nRotors: ${run.rotors.join(", ")}\nReflector: ${run.reflector}\n`; + html += ""; + for (const [setting, stecker, decrypt] of run.result) { + html += `\n`; + } + html += "
Rotor stopsPartial plugboardDecryption preview
${setting}${stecker}${decrypt}
\n"; + } + return html; } } From 5a2a8b4c8ecb4b91b3876d31f6260159f5f9f80f Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Fri, 8 Feb 2019 18:08:13 +0000 Subject: [PATCH 045/110] Typex: input wiring is reversed --- src/core/lib/Typex.mjs | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/core/lib/Typex.mjs b/src/core/lib/Typex.mjs index b4cf297c..484a1e6b 100644 --- a/src/core/lib/Typex.mjs +++ b/src/core/lib/Typex.mjs @@ -2,6 +2,7 @@ * Emulation of the Typex machine. * * @author s2224834 + * @author The National Museum of Computing - Bombe Rebuild Project * @copyright Crown Copyright 2019 * @license Apache-2.0 */ @@ -28,7 +29,7 @@ export const ROTORS = [ * An example Typex reflector. Again, randomised. */ export const REFLECTORS = [ - {name: "Standard", value: "AN BC FG IE KD LU MH OR TS VZ WQ XJ YP"}, + {name: "Example", value: "AN BC FG IE KD LU MH OR TS VZ WQ XJ YP"}, ]; // Special character handling on Typex keyboard @@ -172,6 +173,8 @@ export class Rotor extends Enigma.Rotor { * Typex input plugboard. Based on a Rotor, because it allows arbitrary maps, not just switches * like the Enigma plugboard. * Not to be confused with the reflector plugboard. + * This is also where the Typex's backwards input wiring is implemented - it's a bit of a hack, but + * it means everything else continues to work like in the Enigma. */ export class Plugboard extends Enigma.Rotor { /** @@ -180,10 +183,45 @@ export class Plugboard extends Enigma.Rotor { * @param {string} wiring - 26 character string of mappings from A-Z, as per rotors, or "". */ constructor(wiring) { + // Typex input wiring is backwards vs Enigma: that is, letters enter the rotors in a + // clockwise order, vs. Enigma's anticlockwise (or vice versa depending on which side + // you're looking at it from). I'm doing the transform here to avoid having to rewrite + // the Engima crypt() method in Typex as well. + // Note that the wiring for the reflector is the same way around as Enigma, so no + // transformation is necessary on that side. + // We're going to achieve this by mapping the plugboard settings through an additional + // transform that mirrors the alphabet before we pass it to the superclass. + if (!/^[A-Z]{26}$/.test(wiring)) { + throw new OperationError("Plugboard wiring must be 26 unique uppercase letters"); + } + const reversed = "AZYXWVUTSRQPONMLKJIHGFEDCB"; + wiring = wiring.replace(/./g, x => { + return reversed[Enigma.a2i(x)]; + }); try { super(wiring, "", "A", "A"); } catch (err) { throw new OperationError(err.message.replace("Rotor", "Plugboard")); } } + + /** + * Transform a character through this rotor forwards. + * + * @param {number} c - The character. + * @returns {number} + */ + transform(c) { + return Utils.mod(this.map[Utils.mod(c + this.pos, 26)] - this.pos, 26); + } + + /** + * Transform a character through this rotor backwards. + * + * @param {number} c - The character. + * @returns {number} + */ + revTransform(c) { + return Utils.mod(this.revMap[Utils.mod(c + this.pos, 26)] - this.pos, 26); + } } From 5a8255a9f488817f44791bbc33cf23eb6988cc92 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Fri, 8 Feb 2019 19:25:28 +0000 Subject: [PATCH 046/110] Bombe: fix tests after output table patch --- tests/operations/tests/Bombe.mjs | 12 ++++++------ tests/operations/tests/MultipleBombe.mjs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index fca420d3..0f00f1be 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -11,7 +11,7 @@ TestRegister.addTests([ // Plugboard for this test is BO LC KE GA name: "Bombe: 3 rotor (self-stecker)", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA \(plugboard: SS\): VFISUSGTKSTMPSUNAK/, + expectedMatch: /LGA<\/td>SS<\/td>VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -30,7 +30,7 @@ TestRegister.addTests([ // This test produces a menu that doesn't use the first letter, which is also a good test name: "Bombe: 3 rotor (other stecker)", input: "JBYALIHDYNUAAVKBYM", - expectedMatch: /LGA \(plugboard: AG\): QFIMUMAFKMQSKMYNGW/, + expectedMatch: /LGA<\/td>AG<\/td>QFIMUMAFKMQSKMYNGW<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -48,7 +48,7 @@ TestRegister.addTests([ { name: "Bombe: crib offset", input: "AAABBYFLTHHYIJQAYBBYS", // first three chars here are faked - expectedMatch: /LGA \(plugboard: SS\): VFISUSGTKSTMPSUNAK/, + expectedMatch: /LGA<\/td>SS<\/td>VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -66,7 +66,7 @@ TestRegister.addTests([ { name: "Bombe: multiple stops", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA \(plugboard: TT\): VFISUSGTKSTMPSUNAK/, + expectedMatch: /LGA<\/td>TT<\/td>VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -84,7 +84,7 @@ TestRegister.addTests([ { name: "Bombe: checking machine", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /Stop: LGA \(plugboard: TT AG BO CL EK FF HH II JJ SS YY\): THISISATESTMESSAGE/, + expectedMatch: /LGA<\/td>TT AG BO CL EK FF HH II JJ SS YY<\/td>THISISATESTMESSAGE<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -103,7 +103,7 @@ TestRegister.addTests([ { name: "Bombe: 4 rotor", input: "LUOXGJSHGEDSRDOQQX", - expectedMatch: /LHSC \(plugboard: SS\): HHHSSSGQUUQPKSEKWK/, + expectedMatch: /LHSC<\/td>SS<\/td>HHHSSSGQUUQPKSEKWK<\/td>/, recipeConfig: [ { "op": "Bombe", diff --git a/tests/operations/tests/MultipleBombe.mjs b/tests/operations/tests/MultipleBombe.mjs index 5c06ece4..8e2cc685 100644 --- a/tests/operations/tests/MultipleBombe.mjs +++ b/tests/operations/tests/MultipleBombe.mjs @@ -10,7 +10,7 @@ TestRegister.addTests([ { name: "Multi-Bombe: 3 rotor", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA \(plugboard: SS\): VFISUSGTKSTMPSUNAK/, + expectedMatch: /LGA<\/td>SS<\/td>VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Multiple Bombe", @@ -30,7 +30,7 @@ TestRegister.addTests([ { name: "Multi-Bombe: 4 rotor", input: "LUOXGJSHGEDSRDOQQX", - expectedMatch: /LHSC \(plugboard: SS\): HHHSSSGQUUQPKSEKWK/, + expectedMatch: /LHSC<\/td>SS<\/td>HHHSSSGQUUQPKSEKWK<\/td>/, recipeConfig: [ { "op": "Multiple Bombe", From 61fee3122a5eac968ecaa5fa4ed6107581ba07f8 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Fri, 8 Feb 2019 21:16:42 +0000 Subject: [PATCH 047/110] Bombe: add Rebuild Project to authors --- src/core/lib/Bombe.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index 03413350..a4cf24f4 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -2,6 +2,7 @@ * Emulation of the Bombe machine. * * @author s2224834 + * @author The National Museum of Computing - Bombe Rebuild Project * @copyright Crown Copyright 2019 * @license Apache-2.0 */ From 069d4956aac93021bb1984eb618379c74062da37 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Sat, 9 Feb 2019 22:57:57 +0000 Subject: [PATCH 048/110] Bombe: Handle boxing stop correctly --- src/core/lib/Bombe.mjs | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index a4cf24f4..ef796cd0 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -651,12 +651,34 @@ export class BombeMachine { // This means our hypothesis for the steckering is correct. steckerPair = this.testInput[1]; } else { - // If this happens a lot it implies the menu isn't good enough. We can't do - // anything useful with it as we don't have a stecker partner, so we'll just drop it - // and move on. This does risk eating the actual stop occasionally, but I've only seen - // this happen when the menu is bad enough we have thousands of stops, so I'm not sure - // it matters. - return undefined; + // This was known as a "boxing stop" - we have a stop but not a single hypothesis. + // If this happens a lot it implies the menu isn't good enough. + // If we have the checking machine enabled, we're going to just check each wire in + // turn. If we get 0 or 1 hit, great. + // If we get multiple hits, or the checking machine is off, the user will just have to + // deal with it. + if (!this.check) { + // We can't draw any conclusions about the steckering (one could maybe suggest + // options in some cases, but too hard to present clearly). + return [this.indicator.getPos(), "??", this.tryDecrypt("")]; + } + let stecker = undefined; + for (let i = 0; i < 26; i++) { + const newStecker = this.checkingMachine(i); + if (newStecker !== "") { + if (stecker !== undefined) { + // Multiple hypotheses can't be ruled out. + return [this.indicator.getPos(), "??", this.tryDecrypt("")]; + } + stecker = newStecker; + } + } + if (stecker === undefined) { + // Checking machine ruled all possibilities out. + return undefined; + } + // If we got here, there was just one possibility allowed by the checking machine. Success. + return [this.indicator.getPos(), stecker, this.tryDecrypt(stecker)]; } let stecker; if (this.check) { From dd9cbbac77ef5a8c8e85b14215b2501aad906b5b Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Sat, 9 Feb 2019 23:01:52 +0000 Subject: [PATCH 049/110] Bombe: add note about rotor step in crib --- src/core/lib/Bombe.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/lib/Bombe.mjs b/src/core/lib/Bombe.mjs index ef796cd0..122edd40 100644 --- a/src/core/lib/Bombe.mjs +++ b/src/core/lib/Bombe.mjs @@ -281,6 +281,14 @@ export class BombeMachine { * ciphertext. It will check that the crib is sane (length is vaguely sensible and there's no * matching characters between crib and ciphertext) but cannot check further - if it's wrong * your results will be wrong! + * + * There is also no handling of rotor stepping - if the target Enigma stepped in the middle of + * your crib, you're out of luck. TODO: Allow specifying a step point - this is fairly easy to + * configure on a real Bombe, but we're not clear on whether it was ever actually done for + * real (there would almost certainly have been better ways of attacking in most situations + * than attempting to exhaust options for the stepping point, but in some circumstances, e.g. + * via Banburismus, the stepping point might have been known). + * * @param {string[]} rotors - list of rotor spec strings (without step points!) * @param {Object} reflector - Reflector object * @param {string} ciphertext - The ciphertext to attack From 4db6199fd947e9a06d6bf59c3ea54070054cd273 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sun, 10 Feb 2019 21:00:36 +0000 Subject: [PATCH 050/110] Fixed timings for Bombe animation fast rotor --- src/web/static/images/bombe.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/static/images/bombe.svg b/src/web/static/images/bombe.svg index 1fdca842..1fd40554 100644 --- a/src/web/static/images/bombe.svg +++ b/src/web/static/images/bombe.svg @@ -23,7 +23,7 @@ const bbox = rotor.getBBox(); const x = bbox.width/2 + bbox.x; const y = bbox.height/2 + bbox.y; - const wait = row === 0 ? speed/26 : row === 1 ? speed : speed*26; + const wait = row === 0 ? speed/26/1.5 : row === 1 ? speed : speed*26; rotor.setAttribute("transform", "rotate(" + startPos + ", " + x + ", " + y + ")"); @@ -50,7 +50,7 @@ break; } } - }, speed/26 - 5); + }, speed/26/1.5 - 5); } // ]]> From 91f4681a3cd2d8b9ac41ed18644e6a6625e18f2c Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 19 Feb 2019 15:37:49 +0000 Subject: [PATCH 051/110] Add rotate image operation --- src/core/operations/RotateImage.mjs | 96 +++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/core/operations/RotateImage.mjs diff --git a/src/core/operations/RotateImage.mjs b/src/core/operations/RotateImage.mjs new file mode 100644 index 00000000..1060aeb8 --- /dev/null +++ b/src/core/operations/RotateImage.mjs @@ -0,0 +1,96 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Rotate Image operation + */ +class RotateImage extends Operation { + + /** + * RotateImage constructor + */ + constructor() { + super(); + + this.name = "Rotate Image"; + this.module = "Image"; + this.description = "Rotates an image by the specified number of degrees."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + "name": "Rotation amount (degrees)", + "type": "number", + "value": 90 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + const [degrees] = args; + const type = Magic.magicFileType(input); + + if (type && type.mime.indexOf("image") === 0){ + return new Promise((resolve, reject) => { + jimp.read(Buffer.from(input)) + .then(image => { + image + .rotate(degrees / 100) + .getBuffer(jimp.AUTO, (error, result) => { + if (error){ + reject(new OperationError("Error getting the new image buffer")); + } else { + resolve([...result]); + } + }); + }) + .catch(err => { + reject(new OperationError("Error reading the input image.")); + }); + }); + } else { + throw new OperationError("Invalid file type."); + } + } + + /** + * Displays the rotated image using HTML for web apps + * + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default RotateImage; From 57e1061063c898ec8c73f8315582ebdb75da7bca Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 19 Feb 2019 15:37:59 +0000 Subject: [PATCH 052/110] Add Scale Image operation --- src/core/operations/ScaleImage.mjs | 94 ++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/core/operations/ScaleImage.mjs diff --git a/src/core/operations/ScaleImage.mjs b/src/core/operations/ScaleImage.mjs new file mode 100644 index 00000000..8db50fea --- /dev/null +++ b/src/core/operations/ScaleImage.mjs @@ -0,0 +1,94 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Scale Image operation + */ +class ScaleImage extends Operation { + + /** + * ScaleImage constructor + */ + constructor() { + super(); + + this.name = "Scale Image"; + this.module = "Image"; + this.description = "Uniformly scale an image by a specified factor."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Scale factor (percent)", + type: "number", + value: 100 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + const [scaleFactor] = args; + const type = Magic.magicFileType(input); + + if (type && type.mime.indexOf("image") === 0){ + return new Promise((resolve, reject) => { + jimp.read(Buffer.from(input)) + .then(image => { + image + .scale((scaleFactor / 100)) + .getBuffer(jimp.AUTO, (error, result) => { + if (error){ + reject(new OperationError("Error getting the new image buffer.")); + } else { + resolve([...result]); + } + }); + }) + .catch(err => { + reject(new OperationError("Error reading the input image.")); + }); + }); + } else { + throw new OperationError("Invalid file type."); + } + } + + /** + * Displays the scaled image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default ScaleImage; From eb8725a0db0a04214ac6b77ea44a6b4d3e9c6b55 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 19 Feb 2019 16:10:53 +0000 Subject: [PATCH 053/110] Fix degrees error --- src/core/operations/RotateImage.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/RotateImage.mjs b/src/core/operations/RotateImage.mjs index 1060aeb8..7f01b034 100644 --- a/src/core/operations/RotateImage.mjs +++ b/src/core/operations/RotateImage.mjs @@ -51,7 +51,7 @@ class RotateImage extends Operation { jimp.read(Buffer.from(input)) .then(image => { image - .rotate(degrees / 100) + .rotate(degrees) .getBuffer(jimp.AUTO, (error, result) => { if (error){ reject(new OperationError("Error getting the new image buffer")); From 1a2c5a95c737c0a70236b249ee70caa7f5b4c0aa Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 19 Feb 2019 16:19:34 +0000 Subject: [PATCH 054/110] Add resize image operation --- src/core/operations/ResizeImage.mjs | 125 ++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 src/core/operations/ResizeImage.mjs diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs new file mode 100644 index 00000000..3e177c16 --- /dev/null +++ b/src/core/operations/ResizeImage.mjs @@ -0,0 +1,125 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Resize Image operation + */ +class ResizeImage extends Operation { + + /** + * ResizeImage constructor + */ + constructor() { + super(); + + this.name = "Resize Image"; + this.module = "Image"; + this.description = "Resizes an image to the specified width and height values."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Width", + type: "number", + value: 100 + }, + { + name: "Height", + type: "number", + value: 100 + }, + { + name: "Unit type", + type: "option", + value: ["Pixels", "Percent"] + }, + { + name: "Maintain aspect ratio", + type: "boolean", + value: false + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + let width = args[0], + height = args[1]; + const unit = args[2], + aspect = args[3], + type = Magic.magicFileType(input); + + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + return new Promise((resolve, reject) => { + jimp.read(Buffer.from(input)) + .then(image => { + if (unit === "Percent") { + width = image.getWidth() * (width / 100); + height = image.getHeight() * (height / 100); + } + if (aspect) { + image + .scaleToFit(width, height) + .getBuffer(jimp.AUTO, (error, result) => { + if (error){ + reject(new OperationError("Error scaling the image.")); + } else { + resolve([...result]); + } + }); + } else { + image + .resize(width, height) + .getBuffer(jimp.AUTO, (error, result) => { + if (error){ + reject(new OperationError("Error scaling the image.")); + } else { + resolve([...result]); + } + }); + } + }); + }); + } + + /** + * Displays the resized image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default ResizeImage; From 01acefe4cffec3e6401f9ec4aa1786d6e1adff5c Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 19 Feb 2019 16:20:36 +0000 Subject: [PATCH 055/110] Remove scale image operation. (Same functionality is implemented in Resize Image) --- src/core/operations/ScaleImage.mjs | 94 ------------------------------ 1 file changed, 94 deletions(-) delete mode 100644 src/core/operations/ScaleImage.mjs diff --git a/src/core/operations/ScaleImage.mjs b/src/core/operations/ScaleImage.mjs deleted file mode 100644 index 8db50fea..00000000 --- a/src/core/operations/ScaleImage.mjs +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @author j433866 [j433866@gmail.com] - * @copyright Crown Copyright 2019 - * @license Apache-2.0 - */ - -import Operation from "../Operation"; -import OperationError from "../errors/OperationError"; -import Magic from "../lib/Magic"; -import { toBase64 } from "../lib/Base64"; -import jimp from "jimp"; - -/** - * Scale Image operation - */ -class ScaleImage extends Operation { - - /** - * ScaleImage constructor - */ - constructor() { - super(); - - this.name = "Scale Image"; - this.module = "Image"; - this.description = "Uniformly scale an image by a specified factor."; - this.infoURL = ""; - this.inputType = "byteArray"; - this.outputType = "byteArray"; - this.presentType = "html"; - this.args = [ - { - name: "Scale factor (percent)", - type: "number", - value: 100 - } - ]; - } - - /** - * @param {byteArray} input - * @param {Object[]} args - * @returns {byteArray} - */ - run(input, args) { - const [scaleFactor] = args; - const type = Magic.magicFileType(input); - - if (type && type.mime.indexOf("image") === 0){ - return new Promise((resolve, reject) => { - jimp.read(Buffer.from(input)) - .then(image => { - image - .scale((scaleFactor / 100)) - .getBuffer(jimp.AUTO, (error, result) => { - if (error){ - reject(new OperationError("Error getting the new image buffer.")); - } else { - resolve([...result]); - } - }); - }) - .catch(err => { - reject(new OperationError("Error reading the input image.")); - }); - }); - } else { - throw new OperationError("Invalid file type."); - } - } - - /** - * Displays the scaled image using HTML for web apps - * @param {byteArray} data - * @returns {html} - */ - present(data) { - if (!data.length) return ""; - - let dataURI = "data:"; - const type = Magic.magicFileType(data); - if (type && type.mime.indexOf("image") === 0){ - dataURI += type.mime + ";"; - } else { - throw new OperationError("Invalid file type"); - } - dataURI += "base64," + toBase64(data); - - return ""; - } - -} - -export default ScaleImage; From b691c3067771e11a7775728be8d2cf62090c0055 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 09:20:38 +0000 Subject: [PATCH 056/110] Add dither image operation --- src/core/operations/DitherImage.mjs | 89 +++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/core/operations/DitherImage.mjs diff --git a/src/core/operations/DitherImage.mjs b/src/core/operations/DitherImage.mjs new file mode 100644 index 00000000..a3fd4974 --- /dev/null +++ b/src/core/operations/DitherImage.mjs @@ -0,0 +1,89 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Image Dither operation + */ +class DitherImage extends Operation { + + /** + * DitherImage constructor + */ + constructor() { + super(); + + this.name = "Dither Image"; + this.module = "Image"; + this.description = "Apply a dither effect to an image."; + this.infoURL = "https://wikipedia.org/wiki/Dither"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + const type = Magic.magicFileType(input); + + if (type && type.mime.indexOf("image") === 0){ + return new Promise((resolve, reject) => { + jimp.read(Buffer.from(input)) + .then(image => { + image + .dither565() + .getBuffer(jimp.AUTO, (error, result) => { + if (error){ + reject(new OperationError("Error getting the new image buffer")); + } else { + resolve([...result]); + } + }); + }) + .catch(err => { + reject(new OperationError("Error applying a dither effect to the image.")); + }); + }); + } else { + throw new OperationError("Invalid file type."); + } + } + + /** + * Displays the dithered image using HTML for web apps + * + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default DitherImage; From 74c2a2b5cbb022dd6e6de231861a9668c75c5d13 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 11:12:15 +0000 Subject: [PATCH 057/110] Add Invert Image operation --- src/core/operations/InvertImage.mjs | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/core/operations/InvertImage.mjs diff --git a/src/core/operations/InvertImage.mjs b/src/core/operations/InvertImage.mjs new file mode 100644 index 00000000..87da0156 --- /dev/null +++ b/src/core/operations/InvertImage.mjs @@ -0,0 +1,73 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Invert Image operation + */ +class InvertImage extends Operation { + + /** + * InvertImage constructor + */ + constructor() { + super(); + + this.name = "Invert Image"; + this.module = "Image"; + this.description = "Invert the colours of an image."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const type = Magic.magicFileType(input); + if (!type || type.mime.indexOf("image") !== 0) { + throw new OperationError("Invalid input file format."); + } + const image = await jimp.read(Buffer.from(input)); + image.invert(); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the inverted image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default InvertImage; From a0b94bba4e32e9af2e2cb12a45e0fe02183bf219 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 11:26:39 +0000 Subject: [PATCH 058/110] Change run() functions to be async --- src/core/operations/DitherImage.mjs | 23 ++++----------- src/core/operations/ResizeImage.mjs | 45 +++++++++-------------------- src/core/operations/RotateImage.mjs | 24 ++++----------- 3 files changed, 24 insertions(+), 68 deletions(-) diff --git a/src/core/operations/DitherImage.mjs b/src/core/operations/DitherImage.mjs index a3fd4974..aff95a3c 100644 --- a/src/core/operations/DitherImage.mjs +++ b/src/core/operations/DitherImage.mjs @@ -36,27 +36,14 @@ class DitherImage extends Operation { * @param {Object[]} args * @returns {byteArray} */ - run(input, args) { + async run(input, args) { const type = Magic.magicFileType(input); if (type && type.mime.indexOf("image") === 0){ - return new Promise((resolve, reject) => { - jimp.read(Buffer.from(input)) - .then(image => { - image - .dither565() - .getBuffer(jimp.AUTO, (error, result) => { - if (error){ - reject(new OperationError("Error getting the new image buffer")); - } else { - resolve([...result]); - } - }); - }) - .catch(err => { - reject(new OperationError("Error applying a dither effect to the image.")); - }); - }); + const image = await jimp.read(Buffer.from(input)); + image.dither565(); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; } else { throw new OperationError("Invalid file type."); } diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index 3e177c16..115d8c65 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -57,7 +57,7 @@ class ResizeImage extends Operation { * @param {Object[]} args * @returns {byteArray} */ - run(input, args) { + async run(input, args) { let width = args[0], height = args[1]; const unit = args[2], @@ -67,37 +67,20 @@ class ResizeImage extends Operation { if (!type || type.mime.indexOf("image") !== 0){ throw new OperationError("Invalid file type."); } + const image = await jimp.read(Buffer.from(input)); - return new Promise((resolve, reject) => { - jimp.read(Buffer.from(input)) - .then(image => { - if (unit === "Percent") { - width = image.getWidth() * (width / 100); - height = image.getHeight() * (height / 100); - } - if (aspect) { - image - .scaleToFit(width, height) - .getBuffer(jimp.AUTO, (error, result) => { - if (error){ - reject(new OperationError("Error scaling the image.")); - } else { - resolve([...result]); - } - }); - } else { - image - .resize(width, height) - .getBuffer(jimp.AUTO, (error, result) => { - if (error){ - reject(new OperationError("Error scaling the image.")); - } else { - resolve([...result]); - } - }); - } - }); - }); + if (unit === "Percent") { + width = image.getWidth() * (width / 100); + height = image.getHeight() * (height / 100); + } + if (aspect) { + image.scaleToFit(width, height); + } else { + image.resize(width, height); + } + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; } /** diff --git a/src/core/operations/RotateImage.mjs b/src/core/operations/RotateImage.mjs index 7f01b034..1bab6c98 100644 --- a/src/core/operations/RotateImage.mjs +++ b/src/core/operations/RotateImage.mjs @@ -42,28 +42,15 @@ class RotateImage extends Operation { * @param {Object[]} args * @returns {byteArray} */ - run(input, args) { + async run(input, args) { const [degrees] = args; const type = Magic.magicFileType(input); if (type && type.mime.indexOf("image") === 0){ - return new Promise((resolve, reject) => { - jimp.read(Buffer.from(input)) - .then(image => { - image - .rotate(degrees) - .getBuffer(jimp.AUTO, (error, result) => { - if (error){ - reject(new OperationError("Error getting the new image buffer")); - } else { - resolve([...result]); - } - }); - }) - .catch(err => { - reject(new OperationError("Error reading the input image.")); - }); - }); + const image = await jimp.read(Buffer.from(input)); + image.rotate(degrees); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; } else { throw new OperationError("Invalid file type."); } @@ -71,7 +58,6 @@ class RotateImage extends Operation { /** * Displays the rotated image using HTML for web apps - * * @param {byteArray} data * @returns {html} */ From 0dd430490214c6ba334a3314b39e1da1e28107dc Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 11:48:24 +0000 Subject: [PATCH 059/110] Add new Blur Image operation. Performs both fast blur and gaussian blur --- src/core/operations/BlurImage.mjs | 96 +++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/core/operations/BlurImage.mjs diff --git a/src/core/operations/BlurImage.mjs b/src/core/operations/BlurImage.mjs new file mode 100644 index 00000000..68ae0b0f --- /dev/null +++ b/src/core/operations/BlurImage.mjs @@ -0,0 +1,96 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Blur Image operation + */ +class BlurImage extends Operation { + + /** + * BlurImage constructor + */ + constructor() { + super(); + + this.name = "Blur Image"; + this.module = "Image"; + this.description = "Applies a blur effect to the image.

Gaussian blur is much slower than fast blur, but produces better results."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Blur Amount", + type: "number", + value: 5 + }, + { + name: "Blur Type", + type: "option", + value: ["Fast", "Gaussian"] + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [blurAmount, blurType] = args; + const type = Magic.magicFileType(input); + + if (type && type.mime.indexOf("image") === 0){ + const image = await jimp.read(Buffer.from(input)); + + switch (blurType){ + case "Fast": + image.blur(blurAmount); + break; + case "Gaussian": + image.gaussian(blurAmount); + break; + } + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } else { + throw new OperationError("Invalid file type."); + } + } + + /** + * Displays the blurred image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default BlurImage; From fd160e87e88ac84c177f7d6732127fa1498dd229 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 11:54:59 +0000 Subject: [PATCH 060/110] Add image operations to Categories --- src/core/config/Categories.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 8235ab10..081a5152 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -359,7 +359,12 @@ "Play Media", "Remove EXIF", "Extract EXIF", - "Split Colour Channels" + "Split Colour Channels", + "Rotate Image", + "Resize Image", + "Blur Image", + "Dither Image", + "Invert Image" ] }, { From da838e266e08f676c9fcf72faf3b00c5cbd47350 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 13:04:15 +0000 Subject: [PATCH 061/110] Add flip image operation --- src/core/config/Categories.json | 3 +- src/core/operations/FlipImage.mjs | 90 +++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/FlipImage.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 081a5152..9b0f8249 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -364,7 +364,8 @@ "Resize Image", "Blur Image", "Dither Image", - "Invert Image" + "Invert Image", + "Flip Image" ] }, { diff --git a/src/core/operations/FlipImage.mjs b/src/core/operations/FlipImage.mjs new file mode 100644 index 00000000..fa3054e2 --- /dev/null +++ b/src/core/operations/FlipImage.mjs @@ -0,0 +1,90 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Flip Image operation + */ +class FlipImage extends Operation { + + /** + * FlipImage constructor + */ + constructor() { + super(); + + this.name = "Flip Image"; + this.module = "Image"; + this.description = "Flips an image along its X or Y axis."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType="html"; + this.args = [ + { + name: "Flip Axis", + type: "option", + value: ["Horizontal", "Vertical"] + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [flipAxis] = args; + const type = Magic.magicFileType(input); + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid input file type."); + } + + const image = await jimp.read(Buffer.from(input)); + + switch (flipAxis){ + case "Horizontal": + image.flip(true, false); + break; + case "Vertical": + image.flip(false, true); + break; + } + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the flipped image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default FlipImage; From 9f4aa0a1233683c185cecdb2a7f9fb98d0233519 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 13:17:57 +0000 Subject: [PATCH 062/110] Remove trailing space --- src/core/operations/DitherImage.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/operations/DitherImage.mjs b/src/core/operations/DitherImage.mjs index aff95a3c..2cc9ac2d 100644 --- a/src/core/operations/DitherImage.mjs +++ b/src/core/operations/DitherImage.mjs @@ -51,7 +51,6 @@ class DitherImage extends Operation { /** * Displays the dithered image using HTML for web apps - * * @param {byteArray} data * @returns {html} */ From 0d86a7e42780336ac734aaf4db053715aca803ce Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 20 Feb 2019 15:35:53 +0000 Subject: [PATCH 063/110] Add resize algorithm option --- src/core/operations/ResizeImage.mjs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index 115d8c65..aa5cb24b 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -48,6 +48,18 @@ class ResizeImage extends Operation { name: "Maintain aspect ratio", type: "boolean", value: false + }, + { + name: "Resizing algorithm", + type: "option", + value: [ + "Nearest Neighbour", + "Bilinear", + "Bicubic", + "Hermite", + "Bezier" + ], + defaultIndex: 1 } ]; } @@ -62,8 +74,17 @@ class ResizeImage extends Operation { height = args[1]; const unit = args[2], aspect = args[3], + resizeAlg = args[4], type = Magic.magicFileType(input); + const resizeMap = { + "Nearest Neighbour": jimp.RESIZE_NEAREST_NEIGHBOR, + "Bilinear": jimp.RESIZE_BILINEAR, + "Bicubic": jimp.RESIZE_BICUBIC, + "Hermite": jimp.RESIZE_HERMITE, + "Bezier": jimp.RESIZE_BEZIER + }; + if (!type || type.mime.indexOf("image") !== 0){ throw new OperationError("Invalid file type."); } @@ -74,9 +95,9 @@ class ResizeImage extends Operation { height = image.getHeight() * (height / 100); } if (aspect) { - image.scaleToFit(width, height); + image.scaleToFit(width, height, resizeMap[resizeAlg]); } else { - image.resize(width, height); + image.resize(width, height, resizeMap[resizeAlg]); } const imageBuffer = await image.getBufferAsync(jimp.AUTO); From 68278267e1dc5e619165d7b74877a147812a29c1 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 23 Feb 2019 15:13:58 +0000 Subject: [PATCH 064/110] Update libyara-wasm --- package-lock.json | 146 +++++++++++++++++++++++----------------------- package.json | 2 +- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/package-lock.json b/package-lock.json index 55ad6303..0d1cb4e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1631,7 +1631,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1716,7 +1716,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1864,7 +1864,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -2334,7 +2334,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -2371,7 +2371,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -2436,7 +2436,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2590,7 +2590,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2639,7 +2639,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -3172,7 +3172,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -3185,7 +3185,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -3332,7 +3332,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3700,7 +3700,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3764,7 +3764,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3969,7 +3969,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -4392,7 +4392,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -4404,7 +4404,7 @@ }, "events": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4821,7 +4821,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -5057,7 +5057,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -5726,7 +5726,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5868,7 +5868,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -5945,7 +5945,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5993,7 +5993,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -6013,7 +6013,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6058,7 +6058,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -6157,7 +6157,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -6221,7 +6221,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6538,7 +6538,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -6557,7 +6557,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -6607,7 +6607,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -7053,7 +7053,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -7614,7 +7614,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -7725,7 +7725,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -7798,9 +7798,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.11.tgz", - "integrity": "sha512-rglapPFo0IHPNksWYQXI8oqftXYj5mOGOf4BXtbSySVRX71pro4BehNjJ5qEpjYx+roGvNkcAD9zCsitA08sxw==" + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.12.tgz", + "integrity": "sha512-AjTe4FiBuH4F7HwGT/3UxoRenczXtrbM6oWGrifxb44LrkDh5VxRNg9zwfPpDA5Fcc1iYcXS0WVA/b3DGtD8cQ==" }, "livereload-js": { "version": "2.4.0", @@ -7844,7 +7844,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -7857,7 +7857,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8221,7 +8221,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -8280,7 +8280,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -8501,7 +8501,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -8542,7 +8542,7 @@ "dependencies": { "commander": { "version": "2.15.1", - "resolved": "http://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", "dev": true, "optional": true @@ -8711,7 +8711,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -8810,7 +8810,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -8993,7 +8993,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -9287,13 +9287,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -9302,7 +9302,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -9526,7 +9526,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -9612,7 +9612,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -9653,7 +9653,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9836,7 +9836,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -10207,7 +10207,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -10232,13 +10232,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -10253,7 +10253,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -10476,7 +10476,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -10665,7 +10665,7 @@ "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", "dev": true } @@ -10716,7 +10716,7 @@ }, "htmlparser2": { "version": "3.3.0", - "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", "integrity": "sha1-zHDQWln2VC5D8OaFyYLhTJJKnv4=", "dev": true, "requires": { @@ -10728,7 +10728,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -10995,7 +10995,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -11315,7 +11315,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -11359,7 +11359,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -12080,7 +12080,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -12097,7 +12097,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -12190,7 +12190,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -12348,7 +12348,7 @@ }, "through": { "version": "2.3.8", - "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -13008,7 +13008,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -13034,7 +13034,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -13050,7 +13050,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -13582,7 +13582,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -13736,14 +13736,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true @@ -13776,7 +13776,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/package.json b/package.json index cb59db38..a14af274 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.11", + "libyara-wasm": "0.0.12", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", From c005c86c276eb8a9f16ebb82b21ce1cb94779c66 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 28 Feb 2019 15:27:35 +0000 Subject: [PATCH 065/110] Added argSelector ingredient type and reversed rotors in Enigma and Bombe operations. --- src/core/operations/Bombe.mjs | 57 ++++++--- src/core/operations/Enigma.mjs | 135 +++++++++++--------- src/web/App.mjs | 3 + src/web/HTMLIngredient.mjs | 48 ++++++++ src/web/RecipeWaiter.mjs | 28 +++-- tests/operations/tests/Bombe.mjs | 84 +++++++------ tests/operations/tests/Enigma.mjs | 198 +++++++++++++++++------------- 7 files changed, 344 insertions(+), 209 deletions(-) diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 6b277a03..ea3210fa 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -30,28 +30,42 @@ class Bombe extends Operation { this.presentType = "html"; this.args = [ { - name: "1st (right-hand) rotor", - type: "editableOption", - value: ROTORS, - defaultIndex: 2 + name: "Model", + type: "argSelector", + value: [ + { + name: "3-rotor", + off: [1] + }, + { + name: "4-rotor", + on: [1] + } + ] }, { - name: "2nd (middle) rotor", + name: "Left-most rotor", + type: "editableOption", + value: ROTORS_FOURTH, + defaultIndex: 0 + }, + { + name: "Left-hand rotor", + type: "editableOption", + value: ROTORS, + defaultIndex: 0 + }, + { + name: "Middle rotor", type: "editableOption", value: ROTORS, defaultIndex: 1 }, { - name: "3rd (left-hand) rotor", + name: "Right-hand rotor", type: "editableOption", value: ROTORS, - defaultIndex: 0 - }, - { - name: "4th (left-most, only some models) rotor", - type: "editableOption", - value: ROTORS_FOURTH, - defaultIndex: 0 + defaultIndex: 2 }, { name: "Reflector", @@ -93,23 +107,26 @@ class Bombe extends Operation { * @returns {string} */ run(input, args) { - const reflectorstr = args[4]; - let crib = args[5]; - const offset = args[6]; - const check = args[7]; + const model = args[0]; + const reflectorstr = args[5]; + let crib = args[6]; + const offset = args[7]; + const check = args[8]; const rotors = []; for (let i=0; i<4; i++) { - if (i === 3 && args[i] === "") { + if (i === 0 && model === "3-rotor") { // No fourth rotor - break; + continue; } - let rstr = args[i]; + let rstr = args[i + 1]; // The Bombe doesn't take stepping into account so we'll just ignore it here if (rstr.includes("<")) { rstr = rstr.split("<", 2)[0]; } rotors.push(rstr); } + // Rotors are handled in reverse + rotors.reverse(); if (crib.length === 0) { throw new OperationError("Crib cannot be empty"); } diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs index 4af79993..ace50604 100644 --- a/src/core/operations/Enigma.mjs +++ b/src/core/operations/Enigma.mjs @@ -28,67 +28,81 @@ class Enigma extends Operation { this.outputType = "string"; this.args = [ { - name: "1st (right-hand) rotor", + name: "Model", + type: "argSelector", + value: [ + { + name: "3-rotor", + off: [1, 2, 3] + }, + { + name: "4-rotor", + on: [1, 2, 3] + } + ] + }, + { + name: "Left-most rotor", + type: "editableOption", + value: ROTORS_FOURTH, + defaultIndex: 0 + }, + { + name: "Left-most rotor ring setting", + type: "option", + value: LETTERS + }, + { + name: "Left-most rotor initial value", + type: "option", + value: LETTERS + }, + { + name: "Left-hand rotor", + type: "editableOption", + value: ROTORS, + defaultIndex: 0 + }, + { + name: "Left-hand rotor ring setting", + type: "option", + value: LETTERS + }, + { + name: "Left-hand rotor initial value", + type: "option", + value: LETTERS + }, + { + name: "Middle rotor", + type: "editableOption", + value: ROTORS, + defaultIndex: 1 + }, + { + name: "Middle rotor ring setting", + type: "option", + value: LETTERS + }, + { + name: "Middle rotor initial value", + type: "option", + value: LETTERS + }, + { + name: "Right-hand rotor", type: "editableOption", value: ROTORS, // Default config is the rotors I-III *left to right* defaultIndex: 2 }, { - name: "1st rotor ring setting", + name: "Right-hand rotor ring setting", type: "option", value: LETTERS }, { - name: "1st rotor initial value", - type: "option", - value: LETTERS - }, - { - name: "2nd (middle) rotor", - type: "editableOption", - value: ROTORS, - defaultIndex: 1 - }, - { - name: "2nd rotor ring setting", - type: "option", - value: LETTERS - }, - { - name: "2nd rotor initial value", - type: "option", - value: LETTERS - }, - { - name: "3rd (left-hand) rotor", - type: "editableOption", - value: ROTORS, - defaultIndex: 0 - }, - { - name: "3rd rotor ring setting", - type: "option", - value: LETTERS - }, - { - name: "3rd rotor initial value", - type: "option", - value: LETTERS - }, - { - name: "4th (left-most, only some models) rotor", - type: "editableOption", - value: ROTORS_FOURTH, - defaultIndex: 0 - }, - { - name: "4th rotor ring setting", - type: "option", - value: LETTERS - }, - { - name: "4th rotor initial value", + name: "Right-hand rotor initial value", type: "option", value: LETTERS }, @@ -135,18 +149,21 @@ class Enigma extends Operation { * @returns {string} */ run(input, args) { - const reflectorstr = args[12]; - const plugboardstr = args[13]; - const removeOther = args[14]; + const model = args[0]; + const reflectorstr = args[13]; + const plugboardstr = args[14]; + const removeOther = args[15]; const rotors = []; for (let i=0; i<4; i++) { - if (i === 3 && args[i*3] === "") { - // No fourth rotor - break; + if (i === 0 && model === "3-rotor") { + // Skip the 4th rotor settings + continue; } - const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*3], 1); - rotors.push(new Rotor(rotorwiring, rotorsteps, args[i*3 + 1], args[i*3 + 2])); + const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*3 + 1], 1); + rotors.push(new Rotor(rotorwiring, rotorsteps, args[i*3 + 2], args[i*3 + 3])); } + // Rotors are handled in reverse + rotors.reverse(); const reflector = new Reflector(reflectorstr); const plugboard = new Plugboard(plugboardstr); if (removeOther) { diff --git a/src/web/App.mjs b/src/web/App.mjs index e203b85c..04846fb6 100755 --- a/src/web/App.mjs +++ b/src/web/App.mjs @@ -472,6 +472,7 @@ class App { const item = this.manager.recipe.addOperation(recipeConfig[i].op); // Populate arguments + log.debug(`Populating arguments for ${recipeConfig[i].op}`); const args = item.querySelectorAll(".arg"); for (let j = 0; j < args.length; j++) { if (recipeConfig[i].args[j] === undefined) continue; @@ -497,6 +498,8 @@ class App { item.querySelector(".breakpoint").click(); } + this.manager.recipe.triggerArgEvents(item); + this.progress = 0; } diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index c7c024fb..19c816ea 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -240,6 +240,27 @@ class HTMLIngredient { ${this.hint ? "" + this.hint + "" : ""}
`; break; + case "argSelector": + html += `
+ + + ${this.hint ? "" + this.hint + "" : ""} +
`; + + this.manager.addDynamicListener(".arg-selector", "change", this.argSelectorChange, this); + break; default: break; } @@ -321,6 +342,33 @@ class HTMLIngredient { this.manager.recipe.ingChange(); } + + /** + * Handler for argument selector changes. + * Shows or hides the relevant arguments for this operation. + * + * @param {event} e + */ + argSelectorChange(e) { + e.preventDefault(); + e.stopPropagation(); + + const option = e.target.options[e.target.selectedIndex]; + const op = e.target.closest(".operation"); + const args = op.querySelectorAll(".ingredients .form-group"); + const turnon = JSON.parse(option.getAttribute("turnon")); + const turnoff = JSON.parse(option.getAttribute("turnoff")); + + args.forEach((arg, i) => { + if (turnon.includes(i)) { + arg.classList.remove("d-none"); + } + if (turnoff.includes(i)) { + arg.classList.add("d-none"); + } + }); + } + } export default HTMLIngredient; diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index 4c568c8b..4eca4af7 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -393,15 +393,6 @@ class RecipeWaiter { this.buildRecipeOperation(item); document.getElementById("rec-list").appendChild(item); - // Trigger populateOption events - const populateOptions = item.querySelectorAll(".populate-option"); - const evt = new Event("change", {bubbles: true}); - if (populateOptions.length) { - for (const el of populateOptions) { - el.dispatchEvent(evt); - } - } - item.dispatchEvent(this.manager.operationadd); return item; } @@ -439,6 +430,23 @@ class RecipeWaiter { } + /** + * Triggers various change events for operation arguments that have just been initialised. + * + * @param {HTMLElement} op + */ + triggerArgEvents(op) { + // Trigger populateOption and argSelector events + const triggerableOptions = op.querySelectorAll(".populate-option, .arg-selector"); + const evt = new Event("change", {bubbles: true}); + if (triggerableOptions.length) { + for (const el of triggerableOptions) { + el.dispatchEvent(evt); + } + } + } + + /** * Handler for operationadd events. * @@ -448,6 +456,8 @@ class RecipeWaiter { */ opAdd(e) { log.debug(`'${e.target.querySelector(".op-title").textContent}' added to recipe`); + + this.triggerArgEvents(e.target); window.dispatchEvent(this.manager.statechange); } diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index 0f00f1be..9e5a79c6 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -16,10 +16,11 @@ TestRegister.addTests([ { "op": "Bombe", "args": [ - "BDFHJLCPRTXVZNYEIWGAKMUSQO Date: Thu, 28 Feb 2019 16:56:28 +0000 Subject: [PATCH 066/110] Tweaks for new rotor order --- src/core/lib/Enigma.mjs | 1 - src/core/operations/Bombe.mjs | 2 +- src/core/operations/Enigma.mjs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/lib/Enigma.mjs b/src/core/lib/Enigma.mjs index 1ed0ea2b..39193f69 100644 --- a/src/core/lib/Enigma.mjs +++ b/src/core/lib/Enigma.mjs @@ -25,7 +25,6 @@ export const ROTORS = [ ]; export const ROTORS_FOURTH = [ - {name: "None", value: ""}, {name: "Beta", value: "LEYJVCNIXWPBQMDRTAKZGFUHOS"}, {name: "Gamma", value: "FSOKANUERHMBTIYCWLQPZXVGJD"}, ]; diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index ea3210fa..5e128498 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -44,7 +44,7 @@ class Bombe extends Operation { ] }, { - name: "Left-most rotor", + name: "Left-most (4th) rotor", type: "editableOption", value: ROTORS_FOURTH, defaultIndex: 0 diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs index ace50604..77333b18 100644 --- a/src/core/operations/Enigma.mjs +++ b/src/core/operations/Enigma.mjs @@ -42,7 +42,7 @@ class Enigma extends Operation { ] }, { - name: "Left-most rotor", + name: "Left-most (4th) rotor", type: "editableOption", value: ROTORS_FOURTH, defaultIndex: 0 From 1f9fd92b01db91518855039a41aee470daf3608f Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 28 Feb 2019 17:21:47 +0000 Subject: [PATCH 067/110] Typex: rotors in same order as Enigma --- src/core/operations/Typex.mjs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/core/operations/Typex.mjs b/src/core/operations/Typex.mjs index 504cb891..9c963357 100644 --- a/src/core/operations/Typex.mjs +++ b/src/core/operations/Typex.mjs @@ -29,10 +29,10 @@ class Typex extends Operation { this.outputType = "string"; this.args = [ { - name: "1st (right-hand, static) rotor", + name: "1st (left-hand) rotor", type: "editableOption", value: ROTORS, - defaultIndex: 4 + defaultIndex: 0 }, { name: "1st rotor reversed", @@ -50,10 +50,10 @@ class Typex extends Operation { value: LETTERS }, { - name: "2nd (static) rotor", + name: "2nd rotor", type: "editableOption", value: ROTORS, - defaultIndex: 3 + defaultIndex: 1 }, { name: "2nd rotor reversed", @@ -71,7 +71,7 @@ class Typex extends Operation { value: LETTERS }, { - name: "3rd rotor", + name: "3rd (middle) rotor", type: "editableOption", value: ROTORS, defaultIndex: 2 @@ -92,10 +92,10 @@ class Typex extends Operation { value: LETTERS }, { - name: "4th rotor", + name: "4th (static) rotor", type: "editableOption", value: ROTORS, - defaultIndex: 1 + defaultIndex: 3 }, { name: "4th rotor reversed", @@ -113,10 +113,10 @@ class Typex extends Operation { value: LETTERS }, { - name: "5th rotor", + name: "5th (right-hand, static) rotor", type: "editableOption", value: ROTORS, - defaultIndex: 0 + defaultIndex: 4 }, { name: "5th rotor reversed", @@ -190,6 +190,8 @@ class Typex extends Operation { const [rotorwiring, rotorsteps] = this.parseRotorStr(args[i*4]); rotors.push(new Rotor(rotorwiring, rotorsteps, args[i*4 + 1], args[i*4+2], args[i*4+3])); } + // Rotors are handled in reverse + rotors.reverse(); const reflector = new Reflector(reflectorstr); let plugboardstrMod = plugboardstr; if (plugboardstrMod === "") { From 765aded208b7f87ffef92d30294ac3edca763a2a Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 28 Feb 2019 17:22:09 +0000 Subject: [PATCH 068/110] Typex: add simple tests --- tests/operations/index.mjs | 1 + tests/operations/tests/Typex.mjs | 105 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 tests/operations/tests/Typex.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index ff967163..cff77217 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -86,6 +86,7 @@ import "./tests/ConvertCoordinateFormat"; import "./tests/Enigma"; import "./tests/Bombe"; import "./tests/MultipleBombe"; +import "./tests/Typex"; // Cannot test operations that use the File type yet //import "./tests/SplitColourChannels"; diff --git a/tests/operations/tests/Typex.mjs b/tests/operations/tests/Typex.mjs new file mode 100644 index 00000000..e3751e8a --- /dev/null +++ b/tests/operations/tests/Typex.mjs @@ -0,0 +1,105 @@ +/** + * Typex machine tests. + * @author s2224834 + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + // Unlike Enigma we're not verifying against a real machine here, so this is just a test + // to catch inadvertent breakage. + name: "Typex: basic", + input: "hello world, this is a test message.", + expectedOutput: "VIXQQ VHLPN UCVLA QDZNZ EAYAT HWC", + recipeConfig: [ + { + "op": "Typex", + "args": [ + "MCYLPQUVRXGSAOWNBJEZDTFKHI Date: Thu, 28 Feb 2019 17:50:10 +0000 Subject: [PATCH 069/110] Add some files that escaped commit before --- package.json | 1 + webpack.config.js | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index cb59db38..64ef09cc 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "sass-loader": "^7.1.0", "sitemap": "^2.1.0", "style-loader": "^0.23.1", + "svg-url-loader": "^2.3.2", "url-loader": "^1.1.2", "web-resource-inliner": "^4.2.1", "webpack": "^4.28.3", diff --git a/webpack.config.js b/webpack.config.js index 054152b2..e2a7c728 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -100,8 +100,15 @@ module.exports = { limit: 10000 } }, + { + test: /\.svg$/, + loader: "svg-url-loader", + options: { + encoding: "base64" + } + }, { // First party images are saved as files to be cached - test: /\.(png|jpg|gif|svg)$/, + test: /\.(png|jpg|gif)$/, exclude: /node_modules/, loader: "file-loader", options: { @@ -109,7 +116,7 @@ module.exports = { } }, { // Third party images are inlined - test: /\.(png|jpg|gif|svg)$/, + test: /\.(png|jpg|gif)$/, exclude: /web\/static/, loader: "url-loader", options: { From 9323737d1da3dc7b9a2d4f485e456829e7aa0e98 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 28 Feb 2019 18:37:48 +0000 Subject: [PATCH 070/110] Bombe: fix rotor listing order for multibombe --- src/core/operations/MultipleBombe.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs index 7a0ae2fd..6887bc46 100644 --- a/src/core/operations/MultipleBombe.mjs +++ b/src/core/operations/MultipleBombe.mjs @@ -291,7 +291,7 @@ class MultipleBombe extends Operation { let html = `Bombe run on menu with ${output.nLoops} loop${output.nLoops === 1 ? "" : "s"} (2+ desirable). Note: Rotors and rotor positions are listed left to right, ignore stepping and the ring setting, and positions start at the beginning of the crib. Some plugboard settings are determined. A decryption preview starting at the beginning of the crib and ignoring stepping is also provided.\n`; for (const run of output.bombeRuns) { - html += `\nRotors: ${run.rotors.join(", ")}\nReflector: ${run.reflector}\n`; + html += `\nRotors: ${run.rotors.slice().reverse().join(", ")}\nReflector: ${run.reflector}\n`; html += ""; for (const [setting, stecker, decrypt] of run.result) { html += `\n`; From a446ec31c712d4a820e2cd484bed97b2a71c9e83 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 28 Feb 2019 18:48:36 +0000 Subject: [PATCH 071/110] Improve Enigma/Bombe descriptions a little. --- src/core/operations/Bombe.mjs | 2 +- src/core/operations/Enigma.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 5e128498..f0d7048c 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -23,7 +23,7 @@ class Bombe extends Operation { this.name = "Bombe"; this.module = "Default"; - this.description = "Emulation of the Bombe machine used to attack Enigma.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and known plugboard pairs.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops, a large number of incorrect outputs will be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor.

By default this operation runs the checking machine, a manual process to verify the quality of Bombe stops, on each stop, discarding stops which fail. If you want to see how many times the hardware actually stops for a given input, disable the checking machine."; + this.description = "Emulation of the Bombe machine used at Bletchley Park to attack Enigma, based on work by Polish and British cryptanalysts.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and known plugboard pairs.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops or is too short, a large number of incorrect outputs will usually be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor.

By default this operation runs the checking machine, a manual process to verify the quality of Bombe stops, on each stop, discarding stops which fail. If you want to see how many times the hardware actually stops for a given input, disable the checking machine."; this.infoURL = "https://wikipedia.org/wiki/Bombe"; this.inputType = "string"; this.outputType = "JSON"; diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs index 77333b18..71593070 100644 --- a/src/core/operations/Enigma.mjs +++ b/src/core/operations/Enigma.mjs @@ -22,7 +22,7 @@ class Enigma extends Operation { this.name = "Enigma"; this.module = "Default"; - this.description = "Encipher/decipher with the WW2 Enigma machine.

The standard set of German military rotors and reflectors are provided. To configure the plugboard, enter a string of connected pairs of letters, e.g. AB CD EF connects A to B, C to D, and E to F. This is also used to create your own reflectors. To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points.
This is deliberately fairly permissive with rotor placements etc compared to a real Enigma (on which, for example, a four-rotor Enigma uses the thin reflectors and the beta or gamma rotor in the 4th slot)."; + this.description = "Encipher/decipher with the WW2 Enigma machine.

Enigma was used by the German military, among others, around the WW2 era as a portable cipher machine to protect sensitive military, diplomatic and commercial communications.

The standard set of German military rotors and reflectors are provided. To configure the plugboard, enter a string of connected pairs of letters, e.g. AB CD EF connects A to B, C to D, and E to F. This is also used to create your own reflectors. To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points.
This is deliberately fairly permissive with rotor placements etc compared to a real Enigma (on which, for example, a four-rotor Enigma uses only the thin reflectors and the beta or gamma rotor in the 4th slot)."; this.infoURL = "https://wikipedia.org/wiki/Enigma_machine"; this.inputType = "string"; this.outputType = "string"; From 9a0b78415360c5d7c6ccf9ea025bedbea74f0d41 Mon Sep 17 00:00:00 2001 From: s2224834 <46319860+s2224834@users.noreply.github.com> Date: Thu, 28 Feb 2019 18:56:59 +0000 Subject: [PATCH 072/110] Typex: improve operation description --- src/core/operations/Typex.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Typex.mjs b/src/core/operations/Typex.mjs index 9c963357..760914f5 100644 --- a/src/core/operations/Typex.mjs +++ b/src/core/operations/Typex.mjs @@ -23,7 +23,7 @@ class Typex extends Operation { this.name = "Typex"; this.module = "Default"; - this.description = "Encipher/decipher with the WW2 Typex machine.

Typex rotors were changed regularly and none are public: a random example set are provided. Later Typexes had a reflector which could be configured with a plugboard: to configure this, enter a string of connected pairs of letters in the reflector box, e.g. AB CD EF connects A to B, C to D, and E to F (you'll need to connect every letter). These Typexes also have an input plugboard: unlike Enigma's plugboard, it's not restricted to pairs, so it's entered like a rotor (without stepping). To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points."; + this.description = "Encipher/decipher with the WW2 Typex machine.

Typex was originally built by the British Royal Air Force prior to WW2, and is based on the Enigma machine with some improvements made, including using five rotors with more stepping points and interchangeable wiring cores. It was used across the British and Commonewealth militaries. A number of later variants were produced; here we simulate a WW2 era Mark 22 Typex with plugboards for the reflector and input. Typex rotors were changed regularly and none are public: a random example set are provided.

To configure the reflector plugboard, enter a string of connected pairs of letters in the reflector box, e.g. AB CD EF connects A to B, C to D, and E to F (you'll need to connect every letter). There is also an input plugboard: unlike Enigma's plugboard, it's not restricted to pairs, so it's entered like a rotor (without stepping). To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points."; this.infoURL = "https://wikipedia.org/wiki/Typex"; this.inputType = "string"; this.outputType = "string"; From 9fa7edffbf1d559e54c31501b7f8be4e7b86448b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 2 Mar 2019 16:12:21 +0000 Subject: [PATCH 073/110] Improved file extraction error handling --- src/core/lib/FileSignatures.mjs | 6 +++--- src/core/operations/ExtractFiles.mjs | 22 +++++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 93247413..36e6818e 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1057,7 +1057,7 @@ export function extractJPEG(bytes, offset) { while (stream.hasMore()) { const marker = stream.getBytes(2); - if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + if (marker[0] !== 0xff) throw new Error(`Invalid marker while parsing JPEG at pos ${stream.position}: ${marker}`); let segmentSize = 0; switch (marker[1]) { @@ -1609,7 +1609,7 @@ function parseDEFLATE(stream) { parseHuffmanBlock(stream, dynamicLiteralTable, dynamicDistanceTable); } else { - throw new Error("Invalid block type"); + throw new Error(`Invalid block type while parsing DEFLATE stream at pos ${stream.position}`); } } @@ -1712,7 +1712,7 @@ function readHuffmanCode(stream, table) { const codeLength = codeWithLength >>> 16; if (codeLength > maxCodeLength) { - throw new Error("Invalid code length: " + codeLength); + throw new Error(`Invalid Huffman Code length while parsing DEFLATE block at pos ${stream.position}: ${codeLength}`); } stream.moveBackwardsByBits(maxCodeLength - codeLength); diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index f172d926..d2b87990 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -5,7 +5,7 @@ */ import Operation from "../Operation"; -// import OperationError from "../errors/OperationError"; +import OperationError from "../errors/OperationError"; import Utils from "../Utils"; import {scanForFileTypes, extractFile} from "../lib/FileType"; import {FILE_SIGNATURES} from "../lib/FileSignatures"; @@ -34,7 +34,13 @@ class ExtractFiles extends Operation { type: "boolean", value: cat === "Miscellaneous" ? false : true }; - }); + }).concat([ + { + name: "Ignore failed extractions", + type: "boolean", + value: "true" + } + ]); } /** @@ -44,7 +50,8 @@ class ExtractFiles extends Operation { */ run(input, args) { const bytes = new Uint8Array(input), - categories = []; + categories = [], + ignoreFailedExtractions = args.pop(1); args.forEach((cat, i) => { if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); @@ -59,8 +66,13 @@ class ExtractFiles extends Operation { try { files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); } catch (err) { - if (err.message.indexOf("No extraction algorithm available") < 0) - throw err; + if (!ignoreFailedExtractions && err.message.indexOf("No extraction algorithm available") < 0) { + throw new OperationError( + `Error while attempting to extract ${detectedFile.fileDetails.name} ` + + `at offset ${detectedFile.offset}:\n` + + `${err.message}` + ); + } } }); From 7975fadfe91ebd27b36c99d8eb54273f58efd648 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 11:46:27 +0000 Subject: [PATCH 074/110] Add options for min, max and step values for number inputs. --- src/core/Ingredient.mjs | 6 ++++++ src/core/Operation.mjs | 3 +++ src/web/HTMLIngredient.mjs | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/src/core/Ingredient.mjs b/src/core/Ingredient.mjs index 96cdd400..2c7154d9 100755 --- a/src/core/Ingredient.mjs +++ b/src/core/Ingredient.mjs @@ -27,6 +27,9 @@ class Ingredient { this.toggleValues = []; this.target = null; this.defaultIndex = 0; + this.min = null; + this.max = null; + this.step = 1; if (ingredientConfig) { this._parseConfig(ingredientConfig); @@ -50,6 +53,9 @@ class Ingredient { this.toggleValues = ingredientConfig.toggleValues; this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null; this.defaultIndex = typeof ingredientConfig.defaultIndex !== "undefined" ? ingredientConfig.defaultIndex : 0; + this.min = ingredientConfig.min; + this.max = ingredientConfig.max; + this.step = ingredientConfig.step; } diff --git a/src/core/Operation.mjs b/src/core/Operation.mjs index c0907fe8..c0656151 100755 --- a/src/core/Operation.mjs +++ b/src/core/Operation.mjs @@ -184,6 +184,9 @@ class Operation { if (ing.disabled) conf.disabled = ing.disabled; if (ing.target) conf.target = ing.target; if (ing.defaultIndex) conf.defaultIndex = ing.defaultIndex; + if (typeof ing.min === "number") conf.min = ing.min; + if (typeof ing.max === "number") conf.max = ing.max; + if (ing.step) conf.step = ing.step; return conf; }); } diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index ab7f682b..98d63be7 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -32,6 +32,9 @@ class HTMLIngredient { this.defaultIndex = config.defaultIndex || 0; this.toggleValues = config.toggleValues; this.id = "ing-" + this.app.nextIngId(); + this.min = (typeof config.min === "number") ? config.min : ""; + this.max = (typeof config.max === "number") ? config.max : ""; + this.step = config.step || 1; } @@ -103,6 +106,9 @@ class HTMLIngredient { id="${this.id}" arg-name="${this.name}" value="${this.value}" + min="${this.min}" + max="${this.max}" + step="${this.step}" ${this.disabled ? "disabled" : ""}> ${this.hint ? "" + this.hint + "" : ""} `; From 7b6062a4a287701cb33e4da7b4a70a306305fdf2 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 11:47:50 +0000 Subject: [PATCH 075/110] Set min blur amount to 1, add status message for gaussian blur. --- src/core/operations/BlurImage.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/operations/BlurImage.mjs b/src/core/operations/BlurImage.mjs index 68ae0b0f..562df8c7 100644 --- a/src/core/operations/BlurImage.mjs +++ b/src/core/operations/BlurImage.mjs @@ -32,7 +32,8 @@ class BlurImage extends Operation { { name: "Blur Amount", type: "number", - value: 5 + value: 5, + min: 1 }, { name: "Blur Type", @@ -59,6 +60,8 @@ class BlurImage extends Operation { image.blur(blurAmount); break; case "Gaussian": + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Gaussian blurring image. This will take a while..."); image.gaussian(blurAmount); break; } From d09e6089cac97e5e19c587d608ef3ffef4c03062 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 11:52:54 +0000 Subject: [PATCH 076/110] Add min width and height values --- src/core/operations/ResizeImage.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index aa5cb24b..59d5b2ac 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -32,12 +32,14 @@ class ResizeImage extends Operation { { name: "Width", type: "number", - value: 100 + value: 100, + min: 1 }, { name: "Height", type: "number", - value: 100 + value: 100, + min: 1 }, { name: "Unit type", From f281a32a4e9342d944f8835ae7fcb407089e9cf4 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 13:48:13 +0000 Subject: [PATCH 077/110] Add Wikipedia URLs --- src/core/operations/BlurImage.mjs | 2 +- src/core/operations/ResizeImage.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/BlurImage.mjs b/src/core/operations/BlurImage.mjs index 562df8c7..000f3677 100644 --- a/src/core/operations/BlurImage.mjs +++ b/src/core/operations/BlurImage.mjs @@ -24,7 +24,7 @@ class BlurImage extends Operation { this.name = "Blur Image"; this.module = "Image"; this.description = "Applies a blur effect to the image.

Gaussian blur is much slower than fast blur, but produces better results."; - this.infoURL = ""; + this.infoURL = "https://wikipedia.org/wiki/Gaussian_blur"; this.inputType = "byteArray"; this.outputType = "byteArray"; this.presentType = "html"; diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index 59d5b2ac..ecba7f55 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -24,7 +24,7 @@ class ResizeImage extends Operation { this.name = "Resize Image"; this.module = "Image"; this.description = "Resizes an image to the specified width and height values."; - this.infoURL = ""; + this.infoURL = "https://wikipedia.org/wiki/Image_scaling"; this.inputType = "byteArray"; this.outputType = "byteArray"; this.presentType = "html"; From 588a8b2a3a2fc6cd1feb9ad2d16207356efbf8e7 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 13:48:29 +0000 Subject: [PATCH 078/110] Fix code syntax --- src/core/operations/RotateImage.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/operations/RotateImage.mjs b/src/core/operations/RotateImage.mjs index 1bab6c98..bbeea5c5 100644 --- a/src/core/operations/RotateImage.mjs +++ b/src/core/operations/RotateImage.mjs @@ -30,9 +30,9 @@ class RotateImage extends Operation { this.presentType = "html"; this.args = [ { - "name": "Rotation amount (degrees)", - "type": "number", - "value": 90 + name: "Rotation amount (degrees)", + type: "number", + value: 90 } ]; } From 4f1a897e1876e62039396c4bbfbfe5e0fa6a53cb Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 13:48:48 +0000 Subject: [PATCH 079/110] Add Crop Image operation --- src/core/config/Categories.json | 3 +- src/core/operations/CropImage.mjs | 139 ++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/CropImage.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 9b0f8249..0ab9b1e5 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -365,7 +365,8 @@ "Blur Image", "Dither Image", "Invert Image", - "Flip Image" + "Flip Image", + "Crop Image" ] }, { diff --git a/src/core/operations/CropImage.mjs b/src/core/operations/CropImage.mjs new file mode 100644 index 00000000..9ccc5ec5 --- /dev/null +++ b/src/core/operations/CropImage.mjs @@ -0,0 +1,139 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Crop Image operation + */ +class CropImage extends Operation { + + /** + * CropImage constructor + */ + constructor() { + super(); + + this.name = "Crop Image"; + this.module = "Image"; + this.description = "Crops an image to the specified region, or automatically crop edges.

Autocrop
Automatically crops same-colour borders from the image.

Autocrop tolerance
A percentage value for the tolerance of colour difference between pixels.

Only autocrop frames
Only crop real frames (all sides must have the same border)

Symmetric autocrop
Force autocrop to be symmetric (top/bottom and left/right are cropped by the same amount)

Autocrop keep border
The number of pixels of border to leave around the image."; + this.infoURL = "https://wikipedia.org/wiki/Cropping_(image)"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "X Position", + type: "number", + value: 0, + min: 0 + }, + { + name: "Y Position", + type: "number", + value: 0, + min: 0 + }, + { + name: "Width", + type: "number", + value: 10, + min: 1 + }, + { + name: "Height", + type: "number", + value: 10, + min: 1 + }, + { + name: "Autocrop", + type: "boolean", + value: false + }, + { + name: "Autocrop tolerance (%)", + type: "number", + value: 0.02, + min: 0, + max: 100, + step: 0.01 + }, + { + name: "Only autocrop frames", + type: "boolean", + value: true + }, + { + name: "Symmetric autocrop", + type: "boolean", + value: false + }, + { + name: "Autocrop keep border (px)", + type: "number", + value: 0, + min: 0 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + // const [firstArg, secondArg] = args; + const [xPos, yPos, width, height, autocrop, autoTolerance, autoFrames, autoSymmetric, autoBorder] = args; + const type = Magic.magicFileType(input); + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + if (autocrop) { + image.autocrop({ + tolerance: (autoTolerance / 100), + cropOnlyFrames: autoFrames, + cropSymmetric: autoSymmetric, + leaveBorder: autoBorder + }); + } else { + image.crop(xPos, yPos, width, height); + } + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the cropped image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default CropImage; From 737ce9939823528ba1c79195a78378f8b8bf7483 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 14:24:57 +0000 Subject: [PATCH 080/110] Add image brightness / contrast operation --- src/core/config/Categories.json | 3 +- .../operations/ImageBrightnessContrast.mjs | 91 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/ImageBrightnessContrast.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 0ab9b1e5..411f980f 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -366,7 +366,8 @@ "Dither Image", "Invert Image", "Flip Image", - "Crop Image" + "Crop Image", + "Image Brightness / Contrast" ] }, { diff --git a/src/core/operations/ImageBrightnessContrast.mjs b/src/core/operations/ImageBrightnessContrast.mjs new file mode 100644 index 00000000..51c61c70 --- /dev/null +++ b/src/core/operations/ImageBrightnessContrast.mjs @@ -0,0 +1,91 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Image Brightness / Contrast operation + */ +class ImageBrightnessContrast extends Operation { + + /** + * ImageBrightnessContrast constructor + */ + constructor() { + super(); + + this.name = "Image Brightness / Contrast"; + this.module = "Image"; + this.description = "Adjust the brightness and contrast of an image."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Brightness", + type: "number", + value: 0, + min: -100, + max: 100 + }, + { + name: "Contrast", + type: "number", + value: 0, + min: -100, + max: 100 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [brightness, contrast] = args; + const type = Magic.magicFileType(input); + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + image.brightness(brightness / 100); + image.contrast(contrast / 100); + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default ImageBrightnessContrast; From ec1fd7b923cf1049be2c908ee25e2c66a2e1be1a Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 14:38:25 +0000 Subject: [PATCH 081/110] Add image opacity operation --- src/core/config/Categories.json | 3 +- src/core/operations/ImageOpacity.mjs | 83 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/ImageOpacity.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 411f980f..78270fb0 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -367,7 +367,8 @@ "Invert Image", "Flip Image", "Crop Image", - "Image Brightness / Contrast" + "Image Brightness / Contrast", + "Image Opacity" ] }, { diff --git a/src/core/operations/ImageOpacity.mjs b/src/core/operations/ImageOpacity.mjs new file mode 100644 index 00000000..11a364b8 --- /dev/null +++ b/src/core/operations/ImageOpacity.mjs @@ -0,0 +1,83 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Image Opacity operation + */ +class ImageOpacity extends Operation { + + /** + * ImageOpacity constructor + */ + constructor() { + super(); + + this.name = "Image Opacity"; + this.module = "Image"; + this.description = "Adjust the opacity of an image."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Opacity (%)", + type: "number", + value: 100, + min: 0, + max: 100 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [opacity] = args; + const type = Magic.magicFileType(input); + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + image.opacity(opacity / 100); + + const imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); + return [...imageBuffer]; + } + + /** + * Displays the image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default ImageOpacity; From 514eef50debdf8a57ee46082e64ab6038f5dd046 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 4 Mar 2019 14:48:17 +0000 Subject: [PATCH 082/110] Add image filter operation --- src/core/config/Categories.json | 3 +- src/core/operations/ImageFilter.mjs | 90 +++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/ImageFilter.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 78270fb0..70390c8d 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -368,7 +368,8 @@ "Flip Image", "Crop Image", "Image Brightness / Contrast", - "Image Opacity" + "Image Opacity", + "Image Filter" ] }, { diff --git a/src/core/operations/ImageFilter.mjs b/src/core/operations/ImageFilter.mjs new file mode 100644 index 00000000..370f5e6f --- /dev/null +++ b/src/core/operations/ImageFilter.mjs @@ -0,0 +1,90 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Image Filter operation + */ +class ImageFilter extends Operation { + + /** + * ImageFilter constructor + */ + constructor() { + super(); + + this.name = "Image Filter"; + this.module = "Image"; + this.description = "Applies a greyscale or sepia filter to an image."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Filter type", + type: "option", + value: [ + "Greyscale", + "Sepia" + ] + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [filterType] = args; + const type = Magic.magicFileType(input); + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + + if (filterType === "Greyscale") { + image.greyscale(); + } else { + image.sepia(); + } + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the blurred image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default ImageFilter; From 370ae323f6f0bac878f7987b35e1b23e9dec8ba2 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 5 Mar 2019 11:49:25 +0000 Subject: [PATCH 083/110] Fix linting --- src/core/operations/ResizeImage.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index ecba7f55..8d46b9cf 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -86,10 +86,11 @@ class ResizeImage extends Operation { "Hermite": jimp.RESIZE_HERMITE, "Bezier": jimp.RESIZE_BEZIER }; - + if (!type || type.mime.indexOf("image") !== 0){ throw new OperationError("Invalid file type."); } + const image = await jimp.read(Buffer.from(input)); if (unit === "Percent") { From 662922be6fd6cb9cd6099444d83d065aeb77adf5 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 6 Mar 2019 10:32:58 +0000 Subject: [PATCH 084/110] Add resizing status message --- src/core/operations/ResizeImage.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index 8d46b9cf..e1ce7d45 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -97,6 +97,9 @@ class ResizeImage extends Operation { width = image.getWidth() * (width / 100); height = image.getHeight() * (height / 100); } + + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Resizing image..."); if (aspect) { image.scaleToFit(width, height, resizeMap[resizeAlg]); } else { From 833c1cd98f8257e130dafd5554e5080bf62c7566 Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 7 Mar 2019 10:02:37 +0000 Subject: [PATCH 085/110] Add Contain Image, Cover Image and Image Hue / Saturation / Lightness ops --- src/core/config/Categories.json | 5 +- src/core/operations/ContainImage.mjs | 140 ++++++++++++++++++ src/core/operations/CoverImage.mjs | 139 +++++++++++++++++ .../ImageHueSaturationLightness.mjs | 126 ++++++++++++++++ 4 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/ContainImage.mjs create mode 100644 src/core/operations/CoverImage.mjs create mode 100644 src/core/operations/ImageHueSaturationLightness.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 70390c8d..8430e498 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -369,7 +369,10 @@ "Crop Image", "Image Brightness / Contrast", "Image Opacity", - "Image Filter" + "Image Filter", + "Contain Image", + "Cover Image", + "Image Hue/Saturation/Lightness" ] }, { diff --git a/src/core/operations/ContainImage.mjs b/src/core/operations/ContainImage.mjs new file mode 100644 index 00000000..056244df --- /dev/null +++ b/src/core/operations/ContainImage.mjs @@ -0,0 +1,140 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Contain Image operation + */ +class ContainImage extends Operation { + + /** + * ContainImage constructor + */ + constructor() { + super(); + + this.name = "Contain Image"; + this.module = "Image"; + this.description = "Scales an image to the specified width and height, maintaining the aspect ratio. The image may be letterboxed."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Width", + type: "number", + value: 100, + min: 1 + }, + { + name: "Height", + type: "number", + value: 100, + min: 1 + }, + { + name: "Horizontal align", + type: "option", + value: [ + "Left", + "Center", + "Right" + ], + defaultIndex: 1 + }, + { + name: "Vertical align", + type: "option", + value: [ + "Top", + "Middle", + "Bottom" + ], + defaultIndex: 1 + }, + { + name: "Resizing algorithm", + type: "option", + value: [ + "Nearest Neighbour", + "Bilinear", + "Bicubic", + "Hermite", + "Bezier" + ], + defaultIndex: 1 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [width, height, hAlign, vAlign, alg] = args; + const type = Magic.magicFileType(input); + + const resizeMap = { + "Nearest Neighbour": jimp.RESIZE_NEAREST_NEIGHBOR, + "Bilinear": jimp.RESIZE_BILINEAR, + "Bicubic": jimp.RESIZE_BICUBIC, + "Hermite": jimp.RESIZE_HERMITE, + "Bezier": jimp.RESIZE_BEZIER + }; + + const alignMap = { + "Left": jimp.HORIZONTAL_ALIGN_LEFT, + "Center": jimp.HORIZONTAL_ALIGN_CENTER, + "Right": jimp.HORIZONTAL_ALIGN_RIGHT, + "Top": jimp.VERTICAL_ALIGN_TOP, + "Middle": jimp.VERTICAL_ALIGN_MIDDLE, + "Bottom": jimp.VERTICAL_ALIGN_BOTTOM + }; + + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Containing image..."); + image.contain(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the contained image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default ContainImage; diff --git a/src/core/operations/CoverImage.mjs b/src/core/operations/CoverImage.mjs new file mode 100644 index 00000000..57258ec3 --- /dev/null +++ b/src/core/operations/CoverImage.mjs @@ -0,0 +1,139 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Cover Image operation + */ +class CoverImage extends Operation { + + /** + * CoverImage constructor + */ + constructor() { + super(); + + this.name = "Cover Image"; + this.module = "Image"; + this.description = "Scales the image to the given width and height, keeping the aspect ratio. The image may be clipped."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Width", + type: "number", + value: 100, + min: 1 + }, + { + name: "Height", + type: "number", + value: 100, + min: 1 + }, + { + name: "Horizontal align", + type: "option", + value: [ + "Left", + "Center", + "Right" + ], + defaultIndex: 1 + }, + { + name: "Vertical align", + type: "option", + value: [ + "Top", + "Middle", + "Bottom" + ], + defaultIndex: 1 + }, + { + name: "Resizing algorithm", + type: "option", + value: [ + "Nearest Neighbour", + "Bilinear", + "Bicubic", + "Hermite", + "Bezier" + ], + defaultIndex: 1 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [width, height, hAlign, vAlign, alg] = args; + const type = Magic.magicFileType(input); + + const resizeMap = { + "Nearest Neighbour": jimp.RESIZE_NEAREST_NEIGHBOR, + "Bilinear": jimp.RESIZE_BILINEAR, + "Bicubic": jimp.RESIZE_BICUBIC, + "Hermite": jimp.RESIZE_HERMITE, + "Bezier": jimp.RESIZE_BEZIER + }; + + const alignMap = { + "Left": jimp.HORIZONTAL_ALIGN_LEFT, + "Center": jimp.HORIZONTAL_ALIGN_CENTER, + "Right": jimp.HORIZONTAL_ALIGN_RIGHT, + "Top": jimp.VERTICAL_ALIGN_TOP, + "Middle": jimp.VERTICAL_ALIGN_MIDDLE, + "Bottom": jimp.VERTICAL_ALIGN_BOTTOM + }; + + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Covering image..."); + image.cover(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the covered image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } + +} + +export default CoverImage; diff --git a/src/core/operations/ImageHueSaturationLightness.mjs b/src/core/operations/ImageHueSaturationLightness.mjs new file mode 100644 index 00000000..29293fdb --- /dev/null +++ b/src/core/operations/ImageHueSaturationLightness.mjs @@ -0,0 +1,126 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64.mjs"; +import jimp from "jimp"; + +/** + * Image Hue/Saturation/Lightness operation + */ +class ImageHueSaturationLightness extends Operation { + + /** + * ImageHueSaturationLightness constructor + */ + constructor() { + super(); + + this.name = "Image Hue/Saturation/Lightness"; + this.module = "Image"; + this.description = "Adjusts the hue / saturation / lightness (HSL) values of an image."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + name: "Hue", + type: "number", + value: 0, + min: -360, + max: 360 + }, + { + name: "Saturation", + type: "number", + value: 0, + min: -100, + max: 100 + }, + { + name: "Lightness", + type: "number", + value: 0, + min: -100, + max: 100 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + const [hue, saturation, lightness] = args; + const type = Magic.magicFileType(input); + + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + + if (hue !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image hue..."); + image.colour([ + { + apply: "hue", + params: [hue] + } + ]); + } + if (saturation !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image saturation..."); + image.colour([ + { + apply: "saturate", + params: [saturation] + } + ]); + } + if (lightness !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image lightness..."); + image.colour([ + { + apply: "lighten", + params: [lightness] + } + ]); + } + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type"); + } + dataURI += "base64," + toBase64(data); + + return ""; + } +} + +export default ImageHueSaturationLightness; From 4a7ea469d483e906bd032fd272e9047f52d3b207 Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 7 Mar 2019 10:03:09 +0000 Subject: [PATCH 086/110] Add status messages for image operations --- src/core/operations/CropImage.mjs | 2 ++ src/core/operations/DitherImage.mjs | 2 ++ src/core/operations/FlipImage.mjs | 2 ++ src/core/operations/ImageBrightnessContrast.mjs | 12 ++++++++++-- src/core/operations/ImageFilter.mjs | 3 ++- src/core/operations/ImageOpacity.mjs | 2 ++ src/core/operations/InvertImage.mjs | 2 ++ src/core/operations/RotateImage.mjs | 2 ++ 8 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/core/operations/CropImage.mjs b/src/core/operations/CropImage.mjs index 9ccc5ec5..e29db631 100644 --- a/src/core/operations/CropImage.mjs +++ b/src/core/operations/CropImage.mjs @@ -99,6 +99,8 @@ class CropImage extends Operation { } const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Cropping image..."); if (autocrop) { image.autocrop({ tolerance: (autoTolerance / 100), diff --git a/src/core/operations/DitherImage.mjs b/src/core/operations/DitherImage.mjs index 2cc9ac2d..e6856d4a 100644 --- a/src/core/operations/DitherImage.mjs +++ b/src/core/operations/DitherImage.mjs @@ -41,6 +41,8 @@ class DitherImage extends Operation { if (type && type.mime.indexOf("image") === 0){ const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Applying dither to image..."); image.dither565(); const imageBuffer = await image.getBufferAsync(jimp.AUTO); return [...imageBuffer]; diff --git a/src/core/operations/FlipImage.mjs b/src/core/operations/FlipImage.mjs index fa3054e2..3185df9f 100644 --- a/src/core/operations/FlipImage.mjs +++ b/src/core/operations/FlipImage.mjs @@ -51,6 +51,8 @@ class FlipImage extends Operation { const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Flipping image..."); switch (flipAxis){ case "Horizontal": image.flip(true, false); diff --git a/src/core/operations/ImageBrightnessContrast.mjs b/src/core/operations/ImageBrightnessContrast.mjs index 51c61c70..7d8eca4f 100644 --- a/src/core/operations/ImageBrightnessContrast.mjs +++ b/src/core/operations/ImageBrightnessContrast.mjs @@ -59,8 +59,16 @@ class ImageBrightnessContrast extends Operation { } const image = await jimp.read(Buffer.from(input)); - image.brightness(brightness / 100); - image.contrast(contrast / 100); + if (brightness !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image brightness..."); + image.brightness(brightness / 100); + } + if (contrast !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image contrast..."); + image.contrast(contrast / 100); + } const imageBuffer = await image.getBufferAsync(jimp.AUTO); return [...imageBuffer]; diff --git a/src/core/operations/ImageFilter.mjs b/src/core/operations/ImageFilter.mjs index 370f5e6f..b756b9f2 100644 --- a/src/core/operations/ImageFilter.mjs +++ b/src/core/operations/ImageFilter.mjs @@ -53,7 +53,8 @@ class ImageFilter extends Operation { } const image = await jimp.read(Buffer.from(input)); - + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Applying " + filterType.toLowerCase() + " filter to image..."); if (filterType === "Greyscale") { image.greyscale(); } else { diff --git a/src/core/operations/ImageOpacity.mjs b/src/core/operations/ImageOpacity.mjs index 11a364b8..090a8975 100644 --- a/src/core/operations/ImageOpacity.mjs +++ b/src/core/operations/ImageOpacity.mjs @@ -52,6 +52,8 @@ class ImageOpacity extends Operation { } const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image opacity..."); image.opacity(opacity / 100); const imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); diff --git a/src/core/operations/InvertImage.mjs b/src/core/operations/InvertImage.mjs index 87da0156..99de9f0f 100644 --- a/src/core/operations/InvertImage.mjs +++ b/src/core/operations/InvertImage.mjs @@ -42,6 +42,8 @@ class InvertImage extends Operation { throw new OperationError("Invalid input file format."); } const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Inverting image..."); image.invert(); const imageBuffer = await image.getBufferAsync(jimp.AUTO); return [...imageBuffer]; diff --git a/src/core/operations/RotateImage.mjs b/src/core/operations/RotateImage.mjs index bbeea5c5..76947037 100644 --- a/src/core/operations/RotateImage.mjs +++ b/src/core/operations/RotateImage.mjs @@ -48,6 +48,8 @@ class RotateImage extends Operation { if (type && type.mime.indexOf("image") === 0){ const image = await jimp.read(Buffer.from(input)); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Rotating image..."); image.rotate(degrees); const imageBuffer = await image.getBufferAsync(jimp.AUTO); return [...imageBuffer]; From 1031429550e69f63373e1cc2a5fde2118328c9b3 Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 7 Mar 2019 11:19:04 +0000 Subject: [PATCH 087/110] Add error handling --- src/core/operations/BlurImage.mjs | 34 ++++--- src/core/operations/ContainImage.mjs | 22 +++-- src/core/operations/CoverImage.mjs | 21 +++-- src/core/operations/CropImage.mjs | 37 +++++--- src/core/operations/DitherImage.mjs | 21 +++-- src/core/operations/FlipImage.mjs | 34 ++++--- .../operations/ImageBrightnessContrast.mjs | 33 ++++--- src/core/operations/ImageFilter.mjs | 27 ++++-- .../ImageHueSaturationLightness.mjs | 72 ++++++++------- src/core/operations/ImageOpacity.mjs | 21 +++-- src/core/operations/InvertImage.mjs | 22 +++-- src/core/operations/NormaliseImage.mjs | 91 +++++++++++++++++++ src/core/operations/ResizeImage.mjs | 36 +++++--- src/core/operations/RotateImage.mjs | 21 +++-- 14 files changed, 348 insertions(+), 144 deletions(-) create mode 100644 src/core/operations/NormaliseImage.mjs diff --git a/src/core/operations/BlurImage.mjs b/src/core/operations/BlurImage.mjs index 000f3677..fba3c927 100644 --- a/src/core/operations/BlurImage.mjs +++ b/src/core/operations/BlurImage.mjs @@ -53,21 +53,29 @@ class BlurImage extends Operation { const type = Magic.magicFileType(input); if (type && type.mime.indexOf("image") === 0){ - const image = await jimp.read(Buffer.from(input)); - - switch (blurType){ - case "Fast": - image.blur(blurAmount); - break; - case "Gaussian": - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Gaussian blurring image. This will take a while..."); - image.gaussian(blurAmount); - break; + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } + try { + switch (blurType){ + case "Fast": + image.blur(blurAmount); + break; + case "Gaussian": + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Gaussian blurring image. This will take a while..."); + image.gaussian(blurAmount); + break; + } - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error blurring image. (${err})`); + } } else { throw new OperationError("Invalid file type."); } diff --git a/src/core/operations/ContainImage.mjs b/src/core/operations/ContainImage.mjs index 056244df..a2da5363 100644 --- a/src/core/operations/ContainImage.mjs +++ b/src/core/operations/ContainImage.mjs @@ -106,13 +106,21 @@ class ContainImage extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Containing image..."); - image.contain(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]); - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); + } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Containing image..."); + image.contain(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error containing image. (${err})`); + } } /** diff --git a/src/core/operations/CoverImage.mjs b/src/core/operations/CoverImage.mjs index 57258ec3..f49e08b7 100644 --- a/src/core/operations/CoverImage.mjs +++ b/src/core/operations/CoverImage.mjs @@ -106,12 +106,21 @@ class CoverImage extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Covering image..."); - image.cover(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]); - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); + } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Covering image..."); + image.cover(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error covering image. (${err})`); + } } /** diff --git a/src/core/operations/CropImage.mjs b/src/core/operations/CropImage.mjs index e29db631..7f1eabdf 100644 --- a/src/core/operations/CropImage.mjs +++ b/src/core/operations/CropImage.mjs @@ -98,22 +98,31 @@ class CropImage extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Cropping image..."); - if (autocrop) { - image.autocrop({ - tolerance: (autoTolerance / 100), - cropOnlyFrames: autoFrames, - cropSymmetric: autoSymmetric, - leaveBorder: autoBorder - }); - } else { - image.crop(xPos, yPos, width, height); + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Cropping image..."); + if (autocrop) { + image.autocrop({ + tolerance: (autoTolerance / 100), + cropOnlyFrames: autoFrames, + cropSymmetric: autoSymmetric, + leaveBorder: autoBorder + }); + } else { + image.crop(xPos, yPos, width, height); + } - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error cropping image. (${err})`); + } } /** diff --git a/src/core/operations/DitherImage.mjs b/src/core/operations/DitherImage.mjs index e6856d4a..f7ef4e33 100644 --- a/src/core/operations/DitherImage.mjs +++ b/src/core/operations/DitherImage.mjs @@ -40,12 +40,21 @@ class DitherImage extends Operation { const type = Magic.magicFileType(input); if (type && type.mime.indexOf("image") === 0){ - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Applying dither to image..."); - image.dither565(); - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); + } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Applying dither to image..."); + image.dither565(); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error applying dither to image. (${err})`); + } } else { throw new OperationError("Invalid file type."); } diff --git a/src/core/operations/FlipImage.mjs b/src/core/operations/FlipImage.mjs index 3185df9f..09791ca6 100644 --- a/src/core/operations/FlipImage.mjs +++ b/src/core/operations/FlipImage.mjs @@ -49,21 +49,29 @@ class FlipImage extends Operation { throw new OperationError("Invalid input file type."); } - const image = await jimp.read(Buffer.from(input)); - - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Flipping image..."); - switch (flipAxis){ - case "Horizontal": - image.flip(true, false); - break; - case "Vertical": - image.flip(false, true); - break; + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Flipping image..."); + switch (flipAxis){ + case "Horizontal": + image.flip(true, false); + break; + case "Vertical": + image.flip(false, true); + break; + } - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error flipping image. (${err})`); + } } /** diff --git a/src/core/operations/ImageBrightnessContrast.mjs b/src/core/operations/ImageBrightnessContrast.mjs index 7d8eca4f..2f49bab7 100644 --- a/src/core/operations/ImageBrightnessContrast.mjs +++ b/src/core/operations/ImageBrightnessContrast.mjs @@ -58,20 +58,29 @@ class ImageBrightnessContrast extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - if (brightness !== 0) { - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Changing image brightness..."); - image.brightness(brightness / 100); - } - if (contrast !== 0) { - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Changing image contrast..."); - image.contrast(contrast / 100); + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } + try { + if (brightness !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image brightness..."); + image.brightness(brightness / 100); + } + if (contrast !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image contrast..."); + image.contrast(contrast / 100); + } - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error adjusting image brightness / contrast. (${err})`); + } } /** diff --git a/src/core/operations/ImageFilter.mjs b/src/core/operations/ImageFilter.mjs index b756b9f2..5d7f505d 100644 --- a/src/core/operations/ImageFilter.mjs +++ b/src/core/operations/ImageFilter.mjs @@ -52,17 +52,26 @@ class ImageFilter extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Applying " + filterType.toLowerCase() + " filter to image..."); - if (filterType === "Greyscale") { - image.greyscale(); - } else { - image.sepia(); + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Applying " + filterType.toLowerCase() + " filter to image..."); + if (filterType === "Greyscale") { + image.greyscale(); + } else { + image.sepia(); + } - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error applying filter to image. (${err})`); + } } /** diff --git a/src/core/operations/ImageHueSaturationLightness.mjs b/src/core/operations/ImageHueSaturationLightness.mjs index 29293fdb..9e63a6b3 100644 --- a/src/core/operations/ImageHueSaturationLightness.mjs +++ b/src/core/operations/ImageHueSaturationLightness.mjs @@ -66,40 +66,48 @@ class ImageHueSaturationLightness extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - - if (hue !== 0) { - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Changing image hue..."); - image.colour([ - { - apply: "hue", - params: [hue] - } - ]); + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } - if (saturation !== 0) { - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Changing image saturation..."); - image.colour([ - { - apply: "saturate", - params: [saturation] - } - ]); + try { + if (hue !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image hue..."); + image.colour([ + { + apply: "hue", + params: [hue] + } + ]); + } + if (saturation !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image saturation..."); + image.colour([ + { + apply: "saturate", + params: [saturation] + } + ]); + } + if (lightness !== 0) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image lightness..."); + image.colour([ + { + apply: "lighten", + params: [lightness] + } + ]); + } + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error adjusting image hue / saturation / lightness. (${err})`); } - if (lightness !== 0) { - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Changing image lightness..."); - image.colour([ - { - apply: "lighten", - params: [lightness] - } - ]); - } - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; } /** diff --git a/src/core/operations/ImageOpacity.mjs b/src/core/operations/ImageOpacity.mjs index 090a8975..76a23f77 100644 --- a/src/core/operations/ImageOpacity.mjs +++ b/src/core/operations/ImageOpacity.mjs @@ -51,13 +51,22 @@ class ImageOpacity extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Changing image opacity..."); - image.opacity(opacity / 100); + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); + } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Changing image opacity..."); + image.opacity(opacity / 100); - const imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); - return [...imageBuffer]; + const imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); + return [...imageBuffer]; + } catch (err) { + throw new OperateionError(`Error changing image opacity. (${err})`); + } } /** diff --git a/src/core/operations/InvertImage.mjs b/src/core/operations/InvertImage.mjs index 99de9f0f..c2625d9a 100644 --- a/src/core/operations/InvertImage.mjs +++ b/src/core/operations/InvertImage.mjs @@ -41,12 +41,22 @@ class InvertImage extends Operation { if (!type || type.mime.indexOf("image") !== 0) { throw new OperationError("Invalid input file format."); } - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Inverting image..."); - image.invert(); - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); + } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Inverting image..."); + image.invert(); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error inverting image. (${err})`); + } } /** diff --git a/src/core/operations/NormaliseImage.mjs b/src/core/operations/NormaliseImage.mjs new file mode 100644 index 00000000..1815c7f1 --- /dev/null +++ b/src/core/operations/NormaliseImage.mjs @@ -0,0 +1,91 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import { toBase64 } from "../lib/Base64"; +import jimp from "jimp"; + +/** + * Normalise Image operation + */ +class NormaliseImage extends Operation { + + /** + * NormaliseImage constructor + */ + constructor() { + super(); + + this.name = "Normalise Image"; + this.module = "Image"; + this.description = "Normalise the image colours."; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.presentType= "html"; + this.args = [ + /* Example arguments. See the project wiki for full details. + { + name: "First arg", + type: "string", + value: "Don't Panic" + }, + { + name: "Second arg", + type: "number", + value: 42 + } + */ + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + async run(input, args) { + // const [firstArg, secondArg] = args; + const type = Magic.magicFileType(input); + + if (!type || type.mime.indexOf("image") !== 0){ + throw new OperationError("Invalid file type."); + } + + const image = await jimp.read(Buffer.from(input)); + + image.normalize(); + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } + + /** + * Displays the normalised image using HTML for web apps + * @param {byteArray} data + * @returns {html} + */ + present(data) { + if (!data.length) return ""; + + let dataURI = "data:"; + const type = Magic.magicFileType(data); + if (type && type.mime.indexOf("image") === 0){ + dataURI += type.mime + ";"; + } else { + throw new OperationError("Invalid file type."); + } + dataURI += "base64," + toBase64(data); + + return ""; + + } + +} + +export default NormaliseImage; diff --git a/src/core/operations/ResizeImage.mjs b/src/core/operations/ResizeImage.mjs index e1ce7d45..36b0c805 100644 --- a/src/core/operations/ResizeImage.mjs +++ b/src/core/operations/ResizeImage.mjs @@ -91,23 +91,31 @@ class ResizeImage extends Operation { throw new OperationError("Invalid file type."); } - const image = await jimp.read(Buffer.from(input)); - - if (unit === "Percent") { - width = image.getWidth() * (width / 100); - height = image.getHeight() * (height / 100); + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); } + try { + if (unit === "Percent") { + width = image.getWidth() * (width / 100); + height = image.getHeight() * (height / 100); + } - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Resizing image..."); - if (aspect) { - image.scaleToFit(width, height, resizeMap[resizeAlg]); - } else { - image.resize(width, height, resizeMap[resizeAlg]); + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Resizing image..."); + if (aspect) { + image.scaleToFit(width, height, resizeMap[resizeAlg]); + } else { + image.resize(width, height, resizeMap[resizeAlg]); + } + + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error resizing image. (${err})`); } - - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; } /** diff --git a/src/core/operations/RotateImage.mjs b/src/core/operations/RotateImage.mjs index 76947037..b2b1e059 100644 --- a/src/core/operations/RotateImage.mjs +++ b/src/core/operations/RotateImage.mjs @@ -47,12 +47,21 @@ class RotateImage extends Operation { const type = Magic.magicFileType(input); if (type && type.mime.indexOf("image") === 0){ - const image = await jimp.read(Buffer.from(input)); - if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Rotating image..."); - image.rotate(degrees); - const imageBuffer = await image.getBufferAsync(jimp.AUTO); - return [...imageBuffer]; + let image; + try { + image = await jimp.read(Buffer.from(input)); + } catch (err) { + throw new OperationError(`Error loading image. (${err})`); + } + try { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Rotating image..."); + image.rotate(degrees); + const imageBuffer = await image.getBufferAsync(jimp.AUTO); + return [...imageBuffer]; + } catch (err) { + throw new OperationError(`Error rotating image. (${err})`); + } } else { throw new OperationError("Invalid file type."); } From 0c9db5afe9e3dff8f70f05a634326d036b15a397 Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 7 Mar 2019 11:36:29 +0000 Subject: [PATCH 088/110] Fix typo --- src/core/operations/ImageOpacity.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ImageOpacity.mjs b/src/core/operations/ImageOpacity.mjs index 76a23f77..5a547992 100644 --- a/src/core/operations/ImageOpacity.mjs +++ b/src/core/operations/ImageOpacity.mjs @@ -65,7 +65,7 @@ class ImageOpacity extends Operation { const imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); return [...imageBuffer]; } catch (err) { - throw new OperateionError(`Error changing image opacity. (${err})`); + throw new OperationError(`Error changing image opacity. (${err})`); } } From 2b538061e940c70bc3446a9bc772a83e5cb81b2b Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 7 Mar 2019 16:26:42 +0000 Subject: [PATCH 089/110] Fix fork operation not setting ingredient values correctly. --- src/core/operations/Fork.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Fork.mjs b/src/core/operations/Fork.mjs index 27a1af96..02aba3e8 100644 --- a/src/core/operations/Fork.mjs +++ b/src/core/operations/Fork.mjs @@ -89,7 +89,7 @@ class Fork extends Operation { // Run recipe over each tranche for (i = 0; i < inputs.length; i++) { // Baseline ing values for each tranche so that registers are reset - subOpList.forEach((op, i) => { + recipe.opList.forEach((op, i) => { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); }); From d923c99975b87fc7869ea26c6c1e4ce9981becb0 Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 7 Mar 2019 16:33:38 +0000 Subject: [PATCH 090/110] Fix same bug in subsection --- src/core/operations/Subsection.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs index 8133d31c..548780c8 100644 --- a/src/core/operations/Subsection.mjs +++ b/src/core/operations/Subsection.mjs @@ -116,7 +116,7 @@ class Subsection extends Operation { } // Baseline ing values for each tranche so that registers are reset - subOpList.forEach((op, i) => { + recipe.opList.forEach((op, i) => { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); }); From 58d41f4458b4f442cf10e10b3bce9e95a7121366 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 9 Mar 2019 05:38:13 +0000 Subject: [PATCH 091/110] 8.24.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0d1cb4e1..8a35ebb1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.24.2", + "version": "8.24.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a14af274..f8db4aa5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.24.2", + "version": "8.24.3", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 84d31c1d597921ade565f30e60b887f2cc20ed4c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 9 Mar 2019 06:25:27 +0000 Subject: [PATCH 092/110] Added 'Move to input' button to output file list. Improved zlib extraction efficiency. --- .eslintrc.json | 1 + src/core/Utils.mjs | 31 +++++++++++++-- src/core/lib/FileSignatures.mjs | 50 +++++++++++++----------- src/core/lib/FileType.mjs | 7 +++- src/core/operations/ExtractFiles.mjs | 8 +++- src/web/Manager.mjs | 1 + src/web/OutputWaiter.mjs | 18 +++++++++ src/web/html/index.html | 2 +- src/web/stylesheets/components/_pane.css | 4 ++ 9 files changed, 94 insertions(+), 28 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index d5e4e768..7dcb705c 100755 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -102,6 +102,7 @@ "$": false, "jQuery": false, "log": false, + "app": false, "COMPILE_TIME": false, "COMPILE_MSG": false, diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index f70e2941..8e69b020 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -832,8 +832,9 @@ class Utils { const buff = await Utils.readFile(file); const blob = new Blob( [buff], - {type: "octet/stream"} + {type: file.type || "octet/stream"} ); + const blobURL = URL.createObjectURL(blob); const html = `
@@ -1163,6 +1173,21 @@ String.prototype.count = function(chr) { }; +/** + * Wrapper for self.sendStatusMessage to handle different environments. + * + * @param {string} msg + */ +export function sendStatusMessage(msg) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage(msg); + else if (ENVIRONMENT_IS_WEB()) + app.alert(msg, 10000); + else if (ENVIRONMENT_IS_NODE()) + log.debug(msg); +} + + /* * Polyfills */ diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 36e6818e..61e37b88 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1518,26 +1518,26 @@ export function extractELF(bytes, offset) { } +// Construct required Huffman Tables +const fixedLiteralTableLengths = new Array(288); +for (let i = 0; i < fixedLiteralTableLengths.length; i++) { + fixedLiteralTableLengths[i] = + (i <= 143) ? 8 : + (i <= 255) ? 9 : + (i <= 279) ? 7 : + 8; +} +const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths); +const fixedDistanceTableLengths = new Array(30).fill(5); +const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths); +const huffmanOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]; + /** * Steps through a DEFLATE stream * * @param {Stream} stream */ function parseDEFLATE(stream) { - // Construct required Huffman Tables - const fixedLiteralTableLengths = new Uint8Array(288); - for (let i = 0; i < fixedLiteralTableLengths.length; i++) { - fixedLiteralTableLengths[i] = - (i <= 143) ? 8 : - (i <= 255) ? 9 : - (i <= 279) ? 7 : - 8; - } - const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths); - const fixedDistanceTableLengths = new Uint8Array(30).fill(5); - const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths); - const huffmanOrder = new Uint8Array([16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]); - // Parse DEFLATE data let finalBlock = 0; @@ -1619,6 +1619,14 @@ function parseDEFLATE(stream) { } +// Static length tables +const lengthExtraTable = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 +]; +const distanceExtraTable = [ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 +]; + /** * Parses a Huffman Block given the literal and distance tables * @@ -1627,20 +1635,18 @@ function parseDEFLATE(stream) { * @param {Uint32Array} distTab */ function parseHuffmanBlock(stream, litTab, distTab) { - const lengthExtraTable = new Uint8Array([ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 - ]); - const distanceExtraTable = new Uint8Array([ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 - ]); - let code; + let loops = 0; while ((code = readHuffmanCode(stream, litTab))) { // console.log("Code: " + code + " (" + Utils.chr(code) + ") " + Utils.bin(code)); // End of block if (code === 256) break; + // Detect probably infinite loops + if (++loops > 10000) + throw new Error("Caught in probable infinite loop while parsing Huffman Block"); + // Literal if (code < 256) continue; @@ -1657,7 +1663,7 @@ function parseHuffmanBlock(stream, litTab, distTab) { /** * Builds a Huffman table given the relevant code lengths * - * @param {Uint8Array} lengths + * @param {Array} lengths * @returns {Array} result * @returns {Uint32Array} result.table * @returns {number} result.maxCodeLength diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index e5d990d9..e961a76f 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -7,6 +7,7 @@ * */ import {FILE_SIGNATURES} from "./FileSignatures"; +import {sendStatusMessage} from "../Utils"; /** @@ -148,6 +149,7 @@ export function scanForFileTypes(buf, categories=Object.keys(FILE_SIGNATURES)) { let pos = 0; while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) { if (bytesMatch(sig, buf, pos)) { + sendStatusMessage(`Found potential signature for ${filetype.name} at pos ${pos}`); foundFiles.push({ offset: pos, fileDetails: filetype @@ -249,9 +251,12 @@ export function isImage(buf) { */ export function extractFile(bytes, fileDetail, offset) { if (fileDetail.extractor) { + sendStatusMessage(`Attempting to extract ${fileDetail.name} at pos ${offset}...`); const fileData = fileDetail.extractor(bytes, offset); const ext = fileDetail.extension.split(",")[0]; - return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`); + return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`, { + type: fileDetail.mime + }); } throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index d2b87990..b9b260bb 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -62,12 +62,13 @@ class ExtractFiles extends Operation { // Extract each file that we support const files = []; + const errors = []; detectedFiles.forEach(detectedFile => { try { files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); } catch (err) { if (!ignoreFailedExtractions && err.message.indexOf("No extraction algorithm available") < 0) { - throw new OperationError( + errors.push( `Error while attempting to extract ${detectedFile.fileDetails.name} ` + `at offset ${detectedFile.offset}:\n` + `${err.message}` @@ -76,9 +77,14 @@ class ExtractFiles extends Operation { } }); + if (errors.length) { + throw new OperationError(errors.join("\n\n")); + } + return files; } + /** * Displays the files in HTML for web apps. * diff --git a/src/web/Manager.mjs b/src/web/Manager.mjs index 30cb4943..5fa0e8c1 100755 --- a/src/web/Manager.mjs +++ b/src/web/Manager.mjs @@ -173,6 +173,7 @@ class Manager { this.addDynamicListener("#output-file-download", "click", this.output.downloadFile, this.output); this.addDynamicListener("#output-file-slice i", "click", this.output.displayFileSlice, this.output); document.getElementById("show-file-overlay").addEventListener("click", this.output.showFileOverlayClick.bind(this.output)); + this.addDynamicListener(".extract-file,.extract-file i", "click", this.output.extractFileClick, this.output); // Options document.getElementById("options").addEventListener("click", this.options.optionsClick.bind(this.options)); diff --git a/src/web/OutputWaiter.mjs b/src/web/OutputWaiter.mjs index 2d93507c..0a10b8b2 100755 --- a/src/web/OutputWaiter.mjs +++ b/src/web/OutputWaiter.mjs @@ -494,6 +494,24 @@ class OutputWaiter { magicButton.setAttribute("data-original-title", "Magic!"); } + + /** + * Handler for extract file events. + * + * @param {Event} e + */ + async extractFileClick(e) { + e.preventDefault(); + e.stopPropagation(); + + const el = e.target.nodeName === "I" ? e.target.parentNode : e.target; + const blobURL = el.getAttribute("blob-url"); + const fileName = el.getAttribute("file-name"); + + const blob = await fetch(blobURL).then(r => r.blob()); + this.manager.input.loadFile(new File([blob], fileName, {type: blob.type})); + } + } export default OutputWaiter; diff --git a/src/web/html/index.html b/src/web/html/index.html index 74eb0ed8..302355d9 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -271,7 +271,7 @@ content_copy
Rotor stopsPartial plugboardDecryption preview
${setting}${stecker}${decrypt}
"; + html += "
Rotor stopsPartial plugboardDecryption preview
\n"; for (const [setting, stecker, decrypt] of output.result) { - html += `\n`; + html += `\n`; } html += "
Rotor stops Partial plugboard Decryption preview
${setting}${stecker}${decrypt}
${setting} ${stecker} ${decrypt}
"; return html; diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs index 6887bc46..03364a01 100644 --- a/src/core/operations/MultipleBombe.mjs +++ b/src/core/operations/MultipleBombe.mjs @@ -292,9 +292,9 @@ class MultipleBombe extends Operation { for (const run of output.bombeRuns) { html += `\nRotors: ${run.rotors.slice().reverse().join(", ")}\nReflector: ${run.reflector}\n`; - html += ""; + html += "
Rotor stopsPartial plugboardDecryption preview
\n"; for (const [setting, stecker, decrypt] of run.result) { - html += `\n`; + html += `\n`; } html += "
Rotor stops Partial plugboard Decryption preview
${setting}${stecker}${decrypt}
${setting} ${stecker} ${decrypt}
\n"; } diff --git a/tests/operations/tests/Bombe.mjs b/tests/operations/tests/Bombe.mjs index 9e5a79c6..b44e032c 100644 --- a/tests/operations/tests/Bombe.mjs +++ b/tests/operations/tests/Bombe.mjs @@ -11,7 +11,7 @@ TestRegister.addTests([ // Plugboard for this test is BO LC KE GA name: "Bombe: 3 rotor (self-stecker)", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA<\/td>SS<\/td>VFISUSGTKSTMPSUNAK<\/td>/, + expectedMatch: /LGA<\/td> {2}SS<\/td> {2}VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -31,7 +31,7 @@ TestRegister.addTests([ // This test produces a menu that doesn't use the first letter, which is also a good test name: "Bombe: 3 rotor (other stecker)", input: "JBYALIHDYNUAAVKBYM", - expectedMatch: /LGA<\/td>AG<\/td>QFIMUMAFKMQSKMYNGW<\/td>/, + expectedMatch: /LGA<\/td> {2}AG<\/td> {2}QFIMUMAFKMQSKMYNGW<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -50,7 +50,7 @@ TestRegister.addTests([ { name: "Bombe: crib offset", input: "AAABBYFLTHHYIJQAYBBYS", // first three chars here are faked - expectedMatch: /LGA<\/td>SS<\/td>VFISUSGTKSTMPSUNAK<\/td>/, + expectedMatch: /LGA<\/td> {2}SS<\/td> {2}VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -69,7 +69,7 @@ TestRegister.addTests([ { name: "Bombe: multiple stops", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA<\/td>TT<\/td>VFISUSGTKSTMPSUNAK<\/td>/, + expectedMatch: /LGA<\/td> {2}TT<\/td> {2}VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -88,7 +88,7 @@ TestRegister.addTests([ { name: "Bombe: checking machine", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA<\/td>TT AG BO CL EK FF HH II JJ SS YY<\/td>THISISATESTMESSAGE<\/td>/, + expectedMatch: /LGA<\/td> {2}TT AG BO CL EK FF HH II JJ SS YY<\/td> {2}THISISATESTMESSAGE<\/td>/, recipeConfig: [ { "op": "Bombe", @@ -108,7 +108,7 @@ TestRegister.addTests([ { name: "Bombe: 4 rotor", input: "LUOXGJSHGEDSRDOQQX", - expectedMatch: /LHSC<\/td>SS<\/td>HHHSSSGQUUQPKSEKWK<\/td>/, + expectedMatch: /LHSC<\/td> {2}SS<\/td> {2}HHHSSSGQUUQPKSEKWK<\/td>/, recipeConfig: [ { "op": "Bombe", diff --git a/tests/operations/tests/MultipleBombe.mjs b/tests/operations/tests/MultipleBombe.mjs index 8e2cc685..32d2db08 100644 --- a/tests/operations/tests/MultipleBombe.mjs +++ b/tests/operations/tests/MultipleBombe.mjs @@ -10,7 +10,7 @@ TestRegister.addTests([ { name: "Multi-Bombe: 3 rotor", input: "BBYFLTHHYIJQAYBBYS", - expectedMatch: /LGA<\/td>SS<\/td>VFISUSGTKSTMPSUNAK<\/td>/, + expectedMatch: /LGA<\/td> {2}SS<\/td> {2}VFISUSGTKSTMPSUNAK<\/td>/, recipeConfig: [ { "op": "Multiple Bombe", From cf32372a57e0cf4cf85c3b620979d229c1969895 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 14 Mar 2019 12:08:35 +0000 Subject: [PATCH 105/110] Added Enigma wiki article link to Enigma, Typex, Bombe and Multi-Bombe operation descriptions. --- src/core/operations/Bombe.mjs | 2 +- src/core/operations/Enigma.mjs | 2 +- src/core/operations/MultipleBombe.mjs | 2 +- src/core/operations/Typex.mjs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/operations/Bombe.mjs b/src/core/operations/Bombe.mjs index 00d883ed..c2ea82bf 100644 --- a/src/core/operations/Bombe.mjs +++ b/src/core/operations/Bombe.mjs @@ -23,7 +23,7 @@ class Bombe extends Operation { this.name = "Bombe"; this.module = "Default"; - this.description = "Emulation of the Bombe machine used at Bletchley Park to attack Enigma, based on work by Polish and British cryptanalysts.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and known plugboard pairs.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops or is too short, a large number of incorrect outputs will usually be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor.

By default this operation runs the checking machine, a manual process to verify the quality of Bombe stops, on each stop, discarding stops which fail. If you want to see how many times the hardware actually stops for a given input, disable the checking machine."; + this.description = "Emulation of the Bombe machine used at Bletchley Park to attack Enigma, based on work by Polish and British cryptanalysts.

To run this you need to have a 'crib', which is some known plaintext for a chunk of the target ciphertext, and know the rotors used. (See the 'Bombe (multiple runs)' operation if you don't know the rotors.) The machine will suggest possible configurations of the Enigma. Each suggestion has the rotor start positions (left to right) and known plugboard pairs.

Choosing a crib: First, note that Enigma cannot encrypt a letter to itself, which allows you to rule out some positions for possible cribs. Secondly, the Bombe does not simulate the Enigma's middle rotor stepping. The longer your crib, the more likely a step happened within it, which will prevent the attack working. However, other than that, longer cribs are generally better. The attack produces a 'menu' which maps ciphertext letters to plaintext, and the goal is to produce 'loops': for example, with ciphertext ABC and crib CAB, we have the mappings A<->C, B<->A, and C<->B, which produces a loop A-B-C-A. The more loops, the better the crib. The operation will output this: if your menu has too few loops or is too short, a large number of incorrect outputs will usually be produced. Try a different crib. If the menu seems good but the right answer isn't produced, your crib may be wrong, or you may have overlapped the middle rotor stepping - try a different crib.

Output is not sufficient to fully decrypt the data. You will have to recover the rest of the plugboard settings by inspection. And the ring position is not taken into account: this affects when the middle rotor steps. If your output is correct for a bit, and then goes wrong, adjust the ring and start position on the right-hand rotor together until the output improves. If necessary, repeat for the middle rotor.

By default this operation runs the checking machine, a manual process to verify the quality of Bombe stops, on each stop, discarding stops which fail. If you want to see how many times the hardware actually stops for a given input, disable the checking machine.

More detailed descriptions of the Enigma, Typex and Bombe operations can be found here."; this.infoURL = "https://wikipedia.org/wiki/Bombe"; this.inputType = "string"; this.outputType = "JSON"; diff --git a/src/core/operations/Enigma.mjs b/src/core/operations/Enigma.mjs index 71593070..542e8281 100644 --- a/src/core/operations/Enigma.mjs +++ b/src/core/operations/Enigma.mjs @@ -22,7 +22,7 @@ class Enigma extends Operation { this.name = "Enigma"; this.module = "Default"; - this.description = "Encipher/decipher with the WW2 Enigma machine.

Enigma was used by the German military, among others, around the WW2 era as a portable cipher machine to protect sensitive military, diplomatic and commercial communications.

The standard set of German military rotors and reflectors are provided. To configure the plugboard, enter a string of connected pairs of letters, e.g. AB CD EF connects A to B, C to D, and E to F. This is also used to create your own reflectors. To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points.
This is deliberately fairly permissive with rotor placements etc compared to a real Enigma (on which, for example, a four-rotor Enigma uses only the thin reflectors and the beta or gamma rotor in the 4th slot)."; + this.description = "Encipher/decipher with the WW2 Enigma machine.

Enigma was used by the German military, among others, around the WW2 era as a portable cipher machine to protect sensitive military, diplomatic and commercial communications.

The standard set of German military rotors and reflectors are provided. To configure the plugboard, enter a string of connected pairs of letters, e.g. AB CD EF connects A to B, C to D, and E to F. This is also used to create your own reflectors. To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points.
This is deliberately fairly permissive with rotor placements etc compared to a real Enigma (on which, for example, a four-rotor Enigma uses only the thin reflectors and the beta or gamma rotor in the 4th slot).

More detailed descriptions of the Enigma, Typex and Bombe operations can be found here."; this.infoURL = "https://wikipedia.org/wiki/Enigma_machine"; this.inputType = "string"; this.outputType = "string"; diff --git a/src/core/operations/MultipleBombe.mjs b/src/core/operations/MultipleBombe.mjs index 03364a01..b6a48872 100644 --- a/src/core/operations/MultipleBombe.mjs +++ b/src/core/operations/MultipleBombe.mjs @@ -53,7 +53,7 @@ class MultipleBombe extends Operation { this.name = "Multiple Bombe"; this.module = "Default"; - this.description = "Emulation of the Bombe machine used to attack Enigma. This version carries out multiple Bombe runs to handle unknown rotor configurations.

You should test your menu on the single Bombe operation before running it here. See the description of the Bombe operation for instructions on choosing a crib."; + this.description = "Emulation of the Bombe machine used to attack Enigma. This version carries out multiple Bombe runs to handle unknown rotor configurations.

You should test your menu on the single Bombe operation before running it here. See the description of the Bombe operation for instructions on choosing a crib.

More detailed descriptions of the Enigma, Typex and Bombe operations can be found here."; this.infoURL = "https://wikipedia.org/wiki/Bombe"; this.inputType = "string"; this.outputType = "JSON"; diff --git a/src/core/operations/Typex.mjs b/src/core/operations/Typex.mjs index 760914f5..70b5f6c3 100644 --- a/src/core/operations/Typex.mjs +++ b/src/core/operations/Typex.mjs @@ -23,7 +23,7 @@ class Typex extends Operation { this.name = "Typex"; this.module = "Default"; - this.description = "Encipher/decipher with the WW2 Typex machine.

Typex was originally built by the British Royal Air Force prior to WW2, and is based on the Enigma machine with some improvements made, including using five rotors with more stepping points and interchangeable wiring cores. It was used across the British and Commonewealth militaries. A number of later variants were produced; here we simulate a WW2 era Mark 22 Typex with plugboards for the reflector and input. Typex rotors were changed regularly and none are public: a random example set are provided.

To configure the reflector plugboard, enter a string of connected pairs of letters in the reflector box, e.g. AB CD EF connects A to B, C to D, and E to F (you'll need to connect every letter). There is also an input plugboard: unlike Enigma's plugboard, it's not restricted to pairs, so it's entered like a rotor (without stepping). To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points."; + this.description = "Encipher/decipher with the WW2 Typex machine.

Typex was originally built by the British Royal Air Force prior to WW2, and is based on the Enigma machine with some improvements made, including using five rotors with more stepping points and interchangeable wiring cores. It was used across the British and Commonewealth militaries. A number of later variants were produced; here we simulate a WW2 era Mark 22 Typex with plugboards for the reflector and input. Typex rotors were changed regularly and none are public: a random example set are provided.

To configure the reflector plugboard, enter a string of connected pairs of letters in the reflector box, e.g. AB CD EF connects A to B, C to D, and E to F (you'll need to connect every letter). There is also an input plugboard: unlike Enigma's plugboard, it's not restricted to pairs, so it's entered like a rotor (without stepping). To create your own rotor, enter the letters that the rotor maps A to Z to, in order, optionally followed by < then a list of stepping points.

More detailed descriptions of the Enigma, Typex and Bombe operations can be found here."; this.infoURL = "https://wikipedia.org/wiki/Typex"; this.inputType = "string"; this.outputType = "string"; From 33db0e666a5b2a0115118a0e6c6e0ebc94769c07 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 14 Mar 2019 12:11:41 +0000 Subject: [PATCH 106/110] Final tweaks to Bombe svg and preloader css --- src/web/static/images/bombe.svg | 4 ++-- src/web/stylesheets/preloader.css | 9 --------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/web/static/images/bombe.svg b/src/web/static/images/bombe.svg index 1fd40554..a970903a 100644 --- a/src/web/static/images/bombe.svg +++ b/src/web/static/images/bombe.svg @@ -1,8 +1,8 @@ diff --git a/src/web/stylesheets/preloader.css b/src/web/stylesheets/preloader.css index 690fe5c1..288ffc28 100755 --- a/src/web/stylesheets/preloader.css +++ b/src/web/stylesheets/preloader.css @@ -160,12 +160,3 @@ transform: translate3d(0, 200px, 0); } } - -@keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } -} From ef38897a010208f5311850351c71218714294a26 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 14 Mar 2019 12:20:05 +0000 Subject: [PATCH 107/110] Updated CHANGELOG --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f3eca29..0d2eca7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.26.0] - 2019-03-09 +- Various image manipulation operations added [@j433866] | [#506] + +### [8.25.0] - 2019-03-09 +- 'Extract Files' operation added and more file formats supported [@n1474335] | [#440] + ### [8.24.0] - 2019-02-08 - 'DNS over HTTPS' operation added [@h345983745] | [#489] @@ -106,6 +112,8 @@ All major and minor version changes will be documented in this file. Details of +[8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0 +[8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0 [8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0 [8.23.1]: https://github.com/gchq/CyberChef/releases/tag/v8.23.1 [8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0 @@ -180,6 +188,7 @@ All major and minor version changes will be documented in this file. Details of [#394]: https://github.com/gchq/CyberChef/pull/394 [#428]: https://github.com/gchq/CyberChef/pull/428 [#439]: https://github.com/gchq/CyberChef/pull/439 +[#440]: https://github.com/gchq/CyberChef/pull/440 [#441]: https://github.com/gchq/CyberChef/pull/441 [#443]: https://github.com/gchq/CyberChef/pull/443 [#446]: https://github.com/gchq/CyberChef/pull/446 @@ -192,3 +201,4 @@ All major and minor version changes will be documented in this file. Details of [#468]: https://github.com/gchq/CyberChef/pull/468 [#476]: https://github.com/gchq/CyberChef/pull/476 [#489]: https://github.com/gchq/CyberChef/pull/489 +[#506]: https://github.com/gchq/CyberChef/pull/506 From c8a2a8b003a31ddf9f0860e63db068972d3f820b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 14 Mar 2019 12:26:00 +0000 Subject: [PATCH 108/110] Updated CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d2eca7e..b21944fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.27.0] - 2019-03-14 +- 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516] +- See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations. +- New Bombe-style loading animation added for long-running operations [@n1474335] +- New operation argument types added: `populateMultiOption` and `argSelector` [@n1474335] + ### [8.26.0] - 2019-03-09 - Various image manipulation operations added [@j433866] | [#506] From 3ff10bfeaebad28c72ceff53f8c78ebb4ebb0755 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 14 Mar 2019 12:26:07 +0000 Subject: [PATCH 109/110] 8.27.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 425b3f76..9fd4a068 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.26.3", + "version": "8.27.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 453c9d96..e650b272 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.26.3", + "version": "8.27.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From bb5b92571e62630ad7a899089d5f7d2e4fd5f9d2 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 14 Mar 2019 16:08:25 +0000 Subject: [PATCH 110/110] Updated CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b21944fe..11a18d86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,6 +118,7 @@ All major and minor version changes will be documented in this file. Details of +[8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0 [8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0 [8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0 [8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0