Merge branch 'use-std-esm' of github.com:gchq/CyberChef

.travis.yml

@@ -12,9 +12,9 @@ script:
   - grunt lint
   - grunt test
   - grunt docs
-  - npm run node-prod
   - grunt prod --msg="$COMPILE_MSG"
   - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
+  - grunt testnodeconsumer
 before_deploy:
   - grunt exec:sitemap
   - grunt copy:ghPages
@@ -34,7 +34,7 @@ deploy:
     file_glob: true
     file:
       - build/prod/*.zip
-      - build/node/CyberChef.js
+      - src/node/cjs.js
     on:
       repo: gchq/CyberChef
       tags: true

Gruntfile.js

@@ -3,7 +3,6 @@
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
-const NodeExternals = require("webpack-node-externals");
 const glob = require("glob");
 const path = require("path");
 
@@ -15,7 +14,6 @@ const path = require("path");
  * @license Apache-2.0
  */
 
-const NODE_PROD = process.env.NODE_ENV === "production";
 
 module.exports = function (grunt) {
     grunt.file.defaultEncoding = "utf8";
@@ -36,8 +34,7 @@ module.exports = function (grunt) {
     grunt.registerTask("node",
         "Compiles CyberChef into a single NodeJS module.",
         [
-            "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig",
+            "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
-            "exec:generateNodeIndex", "webpack:node", "webpack:nodeRepl", "chmod:build"
         ]);
 
     grunt.registerTask("test",
@@ -51,6 +48,10 @@ module.exports = function (grunt) {
         "A task which runs all the UI tests in the tests directory. The prod task must already have been run.",
         ["connect:prod", "exec:browserTests"]);
 
+    grunt.registerTask("testnodeconsumer",
+        "A task which checks whether consuming CJS and ESM apps work with the CyberChef build",
+        ["exec:setupNodeConsumers", "exec:testCJSNodeConsumer", "exec:testESMNodeConsumer", "exec:testESMDeepImportNodeConsumer", "exec:teardownNodeConsumers"]);
+
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
         ["clean:docs", "jsdoc", "chmod:docs"]);
@@ -90,7 +91,8 @@ module.exports = function (grunt) {
             COMPILE_MSG: JSON.stringify(grunt.option("compile-msg") || grunt.option("msg") || ""),
             PKG_VERSION: JSON.stringify(pkg.version),
         },
-        moduleEntryPoints = listEntryModules();
+        moduleEntryPoints = listEntryModules(),
+        nodeConsumerTestPath = "~/tmp-cyberchef";
 
 
     /**
@@ -201,46 +203,6 @@ module.exports = function (grunt) {
                     ]
                 };
             },
-            node: {
-                mode: NODE_PROD ? "production" : "development",
-                target: "node",
-                entry: "./src/node/index.mjs",
-                externals: [NodeExternals({
-                    whitelist: ["crypto-api/src/crypto-api"]
-                })],
-                output: {
-                    filename: "CyberChef.js",
-                    path: __dirname + "/build/node",
-                    library: "CyberChef",
-                    libraryTarget: "commonjs2"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS),
-                    new webpack.optimize.LimitChunkCountPlugin({
-                        maxChunks: 1
-                    })
-                ],
-            },
-            nodeRepl: {
-                mode: NODE_PROD ? "production" : "development",
-                target: "node",
-                entry: "./src/node/repl-index.mjs",
-                externals: [NodeExternals({
-                    whitelist: ["crypto-api/src/crypto-api"]
-                })],
-                output: {
-                    filename: "CyberChef-repl.js",
-                    path: __dirname + "/build/node",
-                    library: "CyberChef",
-                    libraryTarget: "commonjs2"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS),
-                    new webpack.optimize.LimitChunkCountPlugin({
-                        maxChunks: 1
-                    })
-                ],
-            }
         },
         "webpack-dev-server": {
             options: {
@@ -428,7 +390,44 @@ module.exports = function (grunt) {
             },
             nodeTests: {
                 command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs"
-            }
+            },
+            setupNodeConsumers: {
+                command: [
+                    "echo '\n--- Testing node conumers ---'",
+                    "npm link",
+                    `mkdir ${nodeConsumerTestPath}`,
+                    `cp tests/node/consumers/* ${nodeConsumerTestPath}`,
+                    `cd ${nodeConsumerTestPath}`,
+                    "npm link cyberchef"
+                ].join(";"),
+            },
+            teardownNodeConsumers: {
+                command: [
+                    `rm -rf ${nodeConsumerTestPath}`,
+                    "echo '\n--- Node consumer tests complete ---'"
+                ].join(";"),
+            },
+            testCJSNodeConsumer: {
+                command: [
+                    `cd ${nodeConsumerTestPath}`,
+                    "node --no-warnings cjs-consumer.js",
+                ].join(";"),
+                stdout: false,
+            },
+            testESMNodeConsumer: {
+                command: [
+                    `cd ${nodeConsumerTestPath}`,
+                    "node --no-warnings --experimental-modules esm-consumer.mjs",
+                ].join(";"),
+                stdout: false,
+            },
+            testESMDeepImportNodeConsumer: {
+                command: [
+                    `cd ${nodeConsumerTestPath}`,
+                    "node --no-warnings --experimental-modules esm-deep-import-consumer.mjs",
+                ].join(";"),
+                stdout: false,
+            },
         },
     });
 };

README.md

@@ -81,6 +81,10 @@ CyberChef is built to support
  - Mozilla Firefox 35+
  - Microsoft Edge 14+
 
+## Node.js support
+
+CyberChef is built to fully support Node.js `v10` and partially supports `v12`. Named imports using a deep import specifier does not work in `v12`. For more information, see the Node API page in the project [wiki pages](https://github.com/gchq/CyberChef/wiki)
+
 
 ## Contributing
 

package-lock.json

@@ -3146,15 +3146,6 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
       "dev": true
     },
-    "catharsis": {
-      "version": "0.8.10",
-      "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.10.tgz",
-      "integrity": "sha512-l2OUaz/3PU3MZylspVFJvwHCVfWyvcduPq4lv3AzZ2pJzZCo7kNKFNyatwujD7XgvGkNAE/Jhhbh2uARNwNkfw==",
-      "dev": true,
-      "requires": {
-        "lodash": "^4.17.11"
-      }
-    },
     "chai-nightwatch": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/chai-nightwatch/-/chai-nightwatch-0.3.0.tgz",
@@ -5097,6 +5088,11 @@
       "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==",
       "dev": true
     },
+    "esm": {
+      "version": "3.2.25",
+      "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
+      "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA=="
+    },
     "esmangle": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/esmangle/-/esmangle-1.0.1.tgz",
@@ -8524,27 +8520,36 @@
       }
     },
     "jsdoc": {
-      "version": "3.6.2",
+      "version": "3.6.3",
-      "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.2.tgz",
+      "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.3.tgz",
-      "integrity": "sha512-S2vzg99C5+gb7FWlrK4TVdyzVPGGkdvpDkCEJH1JABi2PKzPeLu5/zZffcJUifgWUJqXWl41Hoc+MmuM2GukIg==",
+      "integrity": "sha512-Yf1ZKA3r9nvtMWHO1kEuMZTlHOF8uoQ0vyo5eH7SQy5YeIiHM+B0DgKnn+X6y6KDYZcF7G2SPkKF+JORCXWE/A==",
       "dev": true,
       "requires": {
         "@babel/parser": "^7.4.4",
         "bluebird": "^3.5.4",
-        "catharsis": "^0.8.10",
+        "catharsis": "^0.8.11",
         "escape-string-regexp": "^2.0.0",
         "js2xmlparser": "^4.0.0",
         "klaw": "^3.0.0",
         "markdown-it": "^8.4.2",
         "markdown-it-anchor": "^5.0.2",
-        "marked": "^0.6.2",
+        "marked": "^0.7.0",
         "mkdirp": "^0.5.1",
-        "requizzle": "^0.2.2",
+        "requizzle": "^0.2.3",
         "strip-json-comments": "^3.0.1",
         "taffydb": "2.6.2",
         "underscore": "~1.9.1"
       },
       "dependencies": {
+        "catharsis": {
+          "version": "0.8.11",
+          "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.11.tgz",
+          "integrity": "sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g==",
+          "dev": true,
+          "requires": {
+            "lodash": "^4.17.14"
+          }
+        },
         "escape-string-regexp": {
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz",
@@ -8560,6 +8565,21 @@
             "graceful-fs": "^4.1.9"
           }
         },
+        "marked": {
+          "version": "0.7.0",
+          "resolved": "https://registry.npmjs.org/marked/-/marked-0.7.0.tgz",
+          "integrity": "sha512-c+yYdCZJQrsRjTPhUx7VKkApw9bwDkNbHUKo1ovgcfDjb2kc8rLuRbIFyXL5WOEUwzSSKo3IXpph2K6DqB/KZg==",
+          "dev": true
+        },
+        "requizzle": {
+          "version": "0.2.3",
+          "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.3.tgz",
+          "integrity": "sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ==",
+          "dev": true,
+          "requires": {
+            "lodash": "^4.17.14"
+          }
+        },
         "strip-json-comments": {
           "version": "3.0.1",
           "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz",
@@ -8989,9 +9009,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.11",
+      "version": "4.17.15",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+      "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
     },
     "lodash._arraycopy": {
       "version": "3.0.0",
@@ -9083,9 +9103,9 @@
       "dev": true
     },
     "lodash.defaultsdeep": {
-      "version": "4.6.0",
+      "version": "4.6.1",
-      "resolved": "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz",
-      "integrity": "sha1-vsECT4WxvZbL6kBbI8FK1kQ6b4E=",
+      "integrity": "sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==",
       "dev": true
     },
     "lodash.escaperegexp": {
@@ -9154,15 +9174,15 @@
       }
     },
     "lodash.merge": {
-      "version": "4.6.1",
+      "version": "4.6.2",
-      "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
-      "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==",
+      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
       "dev": true
     },
     "lodash.mergewith": {
-      "version": "4.6.1",
+      "version": "4.6.2",
-      "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz",
-      "integrity": "sha512-eWw5r+PYICtEBgrBE5hhlT6aAa75f411bgDz/ZL2KZqYV03USvucsxcHUIlGTDTECs1eunpI7HOV7U+WLDvNdQ==",
+      "integrity": "sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==",
       "dev": true
     },
     "lodash.once": {
@@ -9304,12 +9324,6 @@
       "integrity": "sha512-n8zCGjxA3T+Mx1pG8HEgbJbkB8JFUuRkeTZQuIM8iPY6oQ8sWOPRZJDFC9a/pNg2QkHEjjGkhBEl/RSyzaDZ3A==",
       "dev": true
     },
-    "marked": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-0.6.2.tgz",
-      "integrity": "sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==",
-      "dev": true
-    },
     "md5.js": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@@ -11971,15 +11985,6 @@
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=",
       "dev": true
     },
-    "requizzle": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.2.tgz",
-      "integrity": "sha512-oJ6y7JcUJkblRGhMByGNcszeLgU0qDxNKFCiUZR1XyzHyVsev+Mxb1tyygxLd1ORsKee1SA5BInFdUwY64GE/A==",
-      "dev": true,
-      "requires": {
-        "lodash": "^4.17.11"
-      }
-    },
     "resolve": {
       "version": "1.11.1",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",

package.json

@@ -27,7 +27,7 @@
     "type": "git",
     "url": "https://github.com/gchq/CyberChef/"
   },
-  "main": "build/node/CyberChef.js",
+  "main": "src/node/cjs.js",
   "module": "src/node/index.mjs",
   "bugs": "https://github.com/gchq/CyberChef/issues",
   "browserslist": [
@@ -108,6 +108,7 @@
     "diff": "^4.0.1",
     "es6-promisify": "^6.0.1",
     "escodegen": "^1.11.1",
+    "esm": "^3.2.25",
     "esmangle": "^1.0.1",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
@@ -126,7 +127,7 @@
     "kbpgp": "2.1.2",
     "libbzip2-wasm": "0.0.4",
     "libyara-wasm": "0.0.12",
-    "lodash": "^4.17.11",
+    "lodash": "^4.17.15",
     "loglevel": "^1.6.3",
     "loglevel-message-prefix": "^3.0.0",
     "moment": "^2.24.0",
@@ -156,11 +157,9 @@
   "scripts": {
     "start": "grunt dev",
     "build": "grunt prod",
-    "node": "NODE_ENV=development grunt node",
+    "repl": "node src/node/repl.js",
-    "node-prod": "NODE_ENV=production grunt node",
-    "repl": "grunt node && node build/node/CyberChef-repl.js",
     "test": "grunt test",
-    "test-node": "grunt test-node",
+    "test-node-consumer": "grunt testnodeconsumer",
     "testui": "grunt testui",
     "docs": "grunt docs",
     "lint": "grunt lint",

src/node/cjs.js

@@ -0,0 +1,13 @@
+/**
+ * Export the main ESM module as CommonJS
+ *
+ *
+ * @author d98762656 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+/*eslint no-global-assign: ["off"] */
+require = require("esm")(module);
+module.exports = require("./index.mjs");
+module.exports.File = require("./File.mjs");

src/node/repl.js

@@ -7,9 +7,9 @@
  * @license Apache-2.0
  */
 
-import chef from "./index.mjs";
+const chef = require("./cjs.js");
-import repl from "repl";
+const repl = require("repl");
-import File from "./File.mjs";
+
 
 /*eslint no-console: ["off"] */
 
@@ -26,7 +26,7 @@ const replServer = repl.start({
     prompt: "chef > ",
 });
 
-global.File = File;
+global.File = chef.File;
 
 Object.keys(chef).forEach((key) => {
     if (key !== "operations") {

tests/node/consumers/cjs-consumer.js

@@ -0,0 +1,29 @@
+/**
+ * Tests to ensure that a consuming app can use CJS require
+ *
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+const chef = require("cyberchef");
+const assert = require("assert");
+
+const d = chef.bake("Testing, 1 2 3", [
+    chef.toHex,
+    chef.reverse,
+    {
+        op: chef.unique,
+        args: {
+            delimiter: "Space",
+        }
+    },
+    {
+        op: chef.multiply,
+        args: {
+            delimiter: "Space",
+        }
+    }
+]);
+
+assert.equal(d.value, "630957449041920");

tests/node/consumers/esm-consumer.mjs

@@ -0,0 +1,28 @@
+/**
+ * Tests to ensure that a consuming app can use ESM imports
+ *
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import assert from "assert";
+import chef from "cyberchef";
+
+const d = chef.bake("Testing, 1 2 3", [
+    chef.toHex,
+    chef.reverse,
+    {
+        op: chef.unique,
+        args: {
+            delimiter: "Space",
+        }
+    },
+    {
+        op: chef.multiply,
+        args: {
+            delimiter: "Space",
+        }
+    }
+]);
+
+assert.equal(d.value, "630957449041920");

tests/node/consumers/esm-deep-import-consumer.mjs

@@ -0,0 +1,28 @@
+/**
+ * Tests to ensure that a consuming app can use named imports from deep import patch
+ *
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import assert from "assert";
+import { bake, toHex, reverse, unique, multiply } from "cyberchef/src/node/index.mjs";
+
+const d = bake("Testing, 1 2 3", [
+    toHex,
+    reverse,
+    {
+        op: unique,
+        args: {
+            delimiter: "Space",
+        }
+    },
+    {
+        op: multiply,
+        args: {
+            delimiter: "Space",
+        }
+    }
+]);
+
+assert.equal(d.value, "630957449041920");