Convert test runner to headless

Gruntfile.js

@@ -10,13 +10,13 @@ module.exports = function(grunt) {
         ["clean:dev", "concat:css", "concat:js", "copy:htmlDev", "copy:staticDev", "chmod:build", "watch"]);
 
     grunt.registerTask("test",
-        "A persistent task which creates a test build whenever source files are modified.",
+        "A task which runs all the tests in test/tests.",
-        ["clean:dev", "concat:cssTest", "concat:jsTest", "copy:htmlTest", "copy:staticTest", "chmod:build", "watch"]);
+        ["clean:test", "concat:jsTest", "copy:htmlTest", "chmod:build", "exec:tests"]);
 
     grunt.registerTask("prod",
         "Creates a production-ready build. Use the --msg flag to add a compile message.",
         ["eslint", "exec:stats", "clean", "jsdoc", "concat", "copy:htmlDev", "copy:htmlProd", "copy:htmlInline",
-         "copy:staticDev", "copy:staticProd", "cssmin", "uglify:prod", "inline", "htmlmin", "chmod"]);
+         "copy:staticDev", "copy:staticProd", "cssmin", "uglify:prod", "inline", "htmlmin", "chmod", "test"]);
 
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
@@ -138,6 +138,7 @@ module.exports = function(grunt) {
         "src/js/lib/vkbeautify.js",
         "src/js/lib/Sortable.js",
         "src/js/lib/bootstrap-colorpicker.js",
+        "src/js/lib/es6-promise.auto.js",
         "src/js/lib/xpath.js",
 
         // Custom libraries
@@ -171,11 +172,9 @@ module.exports = function(grunt) {
     var jsTestFiles = [].concat(
         jsIncludes,
         [
-            "src/js/lib/vuejs/vue.min.js",
+            "test/TestRegister.js",
-            "src/js/test/*.js",
+            "test/tests/**/*.js",
-            "src/js/test/tests/**/*",
+            "test/TestRunner.js",
-            // Start the test runner app!
-            "src/js/test/views/html/main.js",
         ]
     );
 
@@ -219,7 +218,11 @@ module.exports = function(grunt) {
             config: ["src/js/config/**/*.js"],
             views: ["src/js/views/**/*.js"],
             operations: ["src/js/operations/**/*.js"],
-            tests: ["src/js/test/**/*.js"],
+            tests: [
+                "test/**/*.js",
+                "!test/PhantomRunner.js",
+                "!test/NodeRunner.js",
+            ],
         },
         jsdoc: {
             options: {
@@ -238,6 +241,7 @@ module.exports = function(grunt) {
         },
         clean: {
             dev: ["build/dev/*"],
+            test: ["build/test/*"],
             prod: ["build/prod/*"],
             docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico"],
         },
@@ -261,22 +265,6 @@ module.exports = function(grunt) {
                 ],
                 dest: "build/dev/styles.css"
             },
-            cssTest: {
-                options: {
-                    banner: banner.replace(/\/\*\*/g, "/*!"),
-                    process: function(content, srcpath) {
-                        // Change special comments from /** to /*! to comply with cssmin
-                        content = content.replace(/^\/\*\* /g, "/*! ");
-                        return grunt.template.process(content);
-                    }
-                },
-                src: [
-                    "src/css/lib/**/*.css",
-                    "src/css/structure/**/*.css",
-                    "src/css/themes/classic.css"
-                ],
-                dest: "build/test/styles.css"
-            },
             js: {
                 options: {
                     banner: '"use strict";\n'
@@ -289,7 +277,7 @@ module.exports = function(grunt) {
                     banner: '"use strict";\n'
                 },
                 src: jsTestFiles,
-                dest: "build/test/scripts.js"
+                dest: "build/test/tests.js"
             }
         },
         copy: {
@@ -348,21 +336,6 @@ module.exports = function(grunt) {
                     }
                 ]
             },
-            staticTest: {
-                files: [
-                    {
-                        expand: true,
-                        cwd: "src/static/",
-                        src: [
-                            "**/*",
-                            "**/.*",
-                            "!stats.txt",
-                            "!ga.html"
-                        ],
-                        dest: "build/test/"
-                    }
-                ]
-            },
             staticProd: {
                 files: [
                     {
@@ -478,6 +451,9 @@ module.exports = function(grunt) {
             }
         },
         exec: {
+            tests: {
+                command: "node test/NodeRunner.js",
+            },
             repoSize: {
                 command: [
                     "git ls-files | wc -l | xargs printf '\n%b\ttracked files\n'",
@@ -537,15 +513,15 @@ module.exports = function(grunt) {
             },
             js: {
                 files: "src/js/**/*.js",
-                tasks: ["concat:js", "concat:jsTest", "chmod:build"]
+                tasks: ["concat:js", "chmod:build"]
             },
             html: {
                 files: "src/html/**/*.html",
-                tasks: ["copy:htmlDev", "copy:htmlTest", "chmod:build"]
+                tasks: ["copy:htmlDev", "chmod:build"]
             },
             static: {
                 files: ["src/static/**/*", "src/static/**/.*"],
-                tasks: ["copy:staticDev", "copy:staticTest", "chmod:build"]
+                tasks: ["copy:staticDev", "chmod:build"]
             },
             grunt: {
                 files: "Gruntfile.js",

package.json

@@ -39,6 +39,7 @@
     "grunt-exec": "~1.0.1",
     "grunt-inline-alt": "~0.3.10",
     "grunt-jsdoc": "^2.1.0",
-    "ink-docstrap": "^1.1.4"
+    "ink-docstrap": "^1.1.4",
+    "phantomjs-prebuilt": "^2.1.14"
   }
 }

src/html/test.html

@@ -25,72 +25,10 @@
 <html>
     <head>
         <meta charset="UTF-8">
-        <title>CyberChef Test Runner</title>
+        <title>CyberChef</title>
-        <link rel="icon" type="image/png" href="images/favicon.ico?__inline" />
-        <link href="styles.css" rel="stylesheet" />
     </head>
     <body>
-        <template id="test-status-icon-template">
+        <main style="white-space: pre"></main>
-            <span>{{ getIcon() }}</span>
+        <script type="application/javascript" src="tests.js"></script>
-        </template>
-
-        <template id="test-stats-template">
-            <div class="text-center row">
-                <div class="col-md-2"
-                     v-for="status in ['Waiting', 'Loading', 'Erroring', 'Failing']">
-                    <test-status-icon :status="status.toLowerCase()"></test-status-icon>
-                    <br>
-                    {{ status }}
-                    <br>
-                    {{ countTestsWithStatus(status.toLowerCase()) }}
-                </div>
-                <div class="col-md-2">
-                    <test-status-icon status="passing"></test-status-icon>
-                    <br>
-                    Passing
-                    <br>
-                    {{ countTestsWithStatus("passing") }}
-                    /
-                    {{ tests.length }}
-                </div>
-                <div class="col-md-2">
-                    <test-status-icon status="passing"></test-status-icon>
-                    <br>
-                    % Passing
-                    <br>
-                    {{ ((100.0 * countTestsWithStatus("passing")) / tests.length).toFixed(1) }}%
-                </div>
-            </div>
-        </template>
-
-        <template id="tests-template">
-            <table class="table table-striped">
-                <tbody>
-                    <tr v-for="test in tests">
-                        <td class="col-md-1 col-sm-4">
-                            <test-status-icon :status="test.status"></test-status-icon>
-                        </td>
-                        <td class="col-md-4 col-sm-8">
-                            {{ test.name }}
-                        </td>
-                        <td class="col-md-7 col-sm-12">
-                            <pre v-if="test.output"><code>{{ test.output }}</code></pre>
-                        </td>
-                    </tr>
-                </tbody>
-            </table>
-        </template>
-
-        <div class="container">
-            <main>
-                <h1>CyberChef Test Runner</h1>
-                <hr>
-                <test-stats :tests="tests"></test-stats>
-                <hr>
-                <tests :tests="tests"></tests>
-            </main>
-        </div>
-
-        <script type="application/javascript" src="scripts.js"></script>
     </body>
 </html>

src/js/.eslintrc.json

@@ -111,8 +111,8 @@
         "SeasonalWaiter": false,
         "WindowWaiter": false,
 
-        /* test */
+        /* tests */
-        "Vue": false,
+        "TestRegister": false,
-        "TestRegister": false
+        "TestRunner": false
     }
 }

src/js/test/TestRegister.js

@@ -1,106 +0,0 @@
-/**
- * TestRegister.js
- *
- * This is so individual files can register their tests in one place, and
- * ensure that they will get run by the frontend.
- *
- * @author tlwr [toby@toby.codes
- *
- * @copyright Crown Copyright 2017
- * @license Apache-2.0
- *
- */
-(function() {
-    /**
-     * Add a list of tests to the register.
-     *
-     * @class
-     */
-    function TestRegister() {
-        this.tests = [];
-    }
-
-    /**
-     * Add a list of tests to the register.
-     *
-     * @param {Object[]} tests
-     */
-    TestRegister.prototype.addTests = function(tests) {
-        this.tests = this.tests.concat(tests.map(function(test) {
-
-            test.status = "waiting";
-            test.output = "";
-
-            return test;
-        }));
-    };
-
-    /**
-     * Returns the list of tests.
-     *
-     * @returns {Object[]} tests
-     */
-    TestRegister.prototype.getTests = function() {
-        return this.tests;
-    };
-
-    /**
-     * Runs all the tests in the register and updates the state of each test.
-     *
-     */
-    TestRegister.prototype.runTests = function() {
-        this.tests.forEach(function(test, i) {
-            var chef = new Chef();
-
-            // This resolve is to not instantly break when async operations are
-            // supported. Marked as TODO.
-            Promise.resolve(chef.bake(
-                test.input,
-                test.recipeConfig,
-                {},
-                0,
-                0
-            ))
-                .then(function(result) {
-                    if (result.error) {
-                        if (test.expectedError) {
-                            test.status = "passing";
-                        } else {
-                            test.status = "erroring";
-                            test.output = [
-                                "Erroring",
-                                "-------",
-                                result.error.displayStr,
-                            ].join("\n");
-                        }
-                    } else {
-                        if (test.expectedError) {
-                            test.status = "failing";
-                            test.output = [
-                                "Failing",
-                                "-------",
-                                "Expected an error but did not receive one.",
-                            ].join("\n");
-                        } else if (result.result === test.expectedOutput) {
-                            test.status = "passing";
-                        } else {
-                            test.status = "failing";
-                            test.output = [
-                                "Failing",
-                                "-------",
-                                "Expected",
-                                "-------",
-                                test.expectedOutput,
-                                "Actual",
-                                "-------",
-                                result.result,
-                            ].join("\n");
-                        }
-                    }
-                });
-        });
-    };
-
-    // Singleton TestRegister, keeping things simple and obvious.
-    window.TestRegister = new TestRegister();
-})();

src/js/test/views/html/main.js

@@ -1,58 +0,0 @@
-/**
- * main.js
- *
- * Simple VueJS app for running all the tests and displaying some basic stats.
- * @author tlwr [toby@toby.codes]
- * @copyright Crown Copyright 2017
- * @license Apache-2.0
- *
- */
-(function() {
-    Vue.component("test-status-icon", {
-        template: "#test-status-icon-template",
-        props: ["status"],
-
-        methods: {
-            getIcon: function() {
-                var icons = {
-                    waiting: "⌚",
-                    loading: "⚡",
-                    passing: "✔️️",
-                    failing: "❌",
-                    erroring: "☠️",
-                };
-
-                return icons[this.status];
-            }
-        },
-    });
-
-    Vue.component("test-stats", {
-        template: "#test-stats-template",
-        props: ["tests"],
-
-        methods: {
-            countTestsWithStatus: function(status) {
-                return this.tests.filter(function(test) {
-                    return test.status === status;
-                }).length;
-            },
-        },
-    });
-
-    Vue.component("tests", {
-        template: "#tests-template",
-        props: ["tests"],
-    });
-
-    window.TestRunner = new Vue({
-        el: "main",
-        data: {
-            tests: TestRegister.getTests(),
-        },
-
-        mounted: function() {
-            TestRegister.runTests();
-        },
-    });
-})();

test/NodeRunner.js

@@ -0,0 +1,24 @@
+/**
+ * NodeRunner.js
+ *
+ * The purpose of this file is to execute via PhantomJS the file
+ * PhantomRunner.js, because PhantomJS is managed by node.
+ *
+ * @author tlwr [toby@toby.codes
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ *
+ */
+var path = require("path");
+var phantomjs = require("phantomjs-prebuilt");
+
+var phantomEntryPoint = path.join(__dirname, "PhantomRunner.js");
+var program = phantomjs.exec(phantomEntryPoint);
+
+program.stdout.pipe(process.stdout);
+program.stderr.pipe(process.stderr);
+
+program.on("exit", function(status) {
+    process.exit(status);
+});

test/PhantomRunner.js

@@ -0,0 +1,80 @@
+/**
+ * PhantomRunner.js
+ *
+ * This file navigates to build/test/index.html and logs the test results.
+ *
+ * @author tlwr [toby@toby.codes
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ *
+ */
+var page = require("webpage").create();
+
+var allTestsPassing = true;
+var testStatusCounts = {
+    total: 0,
+};
+
+function statusToIcon(status) {
+    var icons = {
+        erroring: "🔥",
+        failing: "❌",
+        passing: "✔️️",
+    };
+    return icons[status] || "?";
+}
+
+page.onCallback = function(messageType) {
+    if (messageType === "testResult") {
+        var testResult = arguments[1];
+
+        allTestsPassing = allTestsPassing && testResult.status === "passing";
+        var newCount = (testStatusCounts[testResult.status] || 0) + 1;
+        testStatusCounts[testResult.status] = newCount;
+        testStatusCounts.total += 1;
+
+        console.log([
+            statusToIcon(testResult.status),
+            testResult.test.name
+        ].join(" "));
+
+        if (testResult.output) {
+            console.log(
+                testResult.output
+                    .trim()
+                    .replace(/^/, "\t")
+                    .replace(/\n/g, "\n\t")
+            );
+        }
+    } else if (messageType === "exit") {
+
+        console.log("\n");
+
+        for (var testStatus in testStatusCounts) {
+            var count = testStatusCounts[testStatus];
+            if (count > 0) {
+                console.log(testStatus.toUpperCase(), count);
+            }
+        }
+
+        if (!allTestsPassing) {
+            console.log("\n")
+            console.log("Not all tests are passing");
+        }
+
+        phantom.exit(allTestsPassing ? 0 : 1);
+    }
+};
+
+page.open("file:///home/toby/Code/CyberChef/build/test/index.html", function(status) {
+    if (status !== "success") {
+        console.log("STATUS", status);
+        phantom.exit(1);
+    }
+});
+
+setTimeout(function() {
+    // Timeout
+    phantom.exit(1);
+}, 10 * 1000);

test/TestRegister.js

@@ -0,0 +1,96 @@
+/**
+ * TestRegister.js
+ *
+ * This is so individual files can register their tests in one place, and
+ * ensure that they will get run by the frontend.
+ *
+ * @author tlwr [toby@toby.codes
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ *
+ */
+(function() {
+    /**
+     * Add a list of tests to the register.
+     *
+     * @class
+     */
+    function TestRegister() {
+        this.tests = [];
+    }
+
+    /**
+     * Add a list of tests to the register.
+     *
+     * @param {Object[]} tests
+     */
+    TestRegister.prototype.addTests = function(tests) {
+        this.tests = this.tests.concat(tests);
+    };
+
+    /**
+     * Returns the list of tests.
+     *
+     * @returns {Object[]} tests
+     */
+    TestRegister.prototype.getTests = function() {
+        return this.tests;
+    };
+
+    /**
+     * Runs all the tests in the register.
+     *
+     */
+    TestRegister.prototype.runTests = function() {
+        return Promise.all(
+            this.tests.map(function(test, i) {
+                var chef = new Chef();
+
+                return Promise.resolve(chef.bake(
+                    test.input,
+                    test.recipeConfig,
+                    {},
+                    0,
+                    0
+                ))
+                .then(function(result) {
+                    var ret = {
+                        test: test,
+                        status: null,
+                        output: null,
+                    };
+
+                    if (result.error) {
+                        if (test.expectedError) {
+                            ret.status = "passing";
+                        } else {
+                            ret.status = "erroring";
+                            ret.output = result.error.displayStr;
+                        }
+                    } else {
+                        if (test.expectedError) {
+                            ret.status = "failing";
+                            ret.output = "Expected an error but did not receive one.";
+                        } else if (result.result === test.expectedOutput) {
+                            ret.status = "passing";
+                        } else {
+                            ret.status = "failing";
+                            ret.output = [
+                                "Expected",
+                                "\t" + test.expectedOutput.replace(/\n/g, "\n\t"),
+                                "Received",
+                                "\t" + result.result.replace(/\n/g, "\n\t"),
+                            ].join("\n");
+                        }
+                    }
+
+                    return ret;
+                });
+            })
+        );
+    };
+
+    // Singleton TestRegister, keeping things simple and obvious.
+    window.TestRegister = new TestRegister();
+})();

test/TestRunner.js

@@ -0,0 +1,38 @@
+/**
+ * TestRunner.js
+ *
+ * This is for actually running the tests in the test register.
+ *
+ * @author tlwr [toby@toby.codes
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ *
+ */
+(function() {
+    document.addEventListener("DOMContentLoaded", function() {
+        TestRegister.runTests()
+        .then(function(results) {
+            results.forEach(function(testResult) {
+                if (typeof window.callPhantom === "function") {
+                    window.callPhantom(
+                        "testResult",
+                        testResult
+                    );
+                } else {
+                    var output = [
+                        "----------",
+                        testResult.test.name,
+                        testResult.status,
+                        testResult.output,
+                    ].join("<br>");
+                    document.body.innerHTML += "<div>" + output + "</div>";
+                }
+            });
+
+            if (typeof window.callPhantom === "function") {
+                window.callPhantom("exit");
+            }
+        });
+    });
+})();

test/tests/core.js

@@ -6,47 +6,47 @@
  *
  */
 TestRegister.addTests([
-    {
+    //{
-        name: "Example error",
+    //    name: "Example error",
-        input: "1\n2\na\n4",
+    //    input: "1\n2\na\n4",
-        expectedOutput: "1\n2\n3\n4",
+    //    expectedOutput: "1\n2\n3\n4",
-        recipeConfig: [
+    //    recipeConfig: [
-            {
+    //        {
-                op: "Fork",
+    //            op: "Fork",
-                args: ["\n", "\n", false],
+    //            args: ["\n", "\n", false],
-            },
+    //        },
-            {
+    //        {
-                op: "To Base",
+    //            op: "To Base",
-                args: [16],
+    //            args: [16],
-            },
+    //        },
-        ],
+    //    ],
-    },
+    //},
-    {
+    //{
-        name: "Example non-error when error was expected",
+    //    name: "Example non-error when error was expected",
-        input: "1",
+    //    input: "1",
-        expectedError: true,
+    //    expectedError: true,
-        recipeConfig: [
+    //    recipeConfig: [
-            {
+    //        {
-                op: "To Base",
+    //            op: "To Base",
-                args: [16],
+    //            args: [16],
-            },
+    //        },
-        ],
+    //    ],
-    },
+    //},
-    {
+    //{
-        name: "Example fail",
+    //    name: "Example fail",
-        input: "1\n2\na\n4",
+    //    input: "1\n2\na\n4",
-        expectedOutput: "1\n2\n3\n4",
+    //    expectedOutput: "1\n2\n3\n4",
-        recipeConfig: [
+    //    recipeConfig: [
-            {
+    //        {
-                op: "Fork",
+    //            op: "Fork",
-                args: ["\n", "\n", true],
+    //            args: ["\n", "\n", true],
-            },
+    //        },
-            {
+    //        {
-                op: "To Base",
+    //            op: "To Base",
-                args: [16],
+    //            args: [16],
-            },
+    //        },
-        ],
+    //    ],
-    },
+    //},
     {
         name: "Fork: nothing",
         input: "",