From 4308c717c36bb7e9679bdfc7b4670d52cddceb78 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 13 Mar 2020 14:59:48 +0000 Subject: [PATCH] Tests now display a progress bar and report long running tests --- .travis.yml | 2 +- Gruntfile.js | 14 +--- package-lock.json | 63 ++++++++++++-- package.json | 3 +- src/core/Utils.mjs | 2 +- src/node/api.mjs | 4 +- tests/lib/TestRegister.mjs | 166 +++++++++++++++++++++++++------------ tests/lib/utils.mjs | 11 ++- tests/node/index.mjs | 7 +- tests/operations/index.mjs | 7 +- 10 files changed, 196 insertions(+), 83 deletions(-) diff --git a/.travis.yml b/.travis.yml index a67e99b4..5c247911 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ before_script: - export NODE_OPTIONS=--max_old_space_size=2048 script: - grunt lint - - grunt test + - npm test - grunt testnodeconsumer - grunt prod --msg="$COMPILE_MSG" - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui diff --git a/Gruntfile.js b/Gruntfile.js index 1a9e3f2f..8ae71a46 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -36,11 +36,10 @@ module.exports = function (grunt) { "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex" ]); - grunt.registerTask("test", - "A task which runs all the operation tests in the tests directory.", + grunt.registerTask("configTests", + "A task which configures config files in preparation for tests to be run. Use `npm tests` to run tests.", [ - "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex", - "exec:nodeTests", "exec:opTests" + "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex" ]); grunt.registerTask("testui", @@ -55,7 +54,6 @@ module.exports = function (grunt) { "Lints the code base", ["eslint", "exec:repoSize"]); - grunt.registerTask("tests", "test"); grunt.registerTask("lint", "eslint"); grunt.registerTask("findModules", @@ -385,15 +383,9 @@ module.exports = function (grunt) { ]), sync: true }, - opTests: { - command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs" - }, browserTests: { command: "./node_modules/.bin/nightwatch --env prod" }, - nodeTests: { - command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs" - }, setupNodeConsumers: { command: chainCommands([ "echo '\n--- Testing node consumers ---'", diff --git a/package-lock.json b/package-lock.json index 7d1ef2c3..9e18b885 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4413,6 +4413,49 @@ "restore-cursor": "^3.1.0" } }, + "cli-progress": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.6.0.tgz", + "integrity": "sha512-elg6jkiDedYrvwqWSae2FGvtbMo37Lo04oI9jJ5cI43Ge3jrDPWzeL3axv7MgBLYHDY/kGio/CXa49m4MWMrNw==", + "dev": true, + "requires": { + "colors": "^1.1.2", + "string-width": "^2.1.1" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, "cli-spinners": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz", @@ -12204,6 +12247,14 @@ "request": "^2.81.0", "request-progress": "^2.0.1", "which": "^1.2.10" + }, + "dependencies": { + "progress": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=", + "dev": true + } } }, "phin": { @@ -12589,11 +12640,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, - "progress": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", - "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" - }, "promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", @@ -15027,6 +15073,13 @@ "more-entropy": ">=0.0.7", "progress": "~1.1.2", "uglify-js": "^3.1.9" + }, + "dependencies": { + "progress": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" + } } }, "true-case-path": { diff --git a/package.json b/package.json index 3de758e1..3a4c5f7a 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "babel-loader": "^8.0.6", "babel-plugin-dynamic-import-node": "^2.3.0", "chromedriver": "^80.0.1", + "cli-progress": "^3.6.0", "colors": "^1.4.0", "copy-webpack-plugin": "^5.0.5", "css-loader": "^3.2.1", @@ -159,7 +160,7 @@ "start": "grunt dev", "build": "grunt prod", "repl": "node src/node/repl.js", - "test": "grunt test", + "test": "grunt configTests && node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs && node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs", "test-node-consumer": "grunt testnodeconsumer", "testui": "grunt testui", "testuidev": "npx nightwatch --env=dev", diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index c99eccc9..6e39e436 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -1335,7 +1335,7 @@ export function sendStatusMessage(msg) { self.sendStatusMessage(msg); else if (isWebEnvironment()) app.alert(msg, 10000); - else if (isNodeEnvironment()) + else if (isNodeEnvironment() && !global.TESTING) // eslint-disable-next-line no-console console.debug(msg); } diff --git a/src/node/api.mjs b/src/node/api.mjs index 7d53084b..5733fb1d 100644 --- a/src/node/api.mjs +++ b/src/node/api.mjs @@ -282,11 +282,11 @@ export function help(input) { .map(result => result.hydrated); if (matches && matches.length) { - console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`); + // console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`); return matches; } - console.log("No results found."); + // console.log("No results found."); return null; } diff --git a/tests/lib/TestRegister.mjs b/tests/lib/TestRegister.mjs index 7a0e956e..2557b711 100644 --- a/tests/lib/TestRegister.mjs +++ b/tests/lib/TestRegister.mjs @@ -10,6 +10,8 @@ * @license Apache-2.0 */ import Chef from "../../src/core/Chef.mjs"; +import Utils from "../../src/core/Utils.mjs"; +import cliProgress from "cli-progress"; /** * Object to store and run the list of tests. @@ -47,68 +49,99 @@ class TestRegister { /** * Runs all the tests in the register. */ - runTests () { - console.log("Running tests..."); - return Promise.all( - this.tests.map(function(test, i) { - const chef = new Chef(); + async runTests () { + const progBar = new cliProgress.SingleBar({ + format: formatter, + stopOnComplete: true + }, cliProgress.Presets.shades_classic); + const testResults = []; - return chef.bake( - test.input, - test.recipeConfig, - {}, - 0, - false - ).then(function(result) { - const ret = { - test: test, - status: null, - output: null, - }; + console.log("Running operation tests..."); + progBar.start(this.tests.length, 0, { + msg: "Setting up" + }); - if (result.error) { - if (test.expectedError) { - ret.status = "passing"; - } else { - ret.status = "erroring"; - ret.output = result.error.displayStr; - } - } else { - if (test.expectedError) { - ret.status = "failing"; - ret.output = "Expected an error but did not receive one."; - } else if (result.result === test.expectedOutput) { - ret.status = "passing"; - } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) { - ret.status = "passing"; - } else { - ret.status = "failing"; - const expected = test.expectedOutput ? test.expectedOutput : - test.expectedMatch ? test.expectedMatch.toString() : "unknown"; - ret.output = [ - "Expected", - "\t" + expected.replace(/\n/g, "\n\t"), - "Received", - "\t" + result.result.replace(/\n/g, "\n\t"), - ].join("\n"); - } - } + for (const test of this.tests) { + progBar.update(testResults.length, { + msg: test.name + }); - return ret; - }); - }) - ); + const chef = new Chef(); + const result = await chef.bake( + test.input, + test.recipeConfig, + {}, + 0, + false + ); + + const ret = { + test: test, + status: null, + output: null, + duration: result.duration + }; + + if (result.error) { + if (test.expectedError) { + ret.status = "passing"; + } else { + ret.status = "erroring"; + ret.output = result.error.displayStr; + } + } else { + if (test.expectedError) { + ret.status = "failing"; + ret.output = "Expected an error but did not receive one."; + } else if (result.result === test.expectedOutput) { + ret.status = "passing"; + } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) { + ret.status = "passing"; + } else { + ret.status = "failing"; + const expected = test.expectedOutput ? test.expectedOutput : + test.expectedMatch ? test.expectedMatch.toString() : "unknown"; + ret.output = [ + "Expected", + "\t" + expected.replace(/\n/g, "\n\t"), + "Received", + "\t" + result.result.replace(/\n/g, "\n\t"), + ].join("\n"); + } + } + + testResults.push(ret); + progBar.increment(); + } + + return testResults; } /** * Run all api related tests and wrap results in report format */ - runApiTests() { - return Promise.all(this.apiTests.map(async function(test, i) { + async runApiTests() { + const progBar = new cliProgress.SingleBar({ + format: formatter, + stopOnComplete: true + }, cliProgress.Presets.shades_classic); + const testResults = []; + + console.log("Running Node API tests..."); + progBar.start(this.apiTests.length, 0, { + msg: "Setting up" + }); + + global.TESTING = true; + for (const test of this.apiTests) { + progBar.update(testResults.length, { + msg: test.name + }); + const result = { test: test, status: null, - output: null, + output: null }; try { await test.run(); @@ -117,10 +150,37 @@ class TestRegister { result.status = "erroring"; result.output = e.message; } - return result; - })); + + testResults.push(result); + progBar.increment(); + } + + return testResults; } } + +/** + * Formatter for the progress bar + * + * @param {Object} options + * @param {Object} params + * @param {Object} payload + * @returns {string} + */ +function formatter(options, params, payload) { + const bar = options.barCompleteString.substr(0, Math.round(params.progress * options.barsize)) + + options.barIncompleteString.substr(0, Math.round((1-params.progress) * options.barsize)); + + const percentage = Math.floor(params.progress * 100), + duration = Math.floor((Date.now() - params.startTime) / 1000); + + let testName = payload.msg ? payload.msg : ""; + if (params.value >= params.total) testName = "Tests completed"; + testName = Utils.truncate(testName, 25).padEnd(25, " "); + + return `${testName} ${bar} ${params.value}/${params.total} | ${percentage}% | Duration: ${duration}s`; +} + // Export an instance to make a singleton export default new TestRegister(); diff --git a/tests/lib/utils.mjs b/tests/lib/utils.mjs index 0f85ae30..da10754b 100644 --- a/tests/lib/utils.mjs +++ b/tests/lib/utils.mjs @@ -33,6 +33,10 @@ function handleTestResult(testStatus, testResult) { testStatus.allTestsPassing = testStatus.allTestsPassing && testResult.status === "passing"; testStatus.counts[testResult.status] = (testStatus.counts[testResult.status] || 0) + 1; testStatus.counts.total += 1; + + if (testResult.duration > 2000) { + console.log(`'${testResult.test.name}' took ${(testResult.duration / 1000).toFixed(2)}s to complete`); + } } /** @@ -42,8 +46,6 @@ function handleTestResult(testStatus, testResult) { * @param {Object[]} results - results from TestRegister */ export function logTestReport(testStatus, results) { - console.log("Tests completed."); - results.forEach(r => handleTestResult(testStatus, r)); console.log(); @@ -80,8 +82,9 @@ export function logTestReport(testStatus, results) { * Fail if the process takes longer than 60 seconds. */ export function setLongTestFailure() { + const timeLimit = 60; setTimeout(function() { - console.log("Tests took longer than 60 seconds to run, returning."); + console.log(`Tests took longer than ${timeLimit} seconds to run, returning.`); process.exit(1); - }, 60 * 1000); + }, timeLimit * 1000); } diff --git a/tests/node/index.mjs b/tests/node/index.mjs index 29c8c841..f6abba40 100644 --- a/tests/node/index.mjs +++ b/tests/node/index.mjs @@ -35,6 +35,7 @@ setLongTestFailure(); const logOpsTestReport = logTestReport.bind(null, testStatus); -TestRegister.runApiTests() - .then(logOpsTestReport); - +(async function() { + const results = await TestRegister.runApiTests(); + logOpsTestReport(results); +})(); diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 64b31c5f..7e5ef374 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -116,5 +116,8 @@ setLongTestFailure(); const logOpsTestReport = logTestReport.bind(null, testStatus); -TestRegister.runTests() - .then(logOpsTestReport); +(async function() { + const results = await TestRegister.runTests(); + logOpsTestReport(results); +})(); +