diff --git a/package-lock.json b/package-lock.json index 6df1b2c20..79e6278ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "jasmine": "^6.2.0", "jsdoc": "^4.0.5", "jstransformer-nunjucks": "^1.2.0", + "kleur": "^4.1.5", "metalsmith": "^2.7.0", "metalsmith-html-relative": "^2.0.11", "ordered-read-streams": "^2.0.0", @@ -7916,6 +7917,16 @@ "graceful-fs": "^4.1.9" } }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/last-run": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/last-run/-/last-run-2.0.0.tgz", diff --git a/package.json b/package.json index 68bb88bdb..5b1f65614 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "jasmine": "^6.2.0", "jsdoc": "^4.0.5", "jstransformer-nunjucks": "^1.2.0", + "kleur": "^4.1.5", "metalsmith": "^2.7.0", "metalsmith-html-relative": "^2.0.11", "ordered-read-streams": "^2.0.0", diff --git a/test/color_utils.mjs b/test/color_utils.mjs new file mode 100644 index 000000000..b684b8d5f --- /dev/null +++ b/test/color_utils.mjs @@ -0,0 +1,31 @@ +/* Copyright 2026 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import kleur from "kleur"; + +kleur.enabled = + !process.env.NO_COLOR && + (!!process.stdout.isTTY || + !!process.env.FORCE_COLOR || + process.env.GITHUB_ACTIONS === "true"); + +const TEST_PASSED = kleur.green("TEST-PASS"); +const TEST_UNEXPECTED_FAIL = kleur.red().bold("TEST-UNEXPECTED-FAIL"); + +function colorBrowser(name) { + return kleur.cyan(name); +} + +export { colorBrowser, TEST_PASSED, TEST_UNEXPECTED_FAIL }; diff --git a/test/integration/jasmine-boot.js b/test/integration/jasmine-boot.js index 26e1ce134..1c9d8118d 100644 --- a/test/integration/jasmine-boot.js +++ b/test/integration/jasmine-boot.js @@ -15,6 +15,7 @@ /* eslint-disable no-console */ +import { TEST_PASSED, TEST_UNEXPECTED_FAIL } from "../color_utils.mjs"; import Jasmine from "jasmine"; async function runTests(results) { @@ -50,6 +51,13 @@ async function runTests(results) { ], }); + function failureError(result) { + return result.failedExpectations + ?.map(item => item.message) + .filter(Boolean) + .join(" "); + } + jasmine.addReporter({ jasmineDone(suiteInfo) {}, jasmineStarted(suiteInfo) {}, @@ -62,10 +70,17 @@ async function runTests(results) { // Report on passed or failed tests. ++results.runs; if (result.status === "passed") { - console.log(`TEST-PASSED | ${result.description}`); + console.log(`${TEST_PASSED} | ${result.description}`); } else { ++results.failures; - console.log(`TEST-UNEXPECTED-FAIL | ${result.description}`); + const error = failureError(result); + results.failureList?.push({ + description: result.description, + error, + }); + console.log( + `${TEST_UNEXPECTED_FAIL} | ${result.description}${error ? ` | ${error}` : ""}` + ); } }, specStarted(result) {}, @@ -78,7 +93,14 @@ async function runTests(results) { // Report on failed suites only (indicates problems in setup/teardown). if (result.status === "failed") { ++results.failures; - console.log(`TEST-UNEXPECTED-FAIL | ${result.description}`); + const error = failureError(result); + results.failureList?.push({ + description: result.description, + error, + }); + console.log( + `${TEST_UNEXPECTED_FAIL} | ${result.description}${error ? ` | ${error}` : ""}` + ); } }, suiteStarted(result) {}, diff --git a/test/reporter.js b/test/reporter.js index 25d9b6ab8..ff3e7eb67 100644 --- a/test/reporter.js +++ b/test/reporter.js @@ -69,7 +69,7 @@ const TestReporter = function (browser) { // Report on passed or failed tests. if (result.status === "passed") { - sendResult("TEST-PASSED", result.description); + sendResult("TEST-PASS", result.description); } else { let failedMessages = ""; for (const item of result.failedExpectations) { diff --git a/test/test.mjs b/test/test.mjs index 85805b9e6..62d813b34 100644 --- a/test/test.mjs +++ b/test/test.mjs @@ -14,6 +14,11 @@ */ /* eslint-disable no-var */ +import { + colorBrowser, + TEST_PASSED, + TEST_UNEXPECTED_FAIL, +} from "./color_utils.mjs"; import { copySubtreeSync, ensureDirSync } from "./testutils.mjs"; import { COVERAGE_FORMAT_TO_REPORTER, @@ -423,7 +428,7 @@ function handleSessionTimeout(session) { return; } console.log( - `TEST-UNEXPECTED-FAIL | test failed ${session.name} has not responded in ${browserTimeout}s` + `${TEST_UNEXPECTED_FAIL} | test failed ${session.name} has not responded in ${browserTimeout}s` ); session.numErrors += session.remaining; session.remaining = 0; @@ -558,7 +563,7 @@ async function checkEq(task, results, session, masterMode) { ) === 0; if (!eq) { console.log( - `TEST-UNEXPECTED-FAIL | ${taskType} ${taskId} | in ${session.name} | rendering of page ${page + 1} != reference rendering` + `${TEST_UNEXPECTED_FAIL} | ${taskType} ${taskId} | in ${colorBrowser(session.name)} | rendering of page ${page + 1} != reference rendering` ); if (!testDirCreated) { @@ -605,7 +610,9 @@ async function checkEq(task, results, session, masterMode) { if (numEqFailures > 0) { session.numEqFailures += numEqFailures; } else { - console.log(`TEST-PASS | ${taskType} test ${taskId} | in ${session.name}`); + console.log( + `${TEST_PASSED} | ${taskType} test ${taskId} | in ${colorBrowser(session.name)}` + ); } } @@ -633,13 +640,13 @@ function checkFBF(task, results, session, masterMode) { // https://github.com/mozilla/pdf.js/issues/12371 if (masterMode) { console.log( - `TEST-SKIPPED | forward-back-forward test ${task.id} | in ${session.name} | page${page + 1}` + `TEST-SKIPPED | forward-back-forward test ${task.id} | in ${colorBrowser(session.name)} | page${page + 1}` ); continue; } console.log( - `TEST-UNEXPECTED-FAIL | forward-back-forward test ${task.id} | in ${session.name} | first rendering of page ${page + 1} != second` + `${TEST_UNEXPECTED_FAIL} | forward-back-forward test ${task.id} | in ${colorBrowser(session.name)} | first rendering of page ${page + 1} != second` ); numFBFFailures++; } @@ -649,7 +656,7 @@ function checkFBF(task, results, session, masterMode) { session.numFBFFailures += numFBFFailures; } else { console.log( - `TEST-PASS | forward-back-forward test ${task.id} | in ${session.name}` + `${TEST_PASSED} | forward-back-forward test ${task.id} | in ${colorBrowser(session.name)}` ); } } @@ -657,7 +664,9 @@ function checkFBF(task, results, session, masterMode) { function checkLoad(task, results, browser) { // Load just checks for absence of failure, so if we got here the // test has passed - console.log(`TEST-PASS | load test ${task.id} | in ${browser}`); + console.log( + `${TEST_PASSED} | load test ${task.id} | in ${colorBrowser(browser)}` + ); } async function checkRefTestResults(browser, id, results) { @@ -685,7 +694,7 @@ async function checkRefTestResults(browser, id, results) { "TEST-SKIPPED | PDF was not downloaded " + id + " | in " + - browser + + colorBrowser(browser) + " | page" + (page + 1) + " round " + @@ -698,16 +707,7 @@ async function checkRefTestResults(browser, id, results) { session.numErrors++; } console.log( - "TEST-UNEXPECTED-FAIL | test failed " + - id + - " | in " + - browser + - " | page" + - (page + 1) + - " round " + - (round + 1) + - " | " + - pageResult.failure + `${TEST_UNEXPECTED_FAIL} | test failed ${id} | in ${colorBrowser(browser)} | page${page + 1} round ${round + 1} | ${pageResult.failure}` ); } } @@ -835,6 +835,16 @@ function onAllSessionsClosedAfterTests(name) { } else if (numErrors > 0) { console.log("OHNOES! Some " + name + " tests failed!"); console.log(" " + numErrors + " of " + numRuns + " failed"); + console.log("Here are the failing tests:"); + for (const session of sessions) { + for (const { description, error } of session.failures ?? []) { + let line = ` - in ${colorBrowser(session.name)} | ${description}`; + if (error) { + line += ` | ${error.replaceAll(/\s+/g, " ").trim()}`; + } + console.log(line); + } + } } else { console.log("All " + name + " tests passed."); } @@ -854,6 +864,7 @@ async function startUnitTest(testUrl, name) { initializeSession: session => { session.numRuns = 0; session.numErrors = 0; + session.failures = []; }, }); } @@ -868,15 +879,17 @@ async function startIntegrationTest() { initializeSession: session => { session.numRuns = 0; session.numErrors = 0; + session.failures = []; }, }); global.integrationBaseUrl = `http://${host}:${server.port}/build/generic/web/viewer.html`; global.integrationSessions = sessions; - const results = { runs: 0, failures: 0 }; + const results = { runs: 0, failures: 0, failureList: [] }; await runTests(results); sessions[0].numRuns = results.runs; sessions[0].numErrors = results.failures; + sessions[0].failures = results.failureList; await Promise.all(sessions.map(session => closeSession(session.name))); } @@ -920,10 +933,20 @@ function unitTestPostHandler(parsedUrl, req, res) { } var session = getSession(data.browser); session.numRuns++; + let status = data.status; + if (status === "TEST-UNEXPECTED-FAIL") { + status = TEST_UNEXPECTED_FAIL; + } else if (status === "TEST-PASS") { + status = TEST_PASSED; + } var message = - data.status + " | " + data.description + " | in " + session.name; + status + " | " + data.description + " | in " + colorBrowser(session.name); if (data.status === "TEST-UNEXPECTED-FAIL") { session.numErrors++; + session.failures.push({ + description: data.description, + error: data.error, + }); } if (data.error) { message += " | " + data.error;