diff --git a/external/coverage_search/coverage_search.mjs b/external/coverage_search/coverage_search.mjs new file mode 100644 index 000000000..6055c0855 --- /dev/null +++ b/external/coverage_search/coverage_search.mjs @@ -0,0 +1,110 @@ +/* Copyright 2026 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from "fs"; +import { parseArgs } from "node:util"; +import path from "path"; + +const __dirname = import.meta.dirname; +const PROJECT_ROOT = path.join(__dirname, "../.."); + +const { values } = parseArgs({ + args: process.argv.slice(2), + options: { + code: { type: "string" }, + "coverage-dir": { type: "string", default: "build/coverage" }, + help: { type: "boolean", short: "h", default: false }, + }, +}); + +if (values.help || !values.code) { + console.log( + "Usage: coverage_search.mjs --code=:: [--coverage-dir=]\n\n" + + " --code Source file and line number or function name to search for.\n" + + " Examples:\n" + + " --code=canvas.js::205\n" + + " --code=canvas.js::drawImageAtIntegerCoords\n" + + " --coverage-dir Coverage directory containing per-test-index.json [build/coverage]\n\n" + + "Prints to stdout the IDs of tests whose coverage includes the given line or\n" + + "function (one ID per line).\n" + + "Run browsertest with --coverage-per-test first to generate the index." + ); + process.exit(values.help ? 0 : 1); +} + +const sep = values.code.indexOf("::"); +if (sep === -1) { + console.error( + "Error: --code must be in format 'file.js::line_or_function', e.g. canvas.js::205" + ); + process.exit(1); +} + +const fileName = values.code.slice(0, sep); +const location = values.code.slice(sep + 2); +const isLine = /^\d+$/.test(location); +const lineNum = isLine ? parseInt(location, 10) : null; +const funcName = isLine ? null : location; + +const coverageDir = path.isAbsolute(values["coverage-dir"]) + ? values["coverage-dir"] + : path.join(PROJECT_ROOT, values["coverage-dir"]); + +const indexPath = path.join(coverageDir, "per-test-index.json"); +if (!fs.existsSync(indexPath)) { + console.error(`Error: index file not found: ${indexPath}`); + console.error("Run browsertest with --coverage-per-test first."); + process.exit(1); +} + +const { ids, files } = JSON.parse(fs.readFileSync(indexPath, "utf8")); + +// Find the file entry whose path matches fileName. +let fileEntry = null; +for (const [filePath, entry] of Object.entries(files)) { + if ( + filePath === fileName || + filePath.endsWith(`/${fileName}`) || + filePath.endsWith(`\\${fileName}`) + ) { + fileEntry = entry; + break; + } +} + +if (!fileEntry) { + process.exit(0); +} + +let testIndices = null; + +if (lineNum !== null) { + // Direct line lookup. + testIndices = fileEntry.l?.[lineNum]; + + // If no hit, check whether lineNum is a function declaration start and + // redirect to that function's coverage. + if (!testIndices && fileEntry.fstarts?.[lineNum]) { + testIndices = fileEntry.f?.[fileEntry.fstarts[lineNum]]; + } +} else { + testIndices = fileEntry.f?.[funcName]; +} + +if (testIndices) { + for (const idx of testIndices) { + console.log(ids[idx]); + } +} diff --git a/gulpfile.mjs b/gulpfile.mjs index 09ff243d1..5c00ecedf 100644 --- a/gulpfile.mjs +++ b/gulpfile.mjs @@ -110,6 +110,7 @@ const BABEL_PRESET_ENV_OPTS = Object.freeze({ const DEFINES = Object.freeze({ SKIP_BABEL: true, TESTING: undefined, + COVERAGE: undefined, // The main build targets: GENERIC: false, MOZCENTRAL: false, @@ -288,6 +289,10 @@ function createWebpackConfig( BUNDLE_VERSION: versionInfo.version, BUNDLE_BUILD: versionInfo.commit, TESTING: defines.TESTING ?? process.env.TESTING === "true", + COVERAGE: + defines.COVERAGE ?? + (process.argv.includes("--coverage") || + process.argv.includes("--coverage-per-test")), DEFAULT_FTL: defines.GENERIC ? getDefaultFtl() : "", }; const licenseHeaderLibre = fs @@ -304,7 +309,7 @@ function createWebpackConfig( !bundleDefines.CHROME && !bundleDefines.LIB && !bundleDefines.MINIFIED && - !bundleDefines.TESTING && + (!bundleDefines.TESTING || bundleDefines.COVERAGE) && !disableSourceMaps; const isModule = output.library?.type === "module"; const isMinified = bundleDefines.MINIFIED; @@ -328,6 +333,9 @@ function createWebpackConfig( }, ], ]; + if (bundleDefines.COVERAGE) { + babelPlugins.push("babel-plugin-istanbul"); + } const plugins = []; if (!disableLicenseHeader) { @@ -669,6 +677,18 @@ function getTempFile(prefix, suffix) { return filePath; } +function getArgValue(name) { + for (let i = 0; i < process.argv.length; i++) { + if (process.argv[i] === name && i + 1 < process.argv.length) { + return process.argv[i + 1]; + } + if (process.argv[i].startsWith(name + "=")) { + return process.argv[i].slice(name.length + 1); + } + } + return null; +} + function runTests(testsName, { bot = false } = {}) { return new Promise((resolve, reject) => { console.log("\n### Running " + testsName + " tests"); @@ -719,7 +739,10 @@ function runTests(testsName, { bot = false } = {}) { if (process.argv.includes("--headless")) { args.push("--headless"); } - if (process.argv.includes("--coverage")) { + if ( + process.argv.includes("--coverage") || + process.argv.includes("--coverage-per-test") + ) { args.push("--coverage"); } if (process.argv.includes("--coverage-output")) { @@ -728,6 +751,45 @@ function runTests(testsName, { bot = false } = {}) { process.argv[process.argv.indexOf("--coverage-output") + 1] ); } + const coverageFormatsArg = getArgValue("--coverage-formats"); + if (coverageFormatsArg) { + args.push("--coverageFormats", coverageFormatsArg); + } + if (process.argv.includes("--coverage-per-test")) { + args.push("--coveragePerTest"); + } + + const codeArg = testsName === "browser" ? getArgValue("--code") : null; + if (codeArg) { + const coverageDir = + getArgValue("--coverage-output") || BUILD_DIR + "coverage"; + const result = spawnSync( + "node", + [ + path.join(__dirname, "external/coverage_search/coverage_search.mjs"), + `--code=${codeArg}`, + `--coverage-dir=${coverageDir}`, + ], + { encoding: "utf8" } + ); + if (result.status !== 0) { + reject(new Error(result.stderr?.trim() || "coverage_search failed")); + return; + } + const testIds = result.stdout.trim().split("\n").filter(Boolean); + if (testIds.length === 0) { + console.log(`\n### No tests found covering "${codeArg}"`); + resolve(); + return; + } + console.log( + `\n### Found ${testIds.length} test(s) covering "${codeArg}":\n` + + testIds.map(id => ` ${id}`).join("\n") + ); + for (const id of testIds) { + args.push(`-t=${id}`); + } + } const testProcess = startNode(args, { cwd: TEST_DIR, stdio: "inherit" }); testProcess.on("close", function (code) { @@ -813,6 +875,25 @@ function makeRef(done, bot) { if (process.argv.includes("--headless")) { args.push("--headless"); } + if ( + process.argv.includes("--coverage") || + process.argv.includes("--coverage-per-test") + ) { + args.push("--coverage"); + } + if (process.argv.includes("--coverage-output")) { + args.push( + "--coverageOutput", + process.argv[process.argv.indexOf("--coverage-output") + 1] + ); + } + const coverageFormatsArg = getArgValue("--coverage-formats"); + if (coverageFormatsArg) { + args.push("--coverageFormats", coverageFormatsArg); + } + if (process.argv.includes("--coverage-per-test")) { + args.push("--coveragePerTest"); + } collectArgs( [ { names: ["-t", "--testfilter"], hasValue: true }, @@ -831,6 +912,39 @@ function makeRef(done, bot) { }); } +// Queries the per-test coverage index built by --coverage-per-test and prints +// the IDs of tests that exercised a given source file location. Run with +// --code=::, e.g. --code=canvas.js::205 +gulp.task("coverage_search", function (done) { + const codeArg = getArgValue("--code"); + if (!codeArg) { + done(new Error('Missing --code argument, e.g. --code="canvas.js::205"')); + return; + } + const coverageDir = + getArgValue("--coverage-output") || BUILD_DIR + "coverage"; + const result = spawnSync( + "node", + [ + path.join(__dirname, "external/coverage_search/coverage_search.mjs"), + `--code=${codeArg}`, + `--coverage-dir=${coverageDir}`, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] } + ); + if (result.stderr) { + process.stderr.write(result.stderr); + } + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.status !== 0) { + done(new Error("coverage_search failed")); + return; + } + done(); +}); + gulp.task("default", function (done) { console.log("Available tasks:"); const tasks = Object.keys(gulp.registry().tasks()); diff --git a/src/core/worker.js b/src/core/worker.js index 0544c93ef..3f2e677b9 100644 --- a/src/core/worker.js +++ b/src/core/worker.js @@ -98,6 +98,12 @@ class WorkerMessageHandler { }); handler.on("GetDocRequest", data => this.createDocumentHandler(data, port)); + + if (typeof PDFJSDev === "undefined" || PDFJSDev.test("TESTING")) { + handler.on("GetWorkerCoverage", function () { + return globalThis.__coverage__ ?? {}; + }); + } } static createDocumentHandler(docParams, port) { diff --git a/test/driver.js b/test/driver.js index 7ab52dc52..cf3438800 100644 --- a/test/driver.js +++ b/test/driver.js @@ -21,6 +21,7 @@ const { getDocument, GlobalWorkerOptions, OutputScale, + PDFWorker, PixelsPerInch, shadow, TextLayer, @@ -495,6 +496,14 @@ function buffersEqual(a, b) { } class Driver { + #pdfWorker = null; + + #coveragePerTest = false; + + #prevMainCoverage = null; + + #prevWorkerCoverage = null; + /** * @param {DriverOptions} options */ @@ -523,6 +532,14 @@ class Driver { this.sessionIndex = parseInt(params.get("sessionindex") || "0", 10); this.sessionCount = parseInt(params.get("sessioncount") || "1", 10); + // When coverage is enabled, share a single persistent worker across all + // tasks so that the accumulated self.__coverage__ can be retrieved at the + // end before the worker is destroyed. + if (window.__coverage__) { + this.#pdfWorker = new PDFWorker({ name: "coverage-worker" }); + this.#coveragePerTest = params.get("coveragepertest") === "true"; + } + // Open a persistent WebSocket connection to the server for binary result // submission. this.ws = new WebSocket(`ws://${location.host}`); @@ -631,13 +648,34 @@ class Driver { _nextTask() { let failure = ""; + const completedTask = this.currentTask; this._cleanup().then(async () => { + if (completedTask && this.#coveragePerTest) { + await this._sendPerTestCoverage(completedTask.id); + } + const task = await this._waitForNextTask(); if (!task) { this._done(); return; } + + if (this.#coveragePerTest) { + this.#prevMainCoverage = structuredClone(window.__coverage__ ?? {}); + if (this.#pdfWorker) { + try { + this.#prevWorkerCoverage = + await this.#pdfWorker.messageHandler.sendWithPromise( + "GetWorkerCoverage", + null + ); + } catch { + this.#prevWorkerCoverage = {}; + } + } + } + this.currentTask = task; task.round = 0; @@ -898,6 +936,7 @@ class Driver { isOffscreenCanvasSupported: task.isOffscreenCanvasSupported === false ? false : undefined, disableFontFace: task.disableFontFace === true, + ...(this.#pdfWorker ? { worker: this.#pdfWorker } : {}), }; } @@ -1410,11 +1449,135 @@ class Driver { if (this.inFlightRequests > 0) { this.inflight.textContent = this.inFlightRequests; setTimeout(this._done.bind(this), WAITING_TIME); + } else if (this.#pdfWorker) { + this._collectWorkerCoverage().then(() => { + setTimeout(this._quit.bind(this), WAITING_TIME); + }); } else { setTimeout(this._quit.bind(this), WAITING_TIME); } } + #computeCoverageDelta(afterCoverage, beforeCoverage) { + const delta = {}; + for (const [key, afterFile] of Object.entries(afterCoverage)) { + const beforeFile = beforeCoverage?.[key]; + const lines = new Set(); + const funcs = new Set(); + + for (const [id, count] of Object.entries(afterFile.s)) { + if (count - (beforeFile?.s[id] ?? 0) > 0) { + const line = afterFile.statementMap?.[id]?.start?.line; + if (line !== undefined) { + lines.add(line); + } + } + } + for (const [id, count] of Object.entries(afterFile.f)) { + if (count - (beforeFile?.f[id] ?? 0) > 0) { + const name = afterFile.fnMap?.[id]?.name; + if (name) { + funcs.add(name); + } + } + } + if (lines.size > 0 || funcs.size > 0) { + const fstarts = Object.create(null); + for (const fn of Object.values(afterFile.fnMap ?? {})) { + const startLine = fn.decl?.start?.line; + if (startLine !== undefined && fn.name) { + fstarts[startLine] = fn.name; + } + } + delta[key] = { fstarts, lines: [...lines], funcs: [...funcs] }; + } + } + return delta; + } + + async _sendPerTestCoverage(taskId) { + if (!window.__coverage__) { + return; + } + const delta = this.#computeCoverageDelta( + window.__coverage__, + this.#prevMainCoverage + ); + if (this.#pdfWorker) { + try { + const afterWorker = + await this.#pdfWorker.messageHandler.sendWithPromise( + "GetWorkerCoverage", + null + ); + for (const [key, entry] of Object.entries( + this.#computeCoverageDelta(afterWorker, this.#prevWorkerCoverage) + )) { + if (delta[key]) { + Object.assign(delta[key].fstarts, entry.fstarts); + const lineSet = new Set(delta[key].lines); + for (const l of entry.lines) { + lineSet.add(l); + } + delta[key].lines = [...lineSet]; + const funcSet = new Set(delta[key].funcs); + for (const f of entry.funcs) { + funcSet.add(f); + } + delta[key].funcs = [...funcSet]; + } else { + delta[key] = entry; + } + } + } catch { + // ignore + } + } + if (Object.keys(delta).length > 0) { + this.ws.send( + JSON.stringify({ type: "coverage", id: taskId, counts: delta }) + ); + } + } + + async _collectWorkerCoverage() { + try { + const workerCoverage = + await this.#pdfWorker.messageHandler.sendWithPromise( + "GetWorkerCoverage", + null + ); + if (workerCoverage && Object.keys(workerCoverage).length > 0) { + window.__coverage__ ??= {}; + for (const [key, fileCoverage] of Object.entries(workerCoverage)) { + if (window.__coverage__[key]) { + // Istanbul coverage objects use s (statements), b (branches), and + // f (functions) as shorthand keys for the hit-count maps. + for (const id of Object.keys(fileCoverage.s)) { + window.__coverage__[key].s[id] = + (window.__coverage__[key].s[id] ?? 0) + fileCoverage.s[id]; + } + for (const id of Object.keys(fileCoverage.b)) { + window.__coverage__[key].b[id] = fileCoverage.b[id].map( + (c, i) => (window.__coverage__[key].b[id]?.[i] ?? 0) + c + ); + } + for (const id of Object.keys(fileCoverage.f)) { + window.__coverage__[key].f[id] = + (window.__coverage__[key].f[id] ?? 0) + fileCoverage.f[id]; + } + } else { + window.__coverage__[key] = fileCoverage; + } + } + } + } catch (e) { + console.warn(`Failed to collect worker coverage: ${e}`); + } + this.#pdfWorker.destroy(); + this.#pdfWorker = null; + } + async _sendResult(snapshotBlob, task, failure, baselineBlob = null) { // Build a binary WebSocket frame: // [4 bytes BE: meta_len][meta JSON][4 bytes BE: snapshot_len] diff --git a/test/test.mjs b/test/test.mjs index 6a4f5bd6e..ac74e1c41 100644 --- a/test/test.mjs +++ b/test/test.mjs @@ -66,11 +66,19 @@ function stripPrivatePngChunks(buf) { } function parseOptions() { + // Expand `-X=value` short-option forms into `["-X", "value"]` since + // parseArgs only strips the `=` separator for long options (--foo=bar). + const args = process.argv.slice(2).flatMap(arg => { + const m = arg.match(/^(-[a-zA-Z])=(.*)/s); + return m ? [m[1], m[2]] : [arg]; + }); const { values } = parseArgs({ - args: process.argv.slice(2), + args, options: { coverage: { type: "boolean", default: false }, + coverageFormats: { type: "string", default: "info" }, coverageOutput: { type: "string", default: "build/coverage" }, + coveragePerTest: { type: "boolean", default: false }, downloadOnly: { type: "boolean", default: false }, fontTest: { type: "boolean", default: false }, headless: { type: "boolean", default: false }, @@ -97,7 +105,9 @@ function parseOptions() { console.log( "Usage: test.mjs\n\n" + " --coverage Enable code coverage collection.\n" + + " --coverageFormats Comma-separated list of coverage output formats: info,html,json,text,cobertura,clover. [info]\n" + " --coverageOutput Directory for code coverage data. [build/coverage]\n" + + " --coveragePerTest Generate individual coverage reports per test.\n" + " --downloadOnly Download test PDFs without running the tests.\n" + " --fontTest Run the font tests.\n" + " --headless Run tests without visible browser windows.\n" + @@ -329,6 +339,11 @@ async function startRefTest(masterMode, showRefImages) { } else { ws.send(JSON.stringify({ type: "done" })); } + } else if (msg.type === "coverage") { + if (global.coveragePerTest) { + const { id, counts } = msg; + accumulatePerTestCoverage(id, counts); + } } else if (msg.type === "quit") { const session = getSession(msg.browser); monitorBrowserTimeout(session, null); @@ -1055,7 +1070,8 @@ async function startBrowsers({ baseUrl, initializeSession, numSessions = 1 }) { startUrl = `${baseUrl}?browser=${encodeURIComponent(sessionName)}` + `&testFilter=${JSON.stringify(options.testfilter)}` + - `&delay=${options.statsDelay}&masterMode=${options.masterMode}`; + `&delay=${options.statsDelay}&masterMode=${options.masterMode}` + + `&coveragePerTest=${global.coveragePerTest || false}`; } await startBrowser({ browserName, startUrl }) .then(async function (browser) { @@ -1091,6 +1107,69 @@ function getSession(browser) { return sessions.find(session => session.name === browser); } +const COVERAGE_FORMAT_TO_REPORTER = { + info: "lcovonly", + html: "html", + json: "json", + text: "text", + cobertura: "cobertura", + clover: "clover", +}; + +function parseCoverageFormats(str) { + const formats = new Set(); + for (const fmt of str.split(",")) { + const name = fmt.trim(); + if (name && COVERAGE_FORMAT_TO_REPORTER[name]) { + formats.add(name); + } else if (name) { + console.warn( + `### Unknown coverage format "${name}", valid values: ${Object.keys(COVERAGE_FORMAT_TO_REPORTER).join(", ")}` + ); + } + } + return formats.size > 0 ? formats : new Set(["info"]); +} + +function accumulatePerTestCoverage(testId, counts) { + let testIdx = perTestIdMap.get(testId); + if (testIdx === undefined) { + testIdx = perTestIds.length; + perTestIds.push(testId); + perTestIdMap.set(testId, testIdx); + } + for (const [fileKey, { fstarts, lines, funcs }] of Object.entries(counts)) { + let entry = perTestFileIndex.get(fileKey); + if (!entry) { + entry = { + fstarts: Object.create(null), + lineMap: new Map(), + funcMap: new Map(), + }; + perTestFileIndex.set(fileKey, entry); + } + if (fstarts && Object.keys(entry.fstarts).length === 0) { + Object.assign(entry.fstarts, fstarts); + } + for (const line of lines) { + let set = entry.lineMap.get(line); + if (!set) { + set = new Set(); + entry.lineMap.set(line, set); + } + set.add(testIdx); + } + for (const func of funcs) { + let set = entry.funcMap.get(func); + if (!set) { + set = new Set(); + entry.funcMap.set(func, set); + } + set.add(testIdx); + } + } +} + async function writeCoverageData(outputDirectory) { try { console.log("\n### Writing code coverage data"); @@ -1105,18 +1184,56 @@ async function writeCoverageData(outputDirectory) { } } + const projectRoot = path.join(__dirname, ".."); + // create a context for report generation const context = libReport.createContext({ dir: path.join(__dirname, "..", outputDirectory), coverageMap: mergedCoverage, }); - const report = istanbulReportGenerator.create("lcovonly", { - projectRoot: path.join(__dirname, ".."), - }); - report.execute(context); + for (const fmt of global.coverageFormats ?? ["info"]) { + istanbulReportGenerator + .create(COVERAGE_FORMAT_TO_REPORTER[fmt], { projectRoot }) + .execute(context); + } - console.log(`Total files covered: ${Object.keys(mergedCoverage).length}`); + console.log(`Total files covered: ${mergedCoverage.files().length}`); + + if (global.coveragePerTest && perTestIds.length > 0) { + const files = Object.create(null); + for (const [fileKey, { fstarts, lineMap, funcMap }] of perTestFileIndex) { + const fileObj = Object.create(null); + if (Object.keys(fstarts).length > 0) { + fileObj.fstarts = fstarts; + } + if (lineMap.size > 0) { + const l = Object.create(null); + for (const [line, tests] of lineMap) { + l[line] = [...tests].sort((a, b) => a - b); + } + fileObj.l = l; + } + if (funcMap.size > 0) { + const f = Object.create(null); + for (const [name, tests] of funcMap) { + f[name] = [...tests].sort((a, b) => a - b); + } + fileObj.f = f; + } + files[fileKey] = fileObj; + } + const indexPath = path.join( + __dirname, + "..", + outputDirectory, + "per-test-index.json" + ); + fs.writeFileSync(indexPath, JSON.stringify({ ids: perTestIds, files })); + console.log( + `Per-test index written to ${outputDirectory}/per-test-index.json` + ); + } } catch (err) { console.error("Failed to write coverage data:", err); } @@ -1133,9 +1250,10 @@ async function closeSession(browser) { try { // Extract window.__coverage__ which is populated by // babel-plugin-istanbul - const coverage = await session.page.evaluate( - () => window.__coverage__ + const coverageJson = await session.page.evaluate(() => + JSON.stringify(window.__coverage__) ); + const coverage = coverageJson ? JSON.parse(coverageJson) : null; if (coverage && Object.keys(coverage).length > 0) { session.coverage = coverage; @@ -1194,12 +1312,22 @@ async function main() { stats = []; } - if (options.coverage) { + if (options.coverage || options.coveragePerTest) { global.coverageEnabled = true; console.log("\n### Code coverage enabled for browser tests"); if (options.coverageOutput) { global.coverageOutput = options.coverageOutput; - console.log(`### Code coverage output file: ${options.coverageOutput}`); + console.log( + `### Code coverage output directory: ${options.coverageOutput}` + ); + } + global.coverageFormats = parseCoverageFormats(options.coverageFormats); + console.log( + `### Coverage formats: ${[...global.coverageFormats].join(", ")}` + ); + if (options.coveragePerTest) { + global.coveragePerTest = true; + console.log("### Per-test coverage reports enabled"); } } @@ -1237,6 +1365,9 @@ var stats; var tempDir = null; var taskQueue = new Map(); var refPngCache = new Map(); +const perTestIds = []; +const perTestIdMap = new Map(); +const perTestFileIndex = new Map(); main();