From b6fac7642998b9255d94881a9f33ad15bcf930b5 Mon Sep 17 00:00:00 2001 From: Calixte Denizet Date: Wed, 15 Apr 2026 15:45:12 +0200 Subject: [PATCH] Improve reftest runner performance - Replace base64/JSON POST image submission with binary WebSocket frames, avoiding base64 overhead and per-request HTTP costs; quit is also sent over the same WS channel to guarantee ordering - Prefetch the next task's PDF in the worker while the current task is still rendering - Use `getImageData` instead of `toBlob` for partial-test baseline comparison (synchronous, no encoding); only encode to PNG in master mode - Disable bounce tracking protection in Firefox to prevent EBUSY errors from Puppeteer's profile cleanup on Windows --- test/driver.js | 205 +++++++++++++++++++++---------- test/test.mjs | 293 +++++++++++++++++++++++++-------------------- test/webserver.mjs | 11 ++ 3 files changed, 319 insertions(+), 190 deletions(-) diff --git a/test/driver.js b/test/driver.js index aeb7abfaa..f50245c7a 100644 --- a/test/driver.js +++ b/test/driver.js @@ -480,6 +480,20 @@ class Rasterize { * @property {HTMLDivElement} end - Container for a completion message. */ +function buffersEqual(a, b) { + if (a.byteLength !== b.byteLength) { + return false; + } + const v1 = new Uint8Array(a); + const v2 = new Uint8Array(b); + for (let i = 0; i < v1.length; i++) { + if (v1[i] !== v2[i]) { + return false; + } + } + return true; +} + class Driver { /** * @param {DriverOptions} options @@ -509,6 +523,11 @@ class Driver { this.sessionIndex = parseInt(params.get("sessionindex") || "0", 10); this.sessionCount = parseInt(params.get("sessioncount") || "1", 10); + // Open a persistent WebSocket connection to the server for binary result + // submission. + this.ws = new WebSocket(`ws://${location.host}`); + this.ws.binaryType = "arraybuffer"; + // Create a working canvas this.canvas = document.createElement("canvas"); } @@ -538,6 +557,11 @@ class Driver { // When gathering the stats the numbers seem to be more reliable // if the browser is given more time to start. setTimeout(async () => { + if (this.ws.readyState !== WebSocket.OPEN) { + await new Promise(resolve => { + this.ws.addEventListener("open", resolve, { once: true }); + }); + } const response = await fetch(this.manifestFile); if (!response.ok) { throw new Error(response.statusText); @@ -622,6 +646,8 @@ class Driver { const prevFile = md5FileMap.get(task.md5); if (prevFile) { if (task.file !== prevFile) { + task._prefetchedLoadingTask?.destroy(); + task._prefetchedLoadingTask = null; this._nextPage( task, `The "${task.file}" file is identical to the previously used "${prevFile}" file.` @@ -659,6 +685,10 @@ class Driver { this._log(` Loading file "${task.file}"\n`); + // Start fetching and parsing the next task's PDF in the worker + // now, so it overlaps with the current task's load and render time. + this._prefetchNextTask(); + try { let xfaStyleElement = null; if (task.enableXfa) { @@ -670,28 +700,13 @@ class Driver { .getElementsByTagName("head")[0] .append(xfaStyleElement); } - const isOffscreenCanvasSupported = - task.isOffscreenCanvasSupported === false ? false : undefined; - const disableFontFace = task.disableFontFace === true; - const documentOptions = { - url: new URL(task.file, window.location), - password: task.password, - cMapUrl: CMAP_URL, - iccUrl: ICC_URL, - standardFontDataUrl: STANDARD_FONT_DATA_URL, - wasmUrl: WASM_URL, - disableAutoFetch: !task.enableAutoFetch, - pdfBug: true, - useSystemFonts: task.useSystemFonts, - useWasm: task.useWasm, - useWorkerFetch: task.useWorkerFetch, - enableXfa: task.enableXfa, - isOffscreenCanvasSupported, + ...this._getDocumentOptions(task), styleElement: xfaStyleElement, - disableFontFace, }; - const loadingTask = getDocument(documentOptions); + const loadingTask = + task._prefetchedLoadingTask ?? getDocument(documentOptions); + task._prefetchedLoadingTask = null; let promise = loadingTask.promise; if (!this.masterMode && task.type === "extract") { @@ -849,6 +864,44 @@ class Driver { }); } + _getDocumentOptions(task) { + return { + url: new URL(task.file, window.location), + password: task.password, + cMapUrl: CMAP_URL, + iccUrl: ICC_URL, + standardFontDataUrl: STANDARD_FONT_DATA_URL, + wasmUrl: WASM_URL, + disableAutoFetch: !task.enableAutoFetch, + pdfBug: true, + useSystemFonts: task.useSystemFonts, + useWasm: task.useWasm, + useWorkerFetch: task.useWorkerFetch, + enableXfa: task.enableXfa, + isOffscreenCanvasSupported: + task.isOffscreenCanvasSupported === false ? false : undefined, + disableFontFace: task.disableFontFace === true, + }; + } + + _prefetchNextTask() { + const nextIdx = this.currentTask + 1; + if (nextIdx >= this.manifest.length) { + return; + } + const task = this.manifest[nextIdx]; + // Skip tasks that do not load a PDF or that need DOM setup (XFA style + // element injection) to happen synchronously before getDocument. + if ( + task.type === "skip-because-failing" || + task.type === "other" || + task.enableXfa + ) { + return; + } + task._prefetchedLoadingTask = getDocument(this._getDocumentOptions(task)); + } + _cleanup() { // Clear out all the stylesheets since a new one is created for each font. while (document.styleSheets.length > 0) { @@ -896,14 +949,15 @@ class Driver { let ctx; if (!task.pdfDoc) { - const dataUrl = this.canvas.toDataURL("image/png"); - this._sendResult(dataUrl, task, failure).then(() => { - this._log( - "done" + (failure ? " (failed !: " + failure + ")" : "") + "\n" - ); - this.currentTask++; - this._nextTask(); - }); + new Promise(r => { + this.canvas.toBlob(r, "image/png"); + }) + .then(blob => this._sendResult(blob, task, failure)) + .then(() => { + this._log(`done${failure ? ` (failed !: ${failure})` : ""}\n`); + this.currentTask++; + this._nextTask(); + }); return; } @@ -1177,7 +1231,21 @@ class Driver { }; clearOutsidePartial(); - const baseline = ctx.canvas.toDataURL("image/png"); + // Capture pixel data synchronously for the comparison; + // only encode to PNG in master mode where the server needs + // to save it as the reference image. + const baselinePixels = ctx.getImageData( + 0, + 0, + ctx.canvas.width, + ctx.canvas.height + ); + const baselineBlob = + this.masterMode && !task.knownPartialMismatch + ? await new Promise(r => { + ctx.canvas.toBlob(r, "image/png"); + }) + : null; this._clearCanvas(); const recordedBBoxes = page.recordedBBoxes; @@ -1223,7 +1291,8 @@ class Driver { // one pixel of a very slightly different shade), so we // avoid compating them to the non-optimized version and // instead use the optimized version also for makeref. - task.knownPartialMismatch ? null : baseline + task.knownPartialMismatch ? null : baselinePixels, + baselineBlob ); return; } @@ -1268,32 +1337,31 @@ class Driver { ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); } - _snapshot(task, failure, baselineDataUrl = null) { + async _snapshot(task, failure, baselinePixels = null, baselineBlob = null) { this._log("Snapshotting... "); - - const dataUrl = this.canvas.toDataURL("image/png"); - - if (baselineDataUrl && baselineDataUrl !== dataUrl) { - failure ||= "Optimized rendering differs from full rendering."; - } - - this._sendResult(dataUrl, task, failure, baselineDataUrl).then(() => { - this._log( - "done" + (failure ? " (failed !: " + failure + ")" : "") + "\n" - ); - task.pageNum++; - this._nextPage(task); + const snapshotBlob = await new Promise(r => { + this.canvas.toBlob(r, "image/png"); }); + if (baselinePixels) { + const snapPixels = this.canvas + .getContext("2d") + .getImageData(0, 0, this.canvas.width, this.canvas.height); + if (!buffersEqual(baselinePixels.data.buffer, snapPixels.data.buffer)) { + failure ||= "Optimized rendering differs from full rendering."; + } + } + await this._sendResult(snapshotBlob, task, failure, baselineBlob); + this._log(`done${failure ? ` (failed !: ${failure})` : ""}\n`); + task.pageNum++; + this._nextPage(task); } _quit() { this._log("Done !"); this.end.textContent = "Tests finished. Close this window!"; - - // Send the quit request - fetch(`/tellMeToQuit?browser=${escape(this.browser)}`, { - method: "POST", - }); + // Send quit over the same WebSocket channel so the server processes it + // only after all preceding result frames have been handled. + this.ws.send(JSON.stringify({ type: "quit", browser: this.browser })); } _info(message) { @@ -1331,8 +1399,11 @@ class Driver { } } - _sendResult(snapshot, task, failure, baselineSnapshot = null) { - const result = JSON.stringify({ + async _sendResult(snapshotBlob, task, failure, baselineBlob = null) { + // Build a binary WebSocket frame: + // [4 bytes BE: meta_len][meta JSON][4 bytes BE: snapshot_len] + // [snapshot PNG][baseline PNG] + const meta = JSON.stringify({ browser: this.browser, id: task.id, numPages: task.pdfDoc ? task.lastPage || task.pdfDoc.numPages : 0, @@ -1342,14 +1413,34 @@ class Driver { file: task.file, round: task.round, page: task.pageMapping?.[task.pageNum] ?? task.pageNum, - snapshot, - baselineSnapshot, stats: task.stats.times, viewportWidth: task.viewportWidth, viewportHeight: task.viewportHeight, outputScale: task.outputScale, }); - return this._send("/submit_task_results", result); + const metaBytes = new TextEncoder().encode(meta); + const snapshotBytes = new Uint8Array(await snapshotBlob.arrayBuffer()); + const baselineBytes = baselineBlob + ? new Uint8Array(await baselineBlob.arrayBuffer()) + : null; + + const totalLen = + 4 + + metaBytes.length + + 4 + + snapshotBytes.length + + (baselineBytes?.length ?? 0); + const buf = new ArrayBuffer(totalLen); + const view = new DataView(buf); + const bytes = new Uint8Array(buf); + view.setUint32(0, metaBytes.length); + bytes.set(metaBytes, 4); + view.setUint32(4 + metaBytes.length, snapshotBytes.length); + bytes.set(snapshotBytes, 8 + metaBytes.length); + if (baselineBytes) { + bytes.set(baselineBytes, 8 + metaBytes.length + snapshotBytes.length); + } + this.ws.send(buf); } _send(url, message) { @@ -1358,26 +1449,20 @@ class Driver { fetch(url, { method: "POST", - headers: { - "Content-Type": "application/json", - }, + headers: { "Content-Type": "application/json" }, body: message, }) .then(response => { - // Retry until successful. if (!response.ok || response.status !== 200) { throw new Error(response.statusText); } - this.inFlightRequests--; resolve(); }) .catch(reason => { console.warn(`Driver._send failed (${url}):`, reason); - this.inFlightRequests--; resolve(); - this._send(url, message); }); diff --git a/test/test.mjs b/test/test.mjs index d2fcf1644..bd5f05451 100644 --- a/test/test.mjs +++ b/test/test.mjs @@ -290,6 +290,33 @@ async function startRefTest(masterMode, showRefImages) { startTime = Date.now(); startServer(); server.hooks.POST.push(refTestPostHandler); + server.hooks.WS.push(ws => { + let pendingOps = 0; + let pendingQuit = null; + ws.on("message", (data, isBinary) => { + if (isBinary) { + pendingOps++; + handleWsBinaryResult(data).finally(() => { + if (--pendingOps === 0 && pendingQuit) { + pendingQuit(); + pendingQuit = null; + } + }); + } else { + const msg = JSON.parse(data.toString()); + if (msg.type === "quit") { + const session = getSession(msg.browser); + monitorBrowserTimeout(session, null); + const doQuit = () => closeSession(session.name); + if (pendingOps === 0) { + doQuit(); + } else { + pendingQuit = doQuit; + } + } + } + }); + }); onAllSessionsClosed = finalize; await startBrowsers({ @@ -400,68 +427,94 @@ function getSessionManifest(manifest, sessionIndex, sessionCount) { return manifest.slice(start, end); } -function checkEq(task, results, session, masterMode) { - var taskId = task.id; +async function checkEq(task, results, session, masterMode) { + const taskId = task.id; const browserType = session.browserType ?? session.name; - var refSnapshotDir = path.join(refsDir, os.platform(), browserType, taskId); - var testSnapshotDir = path.join( + const refSnapshotDir = path.join(refsDir, os.platform(), browserType, taskId); + const testSnapshotDir = path.join( testResultDir, os.platform(), browserType, taskId ); + const tmpSnapshotDir = masterMode + ? path.join(refsTmpDir, os.platform(), browserType, taskId) + : null; - var pageResults = results[0]; - var taskType = task.type; - var numEqNoSnapshot = 0; - var numEqFailures = 0; - for (var page = 0; page < pageResults.length; page++) { - if (!pageResults[page]) { + const pageResults = results[0]; + const taskType = task.type; + let numEqNoSnapshot = 0; + let numEqFailures = 0; + + // Read all reference PNGs in parallel, skipping pages with no valid snapshot. + const refSnapshots = await Promise.all( + pageResults.map((pageResult, page) => { + if (!pageResult || !(pageResult.snapshot instanceof Buffer)) { + return null; + } + return fs.promises + .readFile(path.join(refSnapshotDir, `${page + 1}.png`)) + .catch(err => { + if (err.code === "ENOENT") { + return null; + } + throw err; + }); + }) + ); + + // Compare all pages (in-memory) and collect all I/O writes to fire together. + const writePromises = []; + const logEntries = []; + let testDirCreated = false; + let tmpDirCreated = false; + + for (let page = 0; page < pageResults.length; page++) { + const pageResult = pageResults[page]; + if (!pageResult) { continue; } - const pageResult = pageResults[page]; - let testSnapshot = pageResult.snapshot; - if (testSnapshot?.startsWith("data:image/png;base64,")) { - testSnapshot = Buffer.from(testSnapshot.substring(22), "base64"); - } else { + const testSnapshot = pageResult.snapshot; + if (!(testSnapshot instanceof Buffer)) { console.error("Valid snapshot was not found."); + continue; } - let unoptimizedSnapshot = pageResult.baselineSnapshot; - if (unoptimizedSnapshot?.startsWith("data:image/png;base64,")) { - unoptimizedSnapshot = Buffer.from( - unoptimizedSnapshot.substring(22), - "base64" - ); - } + const unoptimizedSnapshot = pageResult.baselineSnapshot ?? null; + const refSnapshot = refSnapshots[page]; - var refSnapshot = null; - var eq = false; - var refPath = path.join(refSnapshotDir, `${page + 1}.png`); - if (!fs.existsSync(refPath)) { + let eq = false; + if (!refSnapshot) { numEqNoSnapshot++; if (!masterMode) { - console.log(`WARNING: no reference snapshot ${refPath}`); + console.log( + `WARNING: no reference snapshot ${path.join(refSnapshotDir, `${page + 1}.png`)}` + ); } } else { - refSnapshot = fs.readFileSync(refPath); eq = - stripPrivatePngChunks(refSnapshot).toString("hex") === - stripPrivatePngChunks(testSnapshot).toString("hex"); + Buffer.compare( + stripPrivatePngChunks(refSnapshot), + stripPrivatePngChunks(testSnapshot) + ) === 0; if (!eq) { console.log( `TEST-UNEXPECTED-FAIL | ${taskType} ${taskId} | in ${session.name} | rendering of page ${page + 1} != reference rendering` ); - ensureDirSync(testSnapshotDir); + if (!testDirCreated) { + ensureDirSync(testSnapshotDir); + testDirCreated = true; + } const testPng = path.join(testSnapshotDir, `${page + 1}.png`); const refPng = path.join(testSnapshotDir, `${page + 1}_ref.png`); - fs.writeFileSync(testPng, testSnapshot); - fs.writeFileSync(refPng, refSnapshot); + writePromises.push( + fs.promises.writeFile(testPng, testSnapshot), + fs.promises.writeFile(refPng, refSnapshot) + ); // This no longer follows the format of Mozilla reftest output. const viewportString = `(${pageResult.viewportWidth}x${pageResult.viewportHeight}x${pageResult.outputScale})`; - fs.appendFileSync( - eqLog, + logEntries.push( `REFTEST TEST-UNEXPECTED-FAIL | ${session.name}-${taskId}-page${page + 1} | image comparison (==)\n` + `REFTEST IMAGE 1 (TEST)${viewportString}: ${testPng}\n` + `REFTEST IMAGE 2 (REFERENCE)${viewportString}: ${refPng}\n` @@ -470,20 +523,24 @@ function checkEq(task, results, session, masterMode) { } } if (masterMode && (!refSnapshot || !eq)) { - var tmpSnapshotDir = path.join( - refsTmpDir, - os.platform(), - browserType, - taskId - ); - ensureDirSync(tmpSnapshotDir); - fs.writeFileSync( - path.join(tmpSnapshotDir, `${page + 1}.png`), - unoptimizedSnapshot ?? testSnapshot + if (!tmpDirCreated) { + ensureDirSync(tmpSnapshotDir); + tmpDirCreated = true; + } + writePromises.push( + fs.promises.writeFile( + path.join(tmpSnapshotDir, `${page + 1}.png`), + unoptimizedSnapshot ?? testSnapshot + ) ); } } + if (logEntries.length) { + writePromises.push(fs.promises.appendFile(eqLog, logEntries.join(""))); + } + await Promise.all(writePromises); + session.numEqNoSnapshot += numEqNoSnapshot; if (numEqFailures > 0) { session.numEqFailures += numEqFailures; @@ -506,7 +563,7 @@ function checkFBF(task, results, session, masterMode) { if (!r0Page) { continue; } - if (r0Page.snapshot !== r1Page.snapshot) { + if (Buffer.compare(r0Page.snapshot, r1Page.snapshot) !== 0) { // The FBF tests fail intermittently in Firefox and Google Chrome when run // on the bots, ignoring `makeref` failures for now; see // - https://github.com/mozilla/pdf.js/pull/12368 @@ -543,7 +600,7 @@ function checkLoad(task, results, browser) { console.log(`TEST-PASS | load test ${task.id} | in ${browser}`); } -function checkRefTestResults(browser, id, results) { +async function checkRefTestResults(browser, id, results) { var failed = false; var session = getSession(browser); var task = session.tasks[id]; @@ -605,7 +662,7 @@ function checkRefTestResults(browser, id, results) { case "text": case "highlight": case "extract": - checkEq(task, results, session, session.masterMode); + await checkEq(task, results, session, session.masterMode); break; case "fbf": checkFBF(task, results, session, session.masterMode); @@ -624,16 +681,61 @@ function checkRefTestResults(browser, id, results) { }); } -function refTestPostHandler(parsedUrl, req, res) { - var pathname = parsedUrl.pathname; - if ( - pathname !== "/tellMeToQuit" && - pathname !== "/info" && - pathname !== "/submit_task_results" - ) { - return false; +async function handleWsBinaryResult(data) { + // Binary frame layout: + // [4 bytes BE: meta_len][meta JSON][4 bytes BE: snapshot_len] + // [snapshot PNG][baseline PNG (rest)] + const metaLen = data.readUInt32BE(0); + const meta = JSON.parse(data.subarray(4, 4 + metaLen).toString("utf8")); + const snapshotLen = data.readUInt32BE(4 + metaLen); + const snapshotOffset = 8 + metaLen; + const snapshot = data.subarray(snapshotOffset, snapshotOffset + snapshotLen); + const baseline = + data.length > snapshotOffset + snapshotLen + ? data.subarray(snapshotOffset + snapshotLen) + : null; + + const { browser, id, round, page, failure, lastPageNum, numberOfTasks } = + meta; + const session = getSession(browser); + monitorBrowserTimeout(session, handleSessionTimeout); + + const taskResults = session.taskResults[id]; + if (!taskResults[round]) { + taskResults[round] = []; + } + if (taskResults[round][page - 1]) { + console.error( + `Results for ${browser}:${id}:${round}:${page - 1} were already submitted` + ); + // TODO abort testing here? + } + taskResults[round][page - 1] = { + failure, + snapshot, + baselineSnapshot: baseline, + viewportWidth: meta.viewportWidth, + viewportHeight: meta.viewportHeight, + outputScale: meta.outputScale, + }; + if (stats) { + stats.push({ browser, pdf: id, page: page - 1, round, stats: meta.stats }); } + const lastTaskResults = taskResults.at(-1); + const isDone = + lastTaskResults?.[lastPageNum - 1] || + lastTaskResults?.filter(result => !!result).length === numberOfTasks; + if (isDone) { + await checkRefTestResults(browser, id, taskResults); + session.remaining--; + } +} + +function refTestPostHandler(parsedUrl, req, res) { + if (parsedUrl.pathname !== "/info") { + return false; + } var body = ""; req.on("data", function (data) { body += data; @@ -641,80 +743,7 @@ function refTestPostHandler(parsedUrl, req, res) { req.on("end", function () { res.writeHead(200, { "Content-Type": "text/plain" }); res.end(); - - var session; - if (pathname === "/tellMeToQuit") { - session = getSession(parsedUrl.searchParams.get("browser")); - monitorBrowserTimeout(session, null); - closeSession(session.name); - return; - } - - var data = JSON.parse(body); - if (pathname === "/info") { - console.log(data.message); - return; - } - - var browser = data.browser; - var round = data.round; - var id = data.id; - var page = data.page - 1; - var failure = data.failure; - var snapshot = data.snapshot; - var baselineSnapshot = data.baselineSnapshot; - var lastPageNum = data.lastPageNum; - var numberOfTasks = data.numberOfTasks; - - session = getSession(browser); - monitorBrowserTimeout(session, handleSessionTimeout); - - var taskResults = session.taskResults[id]; - if (!taskResults[round]) { - taskResults[round] = []; - } - - if (taskResults[round][page]) { - console.error( - "Results for " + - browser + - ":" + - id + - ":" + - round + - ":" + - page + - " were already submitted" - ); - // TODO abort testing here? - } - - taskResults[round][page] = { - failure, - snapshot, - baselineSnapshot, - viewportWidth: data.viewportWidth, - viewportHeight: data.viewportHeight, - outputScale: data.outputScale, - }; - if (stats) { - stats.push({ - browser, - pdf: id, - page, - round, - stats: data.stats, - }); - } - - const lastTaskResults = taskResults.at(-1); - const isDone = - lastTaskResults?.[lastPageNum - 1] || - lastTaskResults?.filter(result => !!result).length === numberOfTasks; - if (isDone) { - checkRefTestResults(browser, id, taskResults); - session.remaining--; - } + console.log(JSON.parse(body).message); }); return true; } @@ -911,6 +940,10 @@ async function startBrowser({ // Disable AI/ML functionality. "browser.ai.control.default": "blocked", "privacy.baselineFingerprintingProtection": false, + // Disable bounce tracking protection to avoid creating a SQLite database + // file that Firefox keeps locked briefly after shutdown, causing EBUSY + // errors in Puppeteer's profile cleanup on Windows. + "privacy.bounceTrackingProtection.mode": 0, ...extraPrefsFirefox, }; } diff --git a/test/webserver.mjs b/test/webserver.mjs index e8aec4e34..a79eb467e 100644 --- a/test/webserver.mjs +++ b/test/webserver.mjs @@ -22,6 +22,7 @@ import fsPromises from "fs/promises"; import http from "http"; import path from "path"; import { pathToFileURL } from "url"; +import { WebSocketServer } from "ws"; const MIME_TYPES = { ".css": "text/css", @@ -49,6 +50,7 @@ class WebServer { this.host = host || "localhost"; this.port = port || 0; this.server = null; + this.wss = null; this.verbose = false; this.cacheExpirationTime = cacheExpirationTime || 0; this.disableRangeRequests = false; @@ -56,6 +58,7 @@ class WebServer { this.hooks = { GET: [crossOriginHandler, redirectHandler], POST: [], + WS: [], }; } @@ -63,10 +66,18 @@ class WebServer { this.#ensureNonZeroPort(); this.server = http.createServer(this.#handler.bind(this)); this.server.listen(this.port, this.host, callback); + this.wss = new WebSocketServer({ server: this.server }); + this.wss.on("connection", ws => { + for (const handler of this.hooks.WS) { + handler(ws); + } + }); console.log(`Server running at http://${this.host}:${this.port}/`); } stop(callback) { + this.wss.close(); + this.wss = null; this.server.close(callback); this.server = null; }