diff --git a/test/unit/pdf_find_utils_spec.js b/test/unit/pdf_find_utils_spec.js index d108a5cc2..8eba795be 100644 --- a/test/unit/pdf_find_utils_spec.js +++ b/test/unit/pdf_find_utils_spec.js @@ -13,44 +13,97 @@ * limitations under the License. */ -import { CharacterType, getCharacterType } from "../../web/pdf_find_utils.js"; +import { isEntireWord } from "../../web/pdf_find_utils.js"; describe("pdf_find_utils", function () { - describe("getCharacterType", function () { - it("gets expected character types", function () { - const characters = { - A: CharacterType.ALPHA_LETTER, - a: CharacterType.ALPHA_LETTER, - 0: CharacterType.ALPHA_LETTER, - 5: CharacterType.ALPHA_LETTER, - "\xC4": CharacterType.ALPHA_LETTER, // "Ä" - "\xE4": CharacterType.ALPHA_LETTER, // "ä" - _: CharacterType.ALPHA_LETTER, - " ": CharacterType.SPACE, - "\t": CharacterType.SPACE, - "\r": CharacterType.SPACE, - "\n": CharacterType.SPACE, - "\xA0": CharacterType.SPACE, // nbsp - "-": CharacterType.PUNCT, - ",": CharacterType.PUNCT, - ".": CharacterType.PUNCT, - ";": CharacterType.PUNCT, - ":": CharacterType.PUNCT, - "\u2122": CharacterType.ALPHA_LETTER, // trademark - "\u0E25": CharacterType.THAI_LETTER, - "\u4000": CharacterType.HAN_LETTER, - "\uF950": CharacterType.HAN_LETTER, - "\u30C0": CharacterType.KATAKANA_LETTER, - "\u3050": CharacterType.HIRAGANA_LETTER, - "\uFF80": CharacterType.HALFWIDTH_KATAKANA_LETTER, - }; - - for (const character in characters) { - const charCode = character.charCodeAt(0); - const type = characters[character]; - - expect(getCharacterType(charCode)).toEqual(type); + describe("isEntireWord", function () { + it("matches whole words and rejects sub-words", function () { + const content = "the quick brown fox"; + // [startIdx, length, expected] + const cases = [ + [0, 3, true], // "the" (start of the content) + [4, 5, true], // "quick" + [10, 5, true], // "brown" + [16, 3, true], // "fox" (end of the content) + [1, 2, false], // "he" inside "the" + [5, 3, false], // "uic" inside "quick" + [10, 4, false], // "brow" inside "brown" + ]; + for (const [startIdx, length, expected] of cases) { + expect(isEntireWord(content, startIdx, length)).toEqual(expected); } }); + + it("treats ASCII punctuation as a word boundary", function () { + const content = "foo-bar (baz)"; + expect(isEntireWord(content, 0, 3)).toEqual(true); // "foo" before "-" + expect(isEntireWord(content, 4, 3)).toEqual(true); // "bar" before " " + expect(isEntireWord(content, 9, 3)).toEqual(true); // "baz" inside "()" + expect(isEntireWord(content, 0, 7)).toEqual(true); // "foo-bar" + }); + + it("treats non-ASCII punctuation as a word boundary", function () { + // These were previously misclassified as letters, so a word wrapped in + // them wasn't recognized as an entire word. + for (const [open, close] of [ + ["«", "»"], // « » + ["“", "”"], // “ ” + ["—", "—"], // em dashes + ]) { + const content = `${open}word${close}`; + expect(isEntireWord(content, 1, 4)).toEqual(true); // "word" + expect(isEntireWord(content, 1, 3)).toEqual(false); // "wor" + } + }); + + it("treats a trailing superscript as a word boundary", function () { + // A footnote/reference marker or an exponent must not prevent the + // preceding word from matching as an entire word. + for (const superscript of ["¹", "²", "³", "⁴"]) { + const content = `word${superscript}`; + expect(isEntireWord(content, 0, 4)).toEqual(true); // "word" + } + }); + + it("matches a word next to a contraction apostrophe", function () { + // The apostrophe is a word break in isolation (as in Firefox's find), so + // the leading part of a contraction is still an entire word. + expect(isEntireWord("I can't do that", 2, 3)).toEqual(true); // "can" + expect(isEntireWord("don't", 0, 3)).toEqual(true); // "don" + expect(isEntireWord("it's", 0, 2)).toEqual(true); // "it" + }); + + it("keeps a word joined by a connecting character", function () { + expect(isEntireWord("foo_bar", 0, 3)).toEqual(false); // "foo" in "foo_bar" + expect(isEntireWord("foo_bar", 4, 3)).toEqual(false); // "bar" in "foo_bar" + }); + + it("handles combining marks (NFD) within a word", function () { + // "café" in the normalized page content is NFD: "cafe" followed by + // U+0301 COMBINING ACUTE ACCENT. + const content = "café".normalize("NFD"); + expect(isEntireWord(content, 0, content.length)).toEqual(true); // whole + expect(isEntireWord(content, 0, 3)).toEqual(false); // "caf" + expect(isEntireWord(content, 0, 4)).toEqual(false); // "cafe", before U+0301 + }); + + it("keeps a base character and its combining mark together", function () { + // "áb" in NFD is "a" + U+0301 + "b"; it's a single word. + const ab = "áb".normalize("NFD"); + expect(isEntireWord(ab, 2, 1)).toEqual(false); // "b" inside "áb" + expect(isEntireWord(ab, 0, 2)).toEqual(false); // "á" inside "áb" + // "café" is a whole word before a space, but not inside "caféteria". + expect(isEntireWord("café bar".normalize("NFD"), 0, 5)).toEqual(true); + expect(isEntireWord("caféteria".normalize("NFD"), 0, 5)).toEqual(false); + }); + + it("keeps non-BMP characters intact", function () { + // "a𝐀b" is a single word (𝐀 = U+1D400, a surrogate pair). + const withBoldA = "a\u{1D400}b"; + expect(isEntireWord(withBoldA, 0, 1)).toEqual(false); // "a" + expect(isEntireWord(withBoldA, 3, 1)).toEqual(false); // "b" + // Each CJK ideograph is its own word (𠀀 = U+20000, a surrogate pair). + expect(isEntireWord("中\u{20000}国", 0, 1)).toEqual(true); // "中" + }); }); }); diff --git a/web/pdf_find_controller.js b/web/pdf_find_controller.js index 5670cbdc6..c0605b1b5 100644 --- a/web/pdf_find_controller.js +++ b/web/pdf_find_controller.js @@ -17,7 +17,7 @@ /** @typedef {import("./event_utils").EventBus} EventBus */ /** @typedef {import("./pdf_link_service.js").PDFLinkService} PDFLinkService */ -import { getCharacterType, getNormalizeWithNFKC } from "./pdf_find_utils.js"; +import { getNormalizeWithNFKC, isEntireWord } from "./pdf_find_utils.js"; import { binarySearchFirstItem } from "./ui_utils.js"; import { internalOpt } from "./internal_evt.js"; @@ -76,8 +76,6 @@ let DIACRITICS_EXCEPTION_STR; // Lazily initialized, see below. const DIACRITICS_REG_EXP = /\p{M}+/gu; const SPECIAL_CHARS_REG_EXP = /([+^$|])|(\p{P}+)|(\s+)|(\p{M})|(\p{L})/gu; -const NOT_DIACRITIC_FROM_END_REG_EXP = /(\P{M})\p{M}*$/u; -const NOT_DIACRITIC_FROM_START_REG_EXP = /^\p{M}*(\P{M})/u; // The range [AC00-D7AF] corresponds to the Hangul syllables. // The few other chars are some CJK Compatibility Ideographs. @@ -680,36 +678,6 @@ class PDFFindController { return true; } - /** - * Determine if the search query constitutes a "whole word", by comparing the - * first/last character type with the preceding/following character type. - */ - #isEntireWord(content, startIdx, length) { - let match = content - .slice(0, startIdx) - .match(NOT_DIACRITIC_FROM_END_REG_EXP); - if (match) { - const first = content.charCodeAt(startIdx); - const limit = match[1].charCodeAt(0); - if (getCharacterType(first) === getCharacterType(limit)) { - return false; - } - } - - match = content - .slice(startIdx + length) - .match(NOT_DIACRITIC_FROM_START_REG_EXP); - if (match) { - const last = content.charCodeAt(startIdx + length - 1); - const limit = match[1].charCodeAt(0); - if (getCharacterType(last) === getCharacterType(limit)) { - return false; - } - } - - return true; - } - #convertToRegExpString(query, hasDiacritics) { const { matchDiacritics } = this.#state; let isUnicode = false; @@ -890,7 +858,7 @@ class PDFFindController { while ((match = query.exec(pageContent)) !== null) { if ( entireWord && - !this.#isEntireWord(pageContent, match.index, match[0].length) + !isEntireWord(pageContent, match.index, match[0].length) ) { continue; } diff --git a/web/pdf_find_utils.js b/web/pdf_find_utils.js index ef8160e2c..8a604552c 100644 --- a/web/pdf_find_utils.js +++ b/web/pdf_find_utils.js @@ -15,103 +15,46 @@ import { FeatureTest } from "pdfjs-lib"; -const CharacterType = { - SPACE: 0, - ALPHA_LETTER: 1, - PUNCT: 2, - HAN_LETTER: 3, - KATAKANA_LETTER: 4, - HIRAGANA_LETTER: 5, - HALFWIDTH_KATAKANA_LETTER: 6, - THAI_LETTER: 7, -}; +let wordSegmenter = null; +let graphemeSegmenter = null; -function isAlphabeticalScript(charCode) { - return charCode < 0x2e80; -} +function isWordBreakAt(content, pos) { + // `pos` must be a grapheme boundary (so a combining mark stays attached to + // its base, and a surrogate pair isn't split) as well as a word boundary, + // the latter tested on the two adjacent grapheme clusters in isolation like + // Firefox's find (see `nsFind::BreakInBetween`). + graphemeSegmenter ||= new Intl.Segmenter(undefined, { + granularity: "grapheme", + }); -function isAscii(charCode) { - return (charCode & 0xff80) === 0; -} + const graphemes = graphemeSegmenter.segment(content); + const after = graphemes.containing(pos); + if (after.index !== pos) { + return false; + } -function isAsciiAlpha(charCode) { + wordSegmenter ||= new Intl.Segmenter(undefined, { granularity: "word" }); + const before = graphemes.containing(pos - 1).segment; return ( - (charCode >= /* a = */ 0x61 && charCode <= /* z = */ 0x7a) || - (charCode >= /* A = */ 0x41 && charCode <= /* Z = */ 0x5a) + wordSegmenter.segment(before + after.segment).containing(before.length) + .index === before.length ); } -function isAsciiDigit(charCode) { - return charCode >= /* 0 = */ 0x30 && charCode <= /* 9 = */ 0x39; -} - -function isAsciiSpace(charCode) { - return ( - charCode === /* SPACE = */ 0x20 || - charCode === /* TAB = */ 0x09 || - charCode === /* CR = */ 0x0d || - charCode === /* LF = */ 0x0a - ); -} - -function isHan(charCode) { - return ( - (charCode >= 0x3400 && charCode <= 0x9fff) || - (charCode >= 0xf900 && charCode <= 0xfaff) - ); -} - -function isKatakana(charCode) { - return charCode >= 0x30a0 && charCode <= 0x30ff; -} - -function isHiragana(charCode) { - return charCode >= 0x3040 && charCode <= 0x309f; -} - -function isHalfwidthKatakana(charCode) { - return charCode >= 0xff60 && charCode <= 0xff9f; -} - -function isThai(charCode) { - return (charCode & 0xff80) === 0x0e00; -} - /** - * This function is based on the word-break detection implemented in: - * https://hg.mozilla.org/mozilla-central/file/tip/intl/lwbrk/WordBreaker.cpp + * Determine if the match spanning `[startIdx, startIdx + length)` in `content` + * is a whole word, i.e. there's a word break on each side of it. Each boundary + * is tested on its two adjacent characters in isolation rather than on the + * whole string, so a contraction such as "can't" doesn't prevent "can" from + * being an entire word, matching Firefox's find: + * https://searchfox.org/firefox-main/source/toolkit/components/find/nsFind.cpp */ -function getCharacterType(charCode) { - if (isAlphabeticalScript(charCode)) { - if (isAscii(charCode)) { - if (isAsciiSpace(charCode)) { - return CharacterType.SPACE; - } else if ( - isAsciiAlpha(charCode) || - isAsciiDigit(charCode) || - charCode === /* UNDERSCORE = */ 0x5f - ) { - return CharacterType.ALPHA_LETTER; - } - return CharacterType.PUNCT; - } else if (isThai(charCode)) { - return CharacterType.THAI_LETTER; - } else if (charCode === /* NBSP = */ 0xa0) { - return CharacterType.SPACE; - } - return CharacterType.ALPHA_LETTER; - } - - if (isHan(charCode)) { - return CharacterType.HAN_LETTER; - } else if (isKatakana(charCode)) { - return CharacterType.KATAKANA_LETTER; - } else if (isHiragana(charCode)) { - return CharacterType.HIRAGANA_LETTER; - } else if (isHalfwidthKatakana(charCode)) { - return CharacterType.HALFWIDTH_KATAKANA_LETTER; - } - return CharacterType.ALPHA_LETTER; +function isEntireWord(content, startIdx, length) { + const endIdx = startIdx + length; + return ( + (startIdx === 0 || isWordBreakAt(content, startIdx)) && + (endIdx === content.length || isWordBreakAt(content, endIdx)) + ); } let NormalizeWithNFKC; @@ -191,4 +134,4 @@ function getNormalizeWithNFKC() { return NormalizeWithNFKC; } -export { CharacterType, getCharacterType, getNormalizeWithNFKC }; +export { getNormalizeWithNFKC, isEntireWord };