Merge pull request #21521 from calixteman/issue21520

Use `Intl.Segmenter` for the entire-word search matching

test/unit/pdf_find_utils_spec.js

@@ -13,44 +13,97 @@
  * limitations under the License.
  */
 
-import { CharacterType, getCharacterType } from "../../web/pdf_find_utils.js";
+import { isEntireWord } from "../../web/pdf_find_utils.js";
 
 describe("pdf_find_utils", function () {
-  describe("getCharacterType", function () {
+  describe("isEntireWord", function () {
-    it("gets expected character types", function () {
+    it("matches whole words and rejects sub-words", function () {
-      const characters = {
+      const content = "the quick brown fox";
-        A: CharacterType.ALPHA_LETTER,
+      // [startIdx, length, expected]
-        a: CharacterType.ALPHA_LETTER,
+      const cases = [
-        0: CharacterType.ALPHA_LETTER,
+        [0, 3, true], // "the" (start of the content)
-        5: CharacterType.ALPHA_LETTER,
+        [4, 5, true], // "quick"
-        "\xC4": CharacterType.ALPHA_LETTER, // "Ä"
+        [10, 5, true], // "brown"
-        "\xE4": CharacterType.ALPHA_LETTER, // "ä"
+        [16, 3, true], // "fox" (end of the content)
-        _: CharacterType.ALPHA_LETTER,
+        [1, 2, false], // "he" inside "the"
-        " ": CharacterType.SPACE,
+        [5, 3, false], // "uic" inside "quick"
-        "\t": CharacterType.SPACE,
+        [10, 4, false], // "brow" inside "brown"
-        "\r": CharacterType.SPACE,
+      ];
-        "\n": CharacterType.SPACE,
+      for (const [startIdx, length, expected] of cases) {
-        "\xA0": CharacterType.SPACE, // nbsp
+        expect(isEntireWord(content, startIdx, length)).toEqual(expected);
-        "-": CharacterType.PUNCT,
-        ",": CharacterType.PUNCT,
-        ".": CharacterType.PUNCT,
-        ";": CharacterType.PUNCT,
-        ":": CharacterType.PUNCT,
-        "\u2122": CharacterType.ALPHA_LETTER, // trademark
-        "\u0E25": CharacterType.THAI_LETTER,
-        "\u4000": CharacterType.HAN_LETTER,
-        "\uF950": CharacterType.HAN_LETTER,
-        "\u30C0": CharacterType.KATAKANA_LETTER,
-        "\u3050": CharacterType.HIRAGANA_LETTER,
-        "\uFF80": CharacterType.HALFWIDTH_KATAKANA_LETTER,
-      };
-
-      for (const character in characters) {
-        const charCode = character.charCodeAt(0);
-        const type = characters[character];
-
-        expect(getCharacterType(charCode)).toEqual(type);
       }
     });
+
+    it("treats ASCII punctuation as a word boundary", function () {
+      const content = "foo-bar (baz)";
+      expect(isEntireWord(content, 0, 3)).toEqual(true); // "foo" before "-"
+      expect(isEntireWord(content, 4, 3)).toEqual(true); // "bar" before " "
+      expect(isEntireWord(content, 9, 3)).toEqual(true); // "baz" inside "()"
+      expect(isEntireWord(content, 0, 7)).toEqual(true); // "foo-bar"
+    });
+
+    it("treats non-ASCII punctuation as a word boundary", function () {
+      // These were previously misclassified as letters, so a word wrapped in
+      // them wasn't recognized as an entire word.
+      for (const [open, close] of [
+        ["«", "»"], // « »
+        ["“", "”"], // “ ”
+        ["—", "—"], // em dashes
+      ]) {
+        const content = `${open}word${close}`;
+        expect(isEntireWord(content, 1, 4)).toEqual(true); // "word"
+        expect(isEntireWord(content, 1, 3)).toEqual(false); // "wor"
+      }
+    });
+
+    it("treats a trailing superscript as a word boundary", function () {
+      // A footnote/reference marker or an exponent must not prevent the
+      // preceding word from matching as an entire word.
+      for (const superscript of ["¹", "²", "³", "⁴"]) {
+        const content = `word${superscript}`;
+        expect(isEntireWord(content, 0, 4)).toEqual(true); // "word"
+      }
+    });
+
+    it("matches a word next to a contraction apostrophe", function () {
+      // The apostrophe is a word break in isolation (as in Firefox's find), so
+      // the leading part of a contraction is still an entire word.
+      expect(isEntireWord("I can't do that", 2, 3)).toEqual(true); // "can"
+      expect(isEntireWord("don't", 0, 3)).toEqual(true); // "don"
+      expect(isEntireWord("it's", 0, 2)).toEqual(true); // "it"
+    });
+
+    it("keeps a word joined by a connecting character", function () {
+      expect(isEntireWord("foo_bar", 0, 3)).toEqual(false); // "foo" in "foo_bar"
+      expect(isEntireWord("foo_bar", 4, 3)).toEqual(false); // "bar" in "foo_bar"
+    });
+
+    it("handles combining marks (NFD) within a word", function () {
+      // "café" in the normalized page content is NFD: "cafe" followed by
+      // U+0301 COMBINING ACUTE ACCENT.
+      const content = "café".normalize("NFD");
+      expect(isEntireWord(content, 0, content.length)).toEqual(true); // whole
+      expect(isEntireWord(content, 0, 3)).toEqual(false); // "caf"
+      expect(isEntireWord(content, 0, 4)).toEqual(false); // "cafe", before U+0301
+    });
+
+    it("keeps a base character and its combining mark together", function () {
+      // "áb" in NFD is "a" + U+0301 + "b"; it's a single word.
+      const ab = "áb".normalize("NFD");
+      expect(isEntireWord(ab, 2, 1)).toEqual(false); // "b" inside "áb"
+      expect(isEntireWord(ab, 0, 2)).toEqual(false); // "á" inside "áb"
+      // "café" is a whole word before a space, but not inside "caféteria".
+      expect(isEntireWord("café bar".normalize("NFD"), 0, 5)).toEqual(true);
+      expect(isEntireWord("caféteria".normalize("NFD"), 0, 5)).toEqual(false);
+    });
+
+    it("keeps non-BMP characters intact", function () {
+      // "a𝐀b" is a single word (𝐀 = U+1D400, a surrogate pair).
+      const withBoldA = "a\u{1D400}b";
+      expect(isEntireWord(withBoldA, 0, 1)).toEqual(false); // "a"
+      expect(isEntireWord(withBoldA, 3, 1)).toEqual(false); // "b"
+      // Each CJK ideograph is its own word (𠀀 = U+20000, a surrogate pair).
+      expect(isEntireWord("中\u{20000}国", 0, 1)).toEqual(true); // "中"
+    });
   });
 });

web/pdf_find_controller.js

@@ -17,7 +17,7 @@
 /** @typedef {import("./event_utils").EventBus} EventBus */
 /** @typedef {import("./pdf_link_service.js").PDFLinkService} PDFLinkService */
 
-import { getCharacterType, getNormalizeWithNFKC } from "./pdf_find_utils.js";
+import { getNormalizeWithNFKC, isEntireWord } from "./pdf_find_utils.js";
 import { binarySearchFirstItem } from "./ui_utils.js";
 import { internalOpt } from "./internal_evt.js";
 
@@ -76,8 +76,6 @@ let DIACRITICS_EXCEPTION_STR; // Lazily initialized, see below.
 
 const DIACRITICS_REG_EXP = /\p{M}+/gu;
 const SPECIAL_CHARS_REG_EXP = /([+^$|])|(\p{P}+)|(\s+)|(\p{M})|(\p{L})/gu;
-const NOT_DIACRITIC_FROM_END_REG_EXP = /(\P{M})\p{M}*$/u;
-const NOT_DIACRITIC_FROM_START_REG_EXP = /^\p{M}*(\P{M})/u;
 
 // The range [AC00-D7AF] corresponds to the Hangul syllables.
 // The few other chars are some CJK Compatibility Ideographs.
@@ -680,36 +678,6 @@ class PDFFindController {
     return true;
   }
 
-  /**
-   * Determine if the search query constitutes a "whole word", by comparing the
-   * first/last character type with the preceding/following character type.
-   */
-  #isEntireWord(content, startIdx, length) {
-    let match = content
-      .slice(0, startIdx)
-      .match(NOT_DIACRITIC_FROM_END_REG_EXP);
-    if (match) {
-      const first = content.charCodeAt(startIdx);
-      const limit = match[1].charCodeAt(0);
-      if (getCharacterType(first) === getCharacterType(limit)) {
-        return false;
-      }
-    }
-
-    match = content
-      .slice(startIdx + length)
-      .match(NOT_DIACRITIC_FROM_START_REG_EXP);
-    if (match) {
-      const last = content.charCodeAt(startIdx + length - 1);
-      const limit = match[1].charCodeAt(0);
-      if (getCharacterType(last) === getCharacterType(limit)) {
-        return false;
-      }
-    }
-
-    return true;
-  }
-
   #convertToRegExpString(query, hasDiacritics) {
     const { matchDiacritics } = this.#state;
     let isUnicode = false;
@@ -890,7 +858,7 @@ class PDFFindController {
     while ((match = query.exec(pageContent)) !== null) {
       if (
         entireWord &&
-        !this.#isEntireWord(pageContent, match.index, match[0].length)
+        !isEntireWord(pageContent, match.index, match[0].length)
       ) {
         continue;
       }

web/pdf_find_utils.js

@@ -15,103 +15,46 @@
 
 import { FeatureTest } from "pdfjs-lib";
 
-const CharacterType = {
+let wordSegmenter = null;
-  SPACE: 0,
+let graphemeSegmenter = null;
-  ALPHA_LETTER: 1,
-  PUNCT: 2,
-  HAN_LETTER: 3,
-  KATAKANA_LETTER: 4,
-  HIRAGANA_LETTER: 5,
-  HALFWIDTH_KATAKANA_LETTER: 6,
-  THAI_LETTER: 7,
-};
 
-function isAlphabeticalScript(charCode) {
+function isWordBreakAt(content, pos) {
-  return charCode < 0x2e80;
+  // `pos` must be a grapheme boundary (so a combining mark stays attached to
-}
+  // its base, and a surrogate pair isn't split) as well as a word boundary,
+  // the latter tested on the two adjacent grapheme clusters in isolation like
+  // Firefox's find (see `nsFind::BreakInBetween`).
+  graphemeSegmenter ||= new Intl.Segmenter(undefined, {
+    granularity: "grapheme",
+  });
 
-function isAscii(charCode) {
+  const graphemes = graphemeSegmenter.segment(content);
-  return (charCode & 0xff80) === 0;
+  const after = graphemes.containing(pos);
-}
+  if (after.index !== pos) {
+    return false;
+  }
 
-function isAsciiAlpha(charCode) {
+  wordSegmenter ||= new Intl.Segmenter(undefined, { granularity: "word" });
+  const before = graphemes.containing(pos - 1).segment;
   return (
-    (charCode >= /* a = */ 0x61 && charCode <= /* z = */ 0x7a) ||
+    wordSegmenter.segment(before + after.segment).containing(before.length)
-    (charCode >= /* A = */ 0x41 && charCode <= /* Z = */ 0x5a)
+      .index === before.length
   );
 }
 
-function isAsciiDigit(charCode) {
-  return charCode >= /* 0 = */ 0x30 && charCode <= /* 9 = */ 0x39;
-}
-
-function isAsciiSpace(charCode) {
-  return (
-    charCode === /* SPACE = */ 0x20 ||
-    charCode === /* TAB = */ 0x09 ||
-    charCode === /* CR = */ 0x0d ||
-    charCode === /* LF = */ 0x0a
-  );
-}
-
-function isHan(charCode) {
-  return (
-    (charCode >= 0x3400 && charCode <= 0x9fff) ||
-    (charCode >= 0xf900 && charCode <= 0xfaff)
-  );
-}
-
-function isKatakana(charCode) {
-  return charCode >= 0x30a0 && charCode <= 0x30ff;
-}
-
-function isHiragana(charCode) {
-  return charCode >= 0x3040 && charCode <= 0x309f;
-}
-
-function isHalfwidthKatakana(charCode) {
-  return charCode >= 0xff60 && charCode <= 0xff9f;
-}
-
-function isThai(charCode) {
-  return (charCode & 0xff80) === 0x0e00;
-}
-
 /**
- * This function is based on the word-break detection implemented in:
+ * Determine if the match spanning `[startIdx, startIdx + length)` in `content`
- * https://hg.mozilla.org/mozilla-central/file/tip/intl/lwbrk/WordBreaker.cpp
+ * is a whole word, i.e. there's a word break on each side of it. Each boundary
+ * is tested on its two adjacent characters in isolation rather than on the
+ * whole string, so a contraction such as "can't" doesn't prevent "can" from
+ * being an entire word, matching Firefox's find:
+ * https://searchfox.org/firefox-main/source/toolkit/components/find/nsFind.cpp
  */
-function getCharacterType(charCode) {
+function isEntireWord(content, startIdx, length) {
-  if (isAlphabeticalScript(charCode)) {
+  const endIdx = startIdx + length;
-    if (isAscii(charCode)) {
+  return (
-      if (isAsciiSpace(charCode)) {
+    (startIdx === 0 || isWordBreakAt(content, startIdx)) &&
-        return CharacterType.SPACE;
+    (endIdx === content.length || isWordBreakAt(content, endIdx))
-      } else if (
+  );
-        isAsciiAlpha(charCode) ||
-        isAsciiDigit(charCode) ||
-        charCode === /* UNDERSCORE = */ 0x5f
-      ) {
-        return CharacterType.ALPHA_LETTER;
-      }
-      return CharacterType.PUNCT;
-    } else if (isThai(charCode)) {
-      return CharacterType.THAI_LETTER;
-    } else if (charCode === /* NBSP = */ 0xa0) {
-      return CharacterType.SPACE;
-    }
-    return CharacterType.ALPHA_LETTER;
-  }
-
-  if (isHan(charCode)) {
-    return CharacterType.HAN_LETTER;
-  } else if (isKatakana(charCode)) {
-    return CharacterType.KATAKANA_LETTER;
-  } else if (isHiragana(charCode)) {
-    return CharacterType.HIRAGANA_LETTER;
-  } else if (isHalfwidthKatakana(charCode)) {
-    return CharacterType.HALFWIDTH_KATAKANA_LETTER;
-  }
-  return CharacterType.ALPHA_LETTER;
 }
 
 let NormalizeWithNFKC;
@@ -191,4 +134,4 @@ function getNormalizeWithNFKC() {
   return NormalizeWithNFKC;
 }
 
-export { CharacterType, getCharacterType, getNormalizeWithNFKC };
+export { getNormalizeWithNFKC, isEntireWord };