mirror of
https://github.com/mozilla/pdf.js.git
synced 2026-07-05 22:55:49 +02:00
Merge pull request #21521 from calixteman/issue21520
Use `Intl.Segmenter` for the entire-word search matching
This commit is contained in:
calixteman
GitHub
commit
bf924ff31b
@ -13,44 +13,97 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { CharacterType, getCharacterType } from "../../web/pdf_find_utils.js";
|
||||
import { isEntireWord } from "../../web/pdf_find_utils.js";
|
||||
|
||||
describe("pdf_find_utils", function () {
|
||||
describe("getCharacterType", function () {
|
||||
it("gets expected character types", function () {
|
||||
const characters = {
|
||||
A: CharacterType.ALPHA_LETTER,
|
||||
a: CharacterType.ALPHA_LETTER,
|
||||
0: CharacterType.ALPHA_LETTER,
|
||||
5: CharacterType.ALPHA_LETTER,
|
||||
"\xC4": CharacterType.ALPHA_LETTER, // "Ä"
|
||||
"\xE4": CharacterType.ALPHA_LETTER, // "ä"
|
||||
_: CharacterType.ALPHA_LETTER,
|
||||
" ": CharacterType.SPACE,
|
||||
"\t": CharacterType.SPACE,
|
||||
"\r": CharacterType.SPACE,
|
||||
"\n": CharacterType.SPACE,
|
||||
"\xA0": CharacterType.SPACE, // nbsp
|
||||
"-": CharacterType.PUNCT,
|
||||
",": CharacterType.PUNCT,
|
||||
".": CharacterType.PUNCT,
|
||||
";": CharacterType.PUNCT,
|
||||
":": CharacterType.PUNCT,
|
||||
"\u2122": CharacterType.ALPHA_LETTER, // trademark
|
||||
"\u0E25": CharacterType.THAI_LETTER,
|
||||
"\u4000": CharacterType.HAN_LETTER,
|
||||
"\uF950": CharacterType.HAN_LETTER,
|
||||
"\u30C0": CharacterType.KATAKANA_LETTER,
|
||||
"\u3050": CharacterType.HIRAGANA_LETTER,
|
||||
"\uFF80": CharacterType.HALFWIDTH_KATAKANA_LETTER,
|
||||
};
|
||||
|
||||
for (const character in characters) {
|
||||
const charCode = character.charCodeAt(0);
|
||||
const type = characters[character];
|
||||
|
||||
expect(getCharacterType(charCode)).toEqual(type);
|
||||
describe("isEntireWord", function () {
|
||||
it("matches whole words and rejects sub-words", function () {
|
||||
const content = "the quick brown fox";
|
||||
// [startIdx, length, expected]
|
||||
const cases = [
|
||||
[0, 3, true], // "the" (start of the content)
|
||||
[4, 5, true], // "quick"
|
||||
[10, 5, true], // "brown"
|
||||
[16, 3, true], // "fox" (end of the content)
|
||||
[1, 2, false], // "he" inside "the"
|
||||
[5, 3, false], // "uic" inside "quick"
|
||||
[10, 4, false], // "brow" inside "brown"
|
||||
];
|
||||
for (const [startIdx, length, expected] of cases) {
|
||||
expect(isEntireWord(content, startIdx, length)).toEqual(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("treats ASCII punctuation as a word boundary", function () {
|
||||
const content = "foo-bar (baz)";
|
||||
expect(isEntireWord(content, 0, 3)).toEqual(true); // "foo" before "-"
|
||||
expect(isEntireWord(content, 4, 3)).toEqual(true); // "bar" before " "
|
||||
expect(isEntireWord(content, 9, 3)).toEqual(true); // "baz" inside "()"
|
||||
expect(isEntireWord(content, 0, 7)).toEqual(true); // "foo-bar"
|
||||
});
|
||||
|
||||
it("treats non-ASCII punctuation as a word boundary", function () {
|
||||
// These were previously misclassified as letters, so a word wrapped in
|
||||
// them wasn't recognized as an entire word.
|
||||
for (const [open, close] of [
|
||||
["«", "»"], // « »
|
||||
["“", "”"], // “ ”
|
||||
["—", "—"], // em dashes
|
||||
]) {
|
||||
const content = `${open}word${close}`;
|
||||
expect(isEntireWord(content, 1, 4)).toEqual(true); // "word"
|
||||
expect(isEntireWord(content, 1, 3)).toEqual(false); // "wor"
|
||||
}
|
||||
});
|
||||
|
||||
it("treats a trailing superscript as a word boundary", function () {
|
||||
// A footnote/reference marker or an exponent must not prevent the
|
||||
// preceding word from matching as an entire word.
|
||||
for (const superscript of ["¹", "²", "³", "⁴"]) {
|
||||
const content = `word${superscript}`;
|
||||
expect(isEntireWord(content, 0, 4)).toEqual(true); // "word"
|
||||
}
|
||||
});
|
||||
|
||||
it("matches a word next to a contraction apostrophe", function () {
|
||||
// The apostrophe is a word break in isolation (as in Firefox's find), so
|
||||
// the leading part of a contraction is still an entire word.
|
||||
expect(isEntireWord("I can't do that", 2, 3)).toEqual(true); // "can"
|
||||
expect(isEntireWord("don't", 0, 3)).toEqual(true); // "don"
|
||||
expect(isEntireWord("it's", 0, 2)).toEqual(true); // "it"
|
||||
});
|
||||
|
||||
it("keeps a word joined by a connecting character", function () {
|
||||
expect(isEntireWord("foo_bar", 0, 3)).toEqual(false); // "foo" in "foo_bar"
|
||||
expect(isEntireWord("foo_bar", 4, 3)).toEqual(false); // "bar" in "foo_bar"
|
||||
});
|
||||
|
||||
it("handles combining marks (NFD) within a word", function () {
|
||||
// "café" in the normalized page content is NFD: "cafe" followed by
|
||||
// U+0301 COMBINING ACUTE ACCENT.
|
||||
const content = "café".normalize("NFD");
|
||||
expect(isEntireWord(content, 0, content.length)).toEqual(true); // whole
|
||||
expect(isEntireWord(content, 0, 3)).toEqual(false); // "caf"
|
||||
expect(isEntireWord(content, 0, 4)).toEqual(false); // "cafe", before U+0301
|
||||
});
|
||||
|
||||
it("keeps a base character and its combining mark together", function () {
|
||||
// "áb" in NFD is "a" + U+0301 + "b"; it's a single word.
|
||||
const ab = "áb".normalize("NFD");
|
||||
expect(isEntireWord(ab, 2, 1)).toEqual(false); // "b" inside "áb"
|
||||
expect(isEntireWord(ab, 0, 2)).toEqual(false); // "á" inside "áb"
|
||||
// "café" is a whole word before a space, but not inside "caféteria".
|
||||
expect(isEntireWord("café bar".normalize("NFD"), 0, 5)).toEqual(true);
|
||||
expect(isEntireWord("caféteria".normalize("NFD"), 0, 5)).toEqual(false);
|
||||
});
|
||||
|
||||
it("keeps non-BMP characters intact", function () {
|
||||
// "a𝐀b" is a single word (𝐀 = U+1D400, a surrogate pair).
|
||||
const withBoldA = "a\u{1D400}b";
|
||||
expect(isEntireWord(withBoldA, 0, 1)).toEqual(false); // "a"
|
||||
expect(isEntireWord(withBoldA, 3, 1)).toEqual(false); // "b"
|
||||
// Each CJK ideograph is its own word (𠀀 = U+20000, a surrogate pair).
|
||||
expect(isEntireWord("中\u{20000}国", 0, 1)).toEqual(true); // "中"
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
/** @typedef {import("./event_utils").EventBus} EventBus */
|
||||
/** @typedef {import("./pdf_link_service.js").PDFLinkService} PDFLinkService */
|
||||
|
||||
import { getCharacterType, getNormalizeWithNFKC } from "./pdf_find_utils.js";
|
||||
import { getNormalizeWithNFKC, isEntireWord } from "./pdf_find_utils.js";
|
||||
import { binarySearchFirstItem } from "./ui_utils.js";
|
||||
import { internalOpt } from "./internal_evt.js";
|
||||
|
||||
@ -76,8 +76,6 @@ let DIACRITICS_EXCEPTION_STR; // Lazily initialized, see below.
|
||||
|
||||
const DIACRITICS_REG_EXP = /\p{M}+/gu;
|
||||
const SPECIAL_CHARS_REG_EXP = /([+^$|])|(\p{P}+)|(\s+)|(\p{M})|(\p{L})/gu;
|
||||
const NOT_DIACRITIC_FROM_END_REG_EXP = /(\P{M})\p{M}*$/u;
|
||||
const NOT_DIACRITIC_FROM_START_REG_EXP = /^\p{M}*(\P{M})/u;
|
||||
|
||||
// The range [AC00-D7AF] corresponds to the Hangul syllables.
|
||||
// The few other chars are some CJK Compatibility Ideographs.
|
||||
@ -680,36 +678,6 @@ class PDFFindController {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the search query constitutes a "whole word", by comparing the
|
||||
* first/last character type with the preceding/following character type.
|
||||
*/
|
||||
#isEntireWord(content, startIdx, length) {
|
||||
let match = content
|
||||
.slice(0, startIdx)
|
||||
.match(NOT_DIACRITIC_FROM_END_REG_EXP);
|
||||
if (match) {
|
||||
const first = content.charCodeAt(startIdx);
|
||||
const limit = match[1].charCodeAt(0);
|
||||
if (getCharacterType(first) === getCharacterType(limit)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
match = content
|
||||
.slice(startIdx + length)
|
||||
.match(NOT_DIACRITIC_FROM_START_REG_EXP);
|
||||
if (match) {
|
||||
const last = content.charCodeAt(startIdx + length - 1);
|
||||
const limit = match[1].charCodeAt(0);
|
||||
if (getCharacterType(last) === getCharacterType(limit)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#convertToRegExpString(query, hasDiacritics) {
|
||||
const { matchDiacritics } = this.#state;
|
||||
let isUnicode = false;
|
||||
@ -890,7 +858,7 @@ class PDFFindController {
|
||||
while ((match = query.exec(pageContent)) !== null) {
|
||||
if (
|
||||
entireWord &&
|
||||
!this.#isEntireWord(pageContent, match.index, match[0].length)
|
||||
!isEntireWord(pageContent, match.index, match[0].length)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -15,103 +15,46 @@
|
||||
|
||||
import { FeatureTest } from "pdfjs-lib";
|
||||
|
||||
const CharacterType = {
|
||||
SPACE: 0,
|
||||
ALPHA_LETTER: 1,
|
||||
PUNCT: 2,
|
||||
HAN_LETTER: 3,
|
||||
KATAKANA_LETTER: 4,
|
||||
HIRAGANA_LETTER: 5,
|
||||
HALFWIDTH_KATAKANA_LETTER: 6,
|
||||
THAI_LETTER: 7,
|
||||
};
|
||||
let wordSegmenter = null;
|
||||
let graphemeSegmenter = null;
|
||||
|
||||
function isAlphabeticalScript(charCode) {
|
||||
return charCode < 0x2e80;
|
||||
}
|
||||
function isWordBreakAt(content, pos) {
|
||||
// `pos` must be a grapheme boundary (so a combining mark stays attached to
|
||||
// its base, and a surrogate pair isn't split) as well as a word boundary,
|
||||
// the latter tested on the two adjacent grapheme clusters in isolation like
|
||||
// Firefox's find (see `nsFind::BreakInBetween`).
|
||||
graphemeSegmenter ||= new Intl.Segmenter(undefined, {
|
||||
granularity: "grapheme",
|
||||
});
|
||||
|
||||
function isAscii(charCode) {
|
||||
return (charCode & 0xff80) === 0;
|
||||
}
|
||||
const graphemes = graphemeSegmenter.segment(content);
|
||||
const after = graphemes.containing(pos);
|
||||
if (after.index !== pos) {
|
||||
return false;
|
||||
}
|
||||
|
||||
function isAsciiAlpha(charCode) {
|
||||
wordSegmenter ||= new Intl.Segmenter(undefined, { granularity: "word" });
|
||||
const before = graphemes.containing(pos - 1).segment;
|
||||
return (
|
||||
(charCode >= /* a = */ 0x61 && charCode <= /* z = */ 0x7a) ||
|
||||
(charCode >= /* A = */ 0x41 && charCode <= /* Z = */ 0x5a)
|
||||
wordSegmenter.segment(before + after.segment).containing(before.length)
|
||||
.index === before.length
|
||||
);
|
||||
}
|
||||
|
||||
function isAsciiDigit(charCode) {
|
||||
return charCode >= /* 0 = */ 0x30 && charCode <= /* 9 = */ 0x39;
|
||||
}
|
||||
|
||||
function isAsciiSpace(charCode) {
|
||||
return (
|
||||
charCode === /* SPACE = */ 0x20 ||
|
||||
charCode === /* TAB = */ 0x09 ||
|
||||
charCode === /* CR = */ 0x0d ||
|
||||
charCode === /* LF = */ 0x0a
|
||||
);
|
||||
}
|
||||
|
||||
function isHan(charCode) {
|
||||
return (
|
||||
(charCode >= 0x3400 && charCode <= 0x9fff) ||
|
||||
(charCode >= 0xf900 && charCode <= 0xfaff)
|
||||
);
|
||||
}
|
||||
|
||||
function isKatakana(charCode) {
|
||||
return charCode >= 0x30a0 && charCode <= 0x30ff;
|
||||
}
|
||||
|
||||
function isHiragana(charCode) {
|
||||
return charCode >= 0x3040 && charCode <= 0x309f;
|
||||
}
|
||||
|
||||
function isHalfwidthKatakana(charCode) {
|
||||
return charCode >= 0xff60 && charCode <= 0xff9f;
|
||||
}
|
||||
|
||||
function isThai(charCode) {
|
||||
return (charCode & 0xff80) === 0x0e00;
|
||||
}
|
||||
|
||||
/**
|
||||
* This function is based on the word-break detection implemented in:
|
||||
* https://hg.mozilla.org/mozilla-central/file/tip/intl/lwbrk/WordBreaker.cpp
|
||||
* Determine if the match spanning `[startIdx, startIdx + length)` in `content`
|
||||
* is a whole word, i.e. there's a word break on each side of it. Each boundary
|
||||
* is tested on its two adjacent characters in isolation rather than on the
|
||||
* whole string, so a contraction such as "can't" doesn't prevent "can" from
|
||||
* being an entire word, matching Firefox's find:
|
||||
* https://searchfox.org/firefox-main/source/toolkit/components/find/nsFind.cpp
|
||||
*/
|
||||
function getCharacterType(charCode) {
|
||||
if (isAlphabeticalScript(charCode)) {
|
||||
if (isAscii(charCode)) {
|
||||
if (isAsciiSpace(charCode)) {
|
||||
return CharacterType.SPACE;
|
||||
} else if (
|
||||
isAsciiAlpha(charCode) ||
|
||||
isAsciiDigit(charCode) ||
|
||||
charCode === /* UNDERSCORE = */ 0x5f
|
||||
) {
|
||||
return CharacterType.ALPHA_LETTER;
|
||||
}
|
||||
return CharacterType.PUNCT;
|
||||
} else if (isThai(charCode)) {
|
||||
return CharacterType.THAI_LETTER;
|
||||
} else if (charCode === /* NBSP = */ 0xa0) {
|
||||
return CharacterType.SPACE;
|
||||
}
|
||||
return CharacterType.ALPHA_LETTER;
|
||||
}
|
||||
|
||||
if (isHan(charCode)) {
|
||||
return CharacterType.HAN_LETTER;
|
||||
} else if (isKatakana(charCode)) {
|
||||
return CharacterType.KATAKANA_LETTER;
|
||||
} else if (isHiragana(charCode)) {
|
||||
return CharacterType.HIRAGANA_LETTER;
|
||||
} else if (isHalfwidthKatakana(charCode)) {
|
||||
return CharacterType.HALFWIDTH_KATAKANA_LETTER;
|
||||
}
|
||||
return CharacterType.ALPHA_LETTER;
|
||||
function isEntireWord(content, startIdx, length) {
|
||||
const endIdx = startIdx + length;
|
||||
return (
|
||||
(startIdx === 0 || isWordBreakAt(content, startIdx)) &&
|
||||
(endIdx === content.length || isWordBreakAt(content, endIdx))
|
||||
);
|
||||
}
|
||||
|
||||
let NormalizeWithNFKC;
|
||||
@ -191,4 +134,4 @@ function getNormalizeWithNFKC() {
|
||||
return NormalizeWithNFKC;
|
||||
}
|
||||
|
||||
export { CharacterType, getCharacterType, getNormalizeWithNFKC };
|
||||
export { getNormalizeWithNFKC, isEntireWord };
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user