Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #499: RED-5562 japanse space characters

Browse Source 
Merge in RED/redaction-service from RED-5562-34x to release/3.163.x

* commit '5d68955be68a6a5bf0d1690c2f69419f78a4291f':
  RED-5562 improved pattern compile
  RED-5562 japanse space characters - fixed PMD
  RED-5562 japanse space characters
This commit is contained in:
Timo Bejan 2022-11-15 10:02:44 +01:00
commit 35f7b1ce1f
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java
View File

@ -1,15 +1,26 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils; package com.iqser.red.service.redaction.v1.server.redaction.utils;
import lombok.extern.slf4j.Slf4j;
import java.util.Set; import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@Slf4j
public class SeparatorUtils { public class SeparatorUtils {
private final static Pattern punctuationPattern = Pattern.compile("\\p{Punct}");
private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E' private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E'
, '\u2039', '\u203A'); , '\u2039', '\u203A');
private final static Set<Integer> japaneseAltPunctuationMarks = Set.of(65288, 65289, 65294, 65339, 65341, 65371, 65373, 65375, 65376, 12443, 12444, 65309);
public static boolean isSeparator(char c) { public static boolean isSeparator(char c) {
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c); return Character.isWhitespace(c) || punctuationPattern.matcher(String.valueOf(c)).matches() || quotes.contains(c) || isJapaneseSeparator(c);
} }
public static boolean isJapaneseSeparator(char c) {
var intValue = (int) c;
return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
}
} }