From 7e95174f57f4a25c8d2a5cbc009ca111fb5a6228 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Mon, 14 Nov 2022 23:33:42 +0200 Subject: [PATCH] RED-5562 japanse space characters --- .../v1/server/redaction/utils/SeparatorUtils.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java index 5e4766ec..76bf964f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java @@ -1,15 +1,25 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; +import lombok.extern.slf4j.Slf4j; + import java.util.Set; import java.util.regex.Pattern; +@Slf4j public class SeparatorUtils { private final static Set quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E' , '\u2039', '\u203A'); + private final static Set japaneseAltPunctuationMarks = Set.of(65288, 65289, 65294, 65339, 65341, 65371, 65373, 65375, 65376, 12443, 12444, 65309); public static boolean isSeparator(char c) { - return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c); + return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c) || isJapaneseSeparator(c); } + + public static boolean isJapaneseSeparator(char c) { + var intValue = (int) c; + return (intValue >= 12288 && intValue <= 12336) || japaneseAltPunctuationMarks.contains(intValue); + } + }