Merge branch 'RED-8650-bp' into 'release/4.190.x'

RED-8650 - Support more date formats

See merge request redactmanager/redaction-service!378

.gitlab-ci.yml

@@ -21,3 +21,5 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
+pmd:
+  allow_failure: true

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -40,7 +40,10 @@ public class DocumentTree {
 
     public TextBlock buildTextBlock() {
 
-        return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return allEntriesInOrder().map(Entry::getNode)
+                .filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -89,8 +92,8 @@ public class DocumentTree {
         if (treeId.isEmpty()) {
             return root != null;
         }
-        Entry entry = root.children.get(treeId.get(0));
-        for (int id : treeId.subList(1, treeId.size())) {
+        Entry entry = root;
+        for (int id : treeId) {
             if (id >= entry.children.size() || 0 > id) {
                 return false;
             }
@@ -114,13 +117,16 @@ public class DocumentTree {
 
     public Stream<SemanticNode> childNodes(List<Integer> treeId) {
 
-        return getEntryById(treeId).children.stream().map(Entry::getNode);
+        return getEntryById(treeId).children.stream()
+                .map(Entry::getNode);
     }
 
 
     public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) {
 
-        return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode);
+        return getEntryById(treeId).children.stream()
+                .filter(entry -> entry.node.getType().equals(nodeType))
+                .map(Entry::getNode);
     }
 
 
@@ -199,26 +205,32 @@ public class DocumentTree {
 
     public Stream<Entry> allEntriesInOrder() {
 
-        return Stream.of(root).flatMap(DocumentTree::flatten);
+        return Stream.of(root)
+                .flatMap(DocumentTree::flatten);
     }
 
 
     public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) {
 
-        return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten);
+        return getEntryById(parentId).children.stream()
+                .flatMap(DocumentTree::flatten);
     }
 
 
     @Override
     public String toString() {
 
-        return String.join("\n", allEntriesInOrder().map(Entry::toString).toList());
+        return String.join("\n",
+                           allEntriesInOrder().map(Entry::toString)
+                                   .toList());
     }
 
 
     private static Stream<Entry> flatten(Entry entry) {
 
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTree::flatten));
+        return Stream.concat(Stream.of(entry),
+                             entry.children.stream()
+                                     .flatMap(DocumentTree::flatten));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -599,6 +599,9 @@ public class EntityCreationService {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
+        if (trimmedTextRange.length() == 0){
+            return Optional.empty();
+        }
         TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
         if (node.getEntities().contains(entity)) {
             return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java

@@ -1,10 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.utils;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.format.ResolverStyle;
 import java.util.Date;
-import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.Optional;
 
 import lombok.AccessLevel;
@@ -17,39 +26,65 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DateConverter {
 
-    static List<SimpleDateFormat> formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH),
-            new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH),
-            new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH),
-            new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH),
-            new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH),
-            new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
+    private static DateTimeFormatter DATE_TIME_FORMATTER;
 
 
     public Optional<Date> parseDate(String dateAsString) {
 
-        Date date = null;
-        for (SimpleDateFormat format : formats) {
-
-            try {
-                date = format.parse(dateAsString);
-                break;
-            } catch (Exception e) {
-                log.warn("Failed to parse date from string {}. \n{}", dateAsString, e.getMessage());
-                // ignore, try next...
-            }
-        }
-        if (date == null) {
+        DateTimeFormatter formatter = getDateTimeFormatter();
+        String cleanDate = dateAsString.trim();
+        cleanDate = removeTrailingDot(cleanDate);
+        try {
+            LocalDate localDate = LocalDate.parse(cleanDate, formatter);
+            Date date = Date.from(localDate.atStartOfDay(ZoneId.systemDefault()).toInstant());
+            return Optional.of(date);
+        } catch (DateTimeParseException e) {
+            log.warn("Failed to parse date: {}", cleanDate);
             return Optional.empty();
         }
-        return Optional.of(date);
+
     }
 
 
     public String convertDate(Date date, String resultFormat) {
 
-        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
-
+        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.UK);
         return resultDateFormat.format(date);
     }
 
+
+    private DateTimeFormatter getDateTimeFormatter() {
+
+        if (DATE_TIME_FORMATTER == null) {
+            DATE_TIME_FORMATTER = createFormatterFromResource();
+        }
+        return DATE_TIME_FORMATTER;
+    }
+
+
+    private DateTimeFormatter createFormatterFromResource() {
+
+        DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(DateConverter.class.getResourceAsStream("/date_formats.txt"))))) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                builder.appendOptional(DateTimeFormatter.ofPattern(line.trim(), Locale.UK));
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Error reading date format file: " + e.getMessage());
+        }
+        return builder.toFormatter().withResolverStyle(ResolverStyle.SMART).withLocale(Locale.UK);
+    }
+
+
+    private String removeTrailingDot(String dateAsString) {
+
+        String str = dateAsString;
+        if (str != null && !str.isEmpty() && str.charAt(str.length() - 1) == '.') {
+            str = str.substring(0, str.length() - 1);
+        }
+
+        return str;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java

@@ -151,7 +151,9 @@ public class RedactionSearchUtility {
         Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getTextRange()));
         List<TextRange> boundaries = new LinkedList<>();
         while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            if (!matcher.group(group).isBlank()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
         }
         return boundaries;
     }
@@ -163,7 +165,9 @@ public class RedactionSearchUtility {
         Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
         List<TextRange> boundaries = new LinkedList<>();
         while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            if (!matcher.group(group).isBlank()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
         }
         return boundaries;
     }

redaction-service-v1/redaction-service-server-v1/src/main/resources/date_formats.txt

@@ -0,0 +1,38 @@
+dd-MMM-yyyy
+dd MMM yyyy
+dd MMM yy
+dd MM yyyy
+dd MMMM yyyy
+MMMM dd, yyyy
+dd.MM.yyyy
+yyyy/MM/dd
+yyyy-MM-dd
+dd-MM-yyyy
+MMMM d, yyyy
+d MMMM yyyy
+MMM d, yyyy
+d['.'] MMM yyyy
+d-MMM-yyyy
+d['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+MMMM d['th']['st']['nd']['rd'], yyyy
+yyyy, MMMM d
+yyyy.MM.dd
+yyyyMMdd
+dd-MM-yy
+dd/MM/yy
+MMMM d, yy
+d MMMM, yy
+d['th']['st']['nd']['rd'] MMM yyyy
+MMM d['th']['st']['nd']['rd'], yy
+yyyy-MMM-dd
+MMM-dd-yyyy
+dd-MM-yyyy
+yyyy, MMMM dd
+dd.MMM.yyyy
+dd.MMMM.yyyy
+dd.MMM.yy
+dd.MMMM.yy
+dd.MMM-yyyy
+dd.MMMM-yyyy
+d['th']['st']['nd']['rd'] MMMM yy
+d['th']['st']['nd']['rd'] MMMM yyyy

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
@@ -106,6 +108,36 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testConvertingVariousDateFormats() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/dates/date_formats.pdf");
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
+        System.out.println("Finished structure analysis");
+        analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+
+        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertNotNull(componentLog);
+
+        var experimentalDates = componentLog.getComponentLogEntries().stream().filter(c -> c.getName().equals("Experimental_Starting_Date")).findFirst().get();
+        assertNotNull(experimentalDates);
+
+        String dates = experimentalDates.getComponentValues().get(0).getValue();
+        String[] dateArray = dates.split(", ");
+        boolean allEqual = true;
+        for (String date : dateArray) {
+            if (!"10/01/2022".equals(date) && !"08/09/2024".equals(date)) {
+                allEqual = false;
+                break;
+            }
+        }
+
+        assertTrue(allEqual);
+    }
+
+
     @Test
 //    @Disabled
     public void testTopOfPage13InNotHeader() throws IOException {