Merge branch 'RED-6940' into 'master'

RED-6940 - Sanitize /r in dictionary entries

Closes RED-6940

See merge request redactmanager/persistence-service!355

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java

@@ -1,26 +1,10 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
-import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
-import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
-import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
-import com.iqser.red.service.persistence.management.v1.processor.utils.TextNormalizationUtilities;
-import com.iqser.red.service.persistence.management.v1.processor.utils.TypeMapper;
-import com.iqser.red.service.persistence.management.v1.processor.validation.DictionaryValidator;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
-import jakarta.transaction.Transactional;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.stereotype.Service;
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.getDossierIdFromTypeId;
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.getDossierTemplateIdFromTypeId;
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.getDosssierTemplateTypeIdFromTypeId;
+import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.isDossierTypeId;
+import static java.util.stream.Collectors.toSet;
 
 import java.util.HashSet;
 import java.util.List;
@@ -31,8 +15,29 @@ import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.*;
-import static java.util.stream.Collectors.toSet;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.StringCleaningUtility;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
+import com.iqser.red.service.persistence.management.v1.processor.utils.TypeMapper;
+import com.iqser.red.service.persistence.management.v1.processor.validation.DictionaryValidator;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
+
+import jakarta.transaction.Transactional;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
@@ -228,7 +233,7 @@ public class DictionaryManagementService {
 
         checkForDossierTypeExistenceAndCreate(typeId);
 
-        Set<String> cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet());
+        Set<String> cleanEntries = entries.stream().map(StringCleaningUtility::cleanString).collect(toSet());
 
         if (CollectionUtils.isEmpty(entries)) {
             throw new BadRequestException("Entry list is empty.");
@@ -291,12 +296,6 @@ public class DictionaryManagementService {
     }
 
 
-    private String cleanDictionaryEntry(String entry) {
-
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-
-
     private Set<String> getInvalidEntries(Set<String> entries) {
 
         Predicate<String> isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent();

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java

@@ -614,6 +614,25 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
     }
 
 
+    @Test
+    public void testAddDictionaryEntryWithCarriageReturn() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var type = typeProvider.testAndProvideType(dossierTemplate);
+        var word = "Luke Skywalker ";
+        var wordWithCarriageReturn = "Luke Skywalker\r";
+
+        var entries = new ArrayList<String>();
+        entries.add(wordWithCarriageReturn);
+        dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY);
+
+        var dictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
+
+        assertThat(dictionary.getEntries()).hasSize(1);
+        assertThat(dictionary.getEntries().get(0)).isEqualTo(word);
+    }
+
+
     @SuppressWarnings("SameParameterValue")
     private List<String> createDummyEntries(int numberOfEntries) {
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -266,6 +266,34 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
     }
 
 
+    @Test
+    public void testAddToDictionaryEntryWithCarriageReturn() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+        var type = typeProvider.testAndProvideType(dossierTemplate);
+
+        manualRedactionClient.addRedactionBulk(dossier.getId(),
+                                               file.getId(),
+                                               Set.of(AddRedactionRequestModel.builder()
+                                                              .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build()))
+                                                              .section("section test")
+                                                              .addToDictionary(true)
+                                                              .addToAllDossiers(false)
+                                                              .type(type.getType())
+                                                              .reason("1")
+                                                              .value("Luke Skywalker\r")
+                                                              .legalBasis("1")
+                                                              .sourceId("SourceId")
+                                                              .build()));
+
+        var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
+        assertThat(dossierDictionary.getEntries().size()).isEqualTo(1);
+        assertEquals(dossierDictionary.getEntries().get(0).getValue(), "Luke Skywalker ");
+    }
+
+
     @Test
     public void testManualRemoveFromAllDossiersAndUndo() {
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/StringCleaningUtilityTest.java

@@ -50,4 +50,11 @@ public class StringCleaningUtilityTest {
         assertEquals("abc def, ghi-jkl mnopqr", StringCleaningUtility.cleanString(originalValue));
     }
 
+    @Test
+    public void testRemoveReturnCarriage() {
+
+        String originalValue = "abc\r";
+        assertEquals("abc ", StringCleaningUtility.cleanString(originalValue));
+    }
+
 }