Pull request #167: RED-3117: Enbled to ignore invalid entries at addToDictionary for automated pushing entries in analysis

Merge in RED/persistence-service from RED-3117 to master

* commit '8e088035dc42cc64f5b4862cde7d1514d40341d7':
  RED-3117: Enbled to ignore invalid entries at addToDictionary for automated pushing entries in analysis

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java

@@ -34,7 +34,8 @@ public interface DictionaryResource {
     @PostMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
     void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId,
                     @RequestBody List<String> entries,
-                    @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent);
+                    @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent,
+                    @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries);
 
 
     @ResponseStatus(HttpStatus.NO_CONTENT)

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java

@@ -36,6 +36,7 @@ import java.util.Set;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 import static java.util.stream.Collectors.toList;
@@ -54,11 +55,22 @@ public class DictionaryController implements DictionaryResource {
 
     @Override
     public void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries,
-                           @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent) {
+                           @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent,
+                           @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries) {
 
         Set<String> cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet());
 
-        validateEntries(cleanEntries);
+        if (CollectionUtils.isEmpty(entries)) {
+            throw new BadRequestException("Entry list is empty.");
+        }
+
+        var invalidEntries = getInvalidEntries(cleanEntries);
+
+        if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) {
+            throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries));
+        } else {
+            cleanEntries.removeAll(invalidEntries);
+        }
 
         // To check whether the type exists, type should not be added into database implicitly by addEntry.
         Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
@@ -99,7 +111,6 @@ public class DictionaryController implements DictionaryResource {
     @Override
     public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries) {
 
-        validateEntries(entries);
         // To check whether the type exists
         Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
 
@@ -240,25 +251,14 @@ public class DictionaryController implements DictionaryResource {
     }
 
 
-    private void validateEntries(List<String> entries) {
+    private Set<String> getInvalidEntries(Set<String> entries) {
-        validateEntries(new HashSet<>(entries));
-    }
 
-    private void validateEntries(Set<String> entries) {
-
-        if (CollectionUtils.isEmpty(entries)) {
-            throw new BadRequestException("Entry list is empty.");
-        }
         Predicate<String> isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent();
         Predicate<String> isStopword = stopwordService::isStopword;
 
-        List<String> errorMessages = entries.stream()
+        return entries.stream()
                 .filter(isDictionaryEntryNotValid.or(isStopword))
-                .collect(toList());
+                .collect(toSet());
-        if (CollectionUtils.isNotEmpty(errorMessages)) {
-            throw new BadRequestException("Error(s) validating dictionary entries:\n" + String.join("\n", errorMessages));
-        }
-
     }
 
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java

@@ -654,7 +654,7 @@ public class ManualRedactionService {
 
         try {
             log.debug("Adding entries for {} / {}", dossierId, fileId);
-            dictionaryController.addEntries(typeId, List.of(value), false);
+            dictionaryController.addEntries(typeId, List.of(value), false, false);
 
             fileStatusService.setStatusReprocess(dossierId, fileId, 2);
         } catch (Exception e) {

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java

@@ -45,7 +45,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         entries.add("age");
         entries.add("page");
         try {
-            dictionaryClient.addEntries(typeId, entries, false);
+            dictionaryClient.addEntries(typeId, entries, false, false);
         } catch (FeignException e) {
             assertThat(e.status()).isEqualTo(400);
         }
@@ -64,7 +64,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         var entries = new ArrayList<String>();
         entries.add(word);
         entries.add(word);
-        dictionaryClient.addEntries(type.getTypeId(), entries, false);
+        dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@@ -72,7 +72,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         // Act & Assert: Add same word again; Only one should exist
         entries = new ArrayList<>();
         entries.add(word);
-        dictionaryClient.addEntries(type.getTypeId(), entries, false);
+        dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@@ -83,7 +83,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         entries.add(word);
         entries.add(word);
         entries.add(word);
-        dictionaryClient.addEntries(type.getTypeId(), entries, false);
+        dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@@ -111,7 +111,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         var entries = new ArrayList<String>();
         entries.add(word);
         entries.add(word);
-        dictionaryClient.addEntries(type.getTypeId(), entries, true);
+        dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@@ -119,7 +119,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         // Act & Assert: Add same word again; Only one should exist
         entries = new ArrayList<>();
         entries.add(word);
-        dictionaryClient.addEntries(type.getTypeId(), entries, true);
+        dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@@ -130,7 +130,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         entries.add(word);
         entries.add(word);
         entries.add(word);
-        dictionaryClient.addEntries(type.getTypeId(), entries, true);
+        dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@@ -161,7 +161,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         entries.add(word1);
         entries.add(word2);
         entries.add(word3);
-        dictionaryClient.addEntries(type.getTypeId(), entries, false);
+        dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
@@ -172,7 +172,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         // Act & Assert: Add same word again; No duplicate should exist
         entries = new ArrayList<>();
         entries.add(word1);
-        dictionaryClient.addEntries(type.getTypeId(), entries, false);
+        dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
@@ -188,7 +188,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
         entries.add(word1);
         entries.add(word2);
         entries.add(word3);
-        dictionaryClient.addEntries(type.getTypeId(), entries, false);
+        dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
 
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
         assertThat(dictionaryClient.getEntriesForType(type.getTypeId())

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java

@@ -62,7 +62,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
         var entries1 = new ArrayList<String>();
         entries1.add("entry1");
         entries1.add("entry2");
-        dictionaryClient.addEntries(addedType.getTypeId(), entries1, false);
+        dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false);
 
         List<DictionaryEntry> entryList = dictionaryClient.getEntriesForType(addedType.getTypeId());
         assertThat(entryList.size()).isEqualTo(entries1.size());
@@ -81,7 +81,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
         entries2.add("entry1");
         entries2.add("entry2");
         entries2.add("entry3");
-        dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false);
+        dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false);
 
         entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId());
         assertThat(entryList.size()).isEqualTo(entries2.size());
@@ -102,7 +102,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
         var entries3 = new ArrayList<String>();
         entries3.add("entry1");
         entries3.add("entry2");
-        dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false);
+        dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false);
 
         entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId());
         assertThat(entryList.size()).isEqualTo(entries3.size());

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java

@@ -35,7 +35,7 @@ public class TypeTest extends AbstractPersistenceServerServiceTest {
 
         var type = typeProvider.testAndProvideType(dossierTemplate);
 
-        dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true);
+        dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false);
 
         var loadedType = dictionaryClient.getDictionaryForType(type.getId());