From 8e088035dc42cc64f5b4862cde7d1514d40341d7 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Mon, 20 Dec 2021 11:40:17 +0100 Subject: [PATCH] RED-3117: Enbled to ignore invalid entries at addToDictionary for automated pushing entries in analysis --- .../v1/api/resources/DictionaryResource.java | 3 +- .../controller/DictionaryController.java | 34 +++++++++---------- .../service/ManualRedactionService.java | 2 +- .../integration/tests/DictionaryTest.java | 20 +++++------ .../tests/DossierTemplateStatsTest.java | 6 ++-- .../v1/server/integration/tests/TypeTest.java | 2 +- 6 files changed, 34 insertions(+), 33 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java index 405c01444..775f50cfe 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java @@ -34,7 +34,8 @@ public interface DictionaryResource { @PostMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries, - @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent); + @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent, + @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries); @ResponseStatus(HttpStatus.NO_CONTENT) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java index e9d387fe6..ae0233712 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; import static java.util.stream.Collectors.toList; @@ -54,11 +55,22 @@ public class DictionaryController implements DictionaryResource { @Override public void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries, - @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent) { + @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent, + @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries) { Set cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet()); - validateEntries(cleanEntries); + if (CollectionUtils.isEmpty(entries)) { + throw new BadRequestException("Entry list is empty."); + } + + var invalidEntries = getInvalidEntries(cleanEntries); + + if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) { + throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries)); + } else { + cleanEntries.removeAll(invalidEntries); + } // To check whether the type exists, type should not be added into database implicitly by addEntry. Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); @@ -99,7 +111,6 @@ public class DictionaryController implements DictionaryResource { @Override public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries) { - validateEntries(entries); // To check whether the type exists Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); @@ -240,25 +251,14 @@ public class DictionaryController implements DictionaryResource { } - private void validateEntries(List entries) { - validateEntries(new HashSet<>(entries)); - } + private Set getInvalidEntries(Set entries) { - private void validateEntries(Set entries) { - - if (CollectionUtils.isEmpty(entries)) { - throw new BadRequestException("Entry list is empty."); - } Predicate isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent(); Predicate isStopword = stopwordService::isStopword; - List errorMessages = entries.stream() + return entries.stream() .filter(isDictionaryEntryNotValid.or(isStopword)) - .collect(toList()); - if (CollectionUtils.isNotEmpty(errorMessages)) { - throw new BadRequestException("Error(s) validating dictionary entries:\n" + String.join("\n", errorMessages)); - } - + .collect(toSet()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 087d80169..eeaa29ec2 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -654,7 +654,7 @@ public class ManualRedactionService { try { log.debug("Adding entries for {} / {}", dossierId, fileId); - dictionaryController.addEntries(typeId, List.of(value), false); + dictionaryController.addEntries(typeId, List.of(value), false, false); fileStatusService.setStatusReprocess(dossierId, fileId, 2); } catch (Exception e) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java index b2db06564..3c50f0bd5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java @@ -45,7 +45,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add("age"); entries.add("page"); try { - dictionaryClient.addEntries(typeId, entries, false); + dictionaryClient.addEntries(typeId, entries, false, false); } catch (FeignException e) { assertThat(e.status()).isEqualTo(400); } @@ -64,7 +64,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { var entries = new ArrayList(); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); @@ -72,7 +72,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { // Act & Assert: Add same word again; Only one should exist entries = new ArrayList<>(); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); @@ -83,7 +83,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); @@ -111,7 +111,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { var entries = new ArrayList(); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, true); + dictionaryClient.addEntries(type.getTypeId(), entries, true, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); @@ -119,7 +119,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { // Act & Assert: Add same word again; Only one should exist entries = new ArrayList<>(); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, true); + dictionaryClient.addEntries(type.getTypeId(), entries, true, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); @@ -130,7 +130,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, true); + dictionaryClient.addEntries(type.getTypeId(), entries, true, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); @@ -161,7 +161,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word1); entries.add(word2); entries.add(word3); - dictionaryClient.addEntries(type.getTypeId(), entries, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()) @@ -172,7 +172,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { // Act & Assert: Add same word again; No duplicate should exist entries = new ArrayList<>(); entries.add(word1); - dictionaryClient.addEntries(type.getTypeId(), entries, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()) @@ -188,7 +188,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word1); entries.add(word2); entries.add(word3); - dictionaryClient.addEntries(type.getTypeId(), entries, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false); assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3); assertThat(dictionaryClient.getEntriesForType(type.getTypeId()) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 25316012b..30323b391 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -62,7 +62,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var entries1 = new ArrayList(); entries1.add("entry1"); entries1.add("entry2"); - dictionaryClient.addEntries(addedType.getTypeId(), entries1, false); + dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false); List entryList = dictionaryClient.getEntriesForType(addedType.getTypeId()); assertThat(entryList.size()).isEqualTo(entries1.size()); @@ -81,7 +81,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe entries2.add("entry1"); entries2.add("entry2"); entries2.add("entry3"); - dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false); + dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false); entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId()); assertThat(entryList.size()).isEqualTo(entries2.size()); @@ -102,7 +102,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var entries3 = new ArrayList(); entries3.add("entry1"); entries3.add("entry2"); - dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false); + dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false); entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId()); assertThat(entryList.size()).isEqualTo(entries3.size()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java index 058a9e759..2586f3140 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java @@ -35,7 +35,7 @@ public class TypeTest extends AbstractPersistenceServerServiceTest { var type = typeProvider.testAndProvideType(dossierTemplate); - dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true); + dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false); var loadedType = dictionaryClient.getDictionaryForType(type.getId());