Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #167: RED-3117: Enbled to ignore invalid entries at addToDictionary for automated pushing entries in analysis

Browse Source 
Merge in RED/persistence-service from RED-3117 to master

* commit '8e088035dc42cc64f5b4862cde7d1514d40341d7':
  RED-3117: Enbled to ignore invalid entries at addToDictionary for automated pushing entries in analysis
This commit is contained in:
Dominique Eiflaender 2021-12-20 11:44:11 +01:00
commit d049bb5e32
6 changed files with 34 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java
View File

@ -34,7 +34,8 @@ public interface DictionaryResource {
@PostMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId,
@RequestBody List<String> entries,
@RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent);
@RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent,
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries);
@ResponseStatus(HttpStatus.NO_CONTENT)

34
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java
View File

@ -36,6 +36,7 @@ import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static java.util.stream.Collectors.toList;
@ -54,11 +55,22 @@ public class DictionaryController implements DictionaryResource {
@Override
public void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries,
@RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent) {
@RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent,
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries) {
Set<String> cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet());
validateEntries(cleanEntries);
if (CollectionUtils.isEmpty(entries)) {
throw new BadRequestException("Entry list is empty.");
}
var invalidEntries = getInvalidEntries(cleanEntries);
if (!ignoreInvalidEntries && CollectionUtils.isNotEmpty(invalidEntries)) {
throw new BadRequestException("Error(s) validating dictionary entries:\\n" + String.join("\\n", invalidEntries));
} else {
cleanEntries.removeAll(invalidEntries);
}
// To check whether the type exists, type should not be added into database implicitly by addEntry.
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
@ -99,7 +111,6 @@ public class DictionaryController implements DictionaryResource {
@Override
public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries) {
validateEntries(entries);
// To check whether the type exists
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
@ -240,25 +251,14 @@ public class DictionaryController implements DictionaryResource {
}
private void validateEntries(List<String> entries) {
validateEntries(new HashSet<>(entries));
}
private Set<String> getInvalidEntries(Set<String> entries) {
private void validateEntries(Set<String> entries) {
if (CollectionUtils.isEmpty(entries)) {
throw new BadRequestException("Entry list is empty.");
}
Predicate<String> isDictionaryEntryNotValid = entry -> DictionaryValidator.validateDictionaryEntry(entry).isPresent();
Predicate<String> isStopword = stopwordService::isStopword;
List<String> errorMessages = entries.stream()
return entries.stream()
.filter(isDictionaryEntryNotValid.or(isStopword))
.collect(toList());
if (CollectionUtils.isNotEmpty(errorMessages)) {
throw new BadRequestException("Error(s) validating dictionary entries:\n" + String.join("\n", errorMessages));
}
.collect(toSet());
}

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -654,7 +654,7 @@ public class ManualRedactionService {
try {
log.debug("Adding entries for {} / {}", dossierId, fileId);
dictionaryController.addEntries(typeId, List.of(value), false);
dictionaryController.addEntries(typeId, List.of(value), false, false);
fileStatusService.setStatusReprocess(dossierId, fileId, 2);
} catch (Exception e) {

20
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java
View File

@ -45,7 +45,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add("age");
entries.add("page");
try {
dictionaryClient.addEntries(typeId, entries, false);
dictionaryClient.addEntries(typeId, entries, false, false);
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(400);
}
@ -64,7 +64,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
var entries = new ArrayList<String>();
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@ -72,7 +72,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
// Act & Assert: Add same word again; Only one should exist
entries = new ArrayList<>();
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@ -83,7 +83,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word);
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@ -111,7 +111,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
var entries = new ArrayList<String>();
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, true);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@ -119,7 +119,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
// Act & Assert: Add same word again; Only one should exist
entries = new ArrayList<>();
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, true);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@ -130,7 +130,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word);
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, true);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
@ -161,7 +161,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word1);
entries.add(word2);
entries.add(word3);
dictionaryClient.addEntries(type.getTypeId(), entries, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
@ -172,7 +172,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
// Act & Assert: Add same word again; No duplicate should exist
entries = new ArrayList<>();
entries.add(word1);
dictionaryClient.addEntries(type.getTypeId(), entries, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
@ -188,7 +188,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word1);
entries.add(word2);
entries.add(word3);
dictionaryClient.addEntries(type.getTypeId(), entries, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())

6
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java
View File

@ -62,7 +62,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var entries1 = new ArrayList<String>();
entries1.add("entry1");
entries1.add("entry2");
dictionaryClient.addEntries(addedType.getTypeId(), entries1, false);
dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false);
List<DictionaryEntry> entryList = dictionaryClient.getEntriesForType(addedType.getTypeId());
assertThat(entryList.size()).isEqualTo(entries1.size());
@ -81,7 +81,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
entries2.add("entry1");
entries2.add("entry2");
entries2.add("entry3");
dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false);
dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false);
entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId());
assertThat(entryList.size()).isEqualTo(entries2.size());
@ -102,7 +102,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var entries3 = new ArrayList<String>();
entries3.add("entry1");
entries3.add("entry2");
dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false);
dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false);
entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId());
assertThat(entryList.size()).isEqualTo(entries3.size());

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java
View File

@ -35,7 +35,7 @@ public class TypeTest extends AbstractPersistenceServerServiceTest {
var type = typeProvider.testAndProvideType(dossierTemplate);
dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true);
dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false);
var loadedType = dictionaryClient.getDictionaryForType(type.getId());