Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Added rank of dictionary to processing entities in redaction service, simplified code

Browse Source
This commit is contained in:
Timo 2020-11-26 18:52:44 +02:00
parent f458a1f930
commit 536d4689f3
4 changed files with 95 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/pom.xml
View File

@ -20,7 +20,7 @@
<dependency> <dependency>
<groupId>com.iqser.red.service</groupId> <groupId>com.iqser.red.service</groupId>
<artifactId>configuration-service-api-v1</artifactId> <artifactId>configuration-service-api-v1</artifactId>
<version>1.2.0</version> <version>1.3.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.drools</groupId> <groupId>org.drools</groupId>

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java Normal file
View File

@ -0,0 +1,25 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.Set;
@Data
@AllArgsConstructor
public class DictionaryModel {
private String type;
private int rank;
private float[] color;
private boolean caseInsensitive;
private boolean hint;
private Set<String> entries;
private Set<String> localEntries;
public Set<String> getValues(boolean local){
return local ? localEntries : entries;
}
}

90
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,6 +1,18 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.awt.Color; import com.iqser.red.service.configuration.v1.api.model.Colors;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import feign.FeignException;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import java.awt.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
@ -8,22 +20,8 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.configuration.v1.api.model.Colors;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import feign.FeignException;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@ -35,19 +33,7 @@ public class DictionaryService {
private long dictionaryVersion = -1; private long dictionaryVersion = -1;
@Getter @Getter
private Map<String, Set<String>> dictionary = new TreeMap<>(Comparator.reverseOrder()); // Using TreeMap, because order of keys is important. private List<DictionaryModel> dictionary = new ArrayList<>();
@Getter
private Map<String, Set<String>> localDictionary = new TreeMap<>(Comparator.reverseOrder()); // Using TreeMap, because order of keys is important.
@Getter
private Map<String, float[]> entryColors = new HashMap<>();
@Getter
private List<String> hintTypes = new ArrayList<>();
@Getter
private List<String> caseInsensitiveTypes = new ArrayList<>();
@Getter @Getter
private float[] defaultColor; private float[] defaultColor;
@ -61,16 +47,18 @@ public class DictionaryService {
@Getter @Getter
private float[] notRedactedColor; private float[] notRedactedColor;
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
public void addToLocalDictionary(String type, String value) { public boolean hasLocalEntries(){
return this.dictionary.stream().anyMatch(dm -> !dm.getLocalEntries().isEmpty());
localDictionary.computeIfAbsent(type, (x) -> new HashSet<>()).add(value);
} }
public void addToLocalDictionary(String type, String value) {
localAccessMap.get(type).getLocalEntries().add(value);
}
public void clearLocalDictionary() { public void clearLocalEntries() {
this.dictionary.forEach(dm -> dm.getLocalEntries().clear());
localDictionary = new TreeMap<>(Comparator.reverseOrder());
} }
@ -89,24 +77,16 @@ public class DictionaryService {
try { try {
TypeResponse typeResponse = dictionaryClient.getAllTypes(); TypeResponse typeResponse = dictionaryClient.getAllTypes();
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) { if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
entryColors = typeResponse.getTypes()
dictionary = typeResponse.getTypes()
.stream() .stream()
.collect(Collectors.toMap(TypeResult::getType, t -> convertColor(t.getHexColor()))); .map(t ->
hintTypes = typeResponse.getTypes() new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t.isHint(), convertEntries(t), new HashSet<>()))
.stream() .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.filter(TypeResult::isHint)
.map(TypeResult::getType)
.collect(Collectors.toList());
caseInsensitiveTypes = typeResponse.getTypes()
.stream()
.filter(TypeResult::isCaseInsensitive)
.map(TypeResult::getType)
.collect(Collectors.toList()); .collect(Collectors.toList());
dictionary = new TreeMap<>(Comparator.reverseOrder()); localAccessMap.clear();
entryColors.keySet().forEach(type -> { dictionary.forEach(dm -> localAccessMap.put(dm.getType(), dm));
dictionary.put(type, convertEntries(type));
});
Colors colors = dictionaryClient.getColors(); Colors colors = dictionaryClient.getColors();
defaultColor = convertColor(colors.getDefaultColor()); defaultColor = convertColor(colors.getDefaultColor());
@ -121,16 +101,17 @@ public class DictionaryService {
} }
private Set<String> convertEntries(String s) { private Set<String> convertEntries(TypeResult t) {
if (caseInsensitiveTypes.contains(s)) { if (t.isCaseInsensitive()) {
return dictionaryClient.getDictionaryForType(s) return dictionaryClient.getDictionaryForType(t.getType())
.getEntries() .getEntries()
.stream() .stream()
.map(String::toLowerCase) .map(String::toLowerCase)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} else {
return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType()).getEntries());
} }
return new HashSet<>(dictionaryClient.getDictionaryForType(s).getEntries());
} }
@ -140,4 +121,7 @@ public class DictionaryService {
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f}; return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
} }
public boolean isCaseInsensitiveDictionary(String type) {
return localAccessMap.get(type).isCaseInsensitive();
}
} }

73
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -1,5 +1,24 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -8,26 +27,6 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class EntityRedactionService { public class EntityRedactionService {
@ -35,18 +34,16 @@ public class EntityRedactionService {
private final DictionaryService dictionaryService; private final DictionaryService dictionaryService;
private final DroolsExecutionService droolsExecutionService; private final DroolsExecutionService droolsExecutionService;
public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) { public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {
dictionaryService.updateDictionary(); dictionaryService.updateDictionary();
droolsExecutionService.updateRules(); droolsExecutionService.updateRules();
dictionaryService.clearLocalDictionary(); dictionaryService.clearLocalEntries();
Set<Entity> documentEntities = new HashSet<>(); Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, false));
documentEntities.addAll(findEntities(classifiedDoc, manualRedactions, dictionaryService.getDictionary()));
if(!dictionaryService.getLocalDictionary().isEmpty()){ if (dictionaryService.hasLocalEntries()) {
Set<Entity> foundByLocal = findEntities(classifiedDoc, manualRedactions, dictionaryService.getLocalDictionary()); Set<Entity> foundByLocal = findEntities(classifiedDoc, manualRedactions, true);
// HashSet keeps the older value, but we want the new only. // HashSet keeps the older value, but we want the new only.
documentEntities.removeAll(foundByLocal); documentEntities.removeAll(foundByLocal);
documentEntities.addAll(foundByLocal); documentEntities.addAll(foundByLocal);
@ -70,7 +67,7 @@ public class EntityRedactionService {
} }
private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions, Map<String, Set<String>> dictionary){ private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions, boolean localEntries) {
Set<Entity> documentEntities = new HashSet<>(); Set<Entity> documentEntities = new HashSet<>();
int sectionNumber = 1; int sectionNumber = 1;
for (Paragraph paragraph : classifiedDoc.getParagraphs()) { for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
@ -105,7 +102,7 @@ public class EntityRedactionService {
searchableRow.addAll(textBlock.getSequences()); searchableRow.addAll(textBlock.getSequences());
} }
} }
Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary); Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, localEntries);
Section analysedRowSection = droolsExecutionService.executeRules(Section.builder() Section analysedRowSection = droolsExecutionService.executeRules(Section.builder()
.dictionaryService(dictionaryService) .dictionaryService(dictionaryService)
@ -124,7 +121,7 @@ public class EntityRedactionService {
} }
addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber); addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, dictionary); Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, localEntries);
Section analysedSection = droolsExecutionService.executeRules(Section.builder() Section analysedSection = droolsExecutionService.executeRules(Section.builder()
.dictionaryService(dictionaryService) .dictionaryService(dictionaryService)
.entities(entities) .entities(entities)
@ -146,18 +143,14 @@ public class EntityRedactionService {
removeEntitiesContainedInLarger(entities); removeEntitiesContainedInLarger(entities);
for (Entity entity : entities) { for (Entity entity : entities) {
if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) { entity.setPositionSequences(text.getSequences(entity.getWord(), dictionaryService.isCaseInsensitiveDictionary(entity.getType()), entity.getTargetSequences()));
entity.setPositionSequences(text.getSequences(entity.getWord(), true, entity.getTargetSequences()));
} else {
entity.setPositionSequences(text.getSequences(entity.getWord(), false, entity.getTargetSequences()));
}
} }
return entities; return entities;
} }
private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber, Map<String, Set<String>> dictionary) { private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber, boolean local) {
Set<Entity> found = new HashSet<>(); Set<Entity> found = new HashSet<>();
String searchableString = searchableText.toString(); String searchableString = searchableText.toString();
@ -166,16 +159,14 @@ public class EntityRedactionService {
} }
String lowercaseInputString = searchableString.toLowerCase(); String lowercaseInputString = searchableString.toLowerCase();
for (Map.Entry<String, Set<String>> entry : dictionary.entrySet()) { for (DictionaryModel model : dictionaryService.getDictionary()) {
if (dictionaryService.getCaseInsensitiveTypes().contains(entry.getKey())) { if (model.isCaseInsensitive()) {
found.addAll(find(lowercaseInputString, entry.getValue(), entry.getKey(), headline, sectionNumber)); found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber));
} else { } else {
found.addAll(find(searchableString, entry.getValue(), entry.getKey(), headline, sectionNumber)); found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber));
} }
} }
removeEntitiesContainedInLarger(found); removeEntitiesContainedInLarger(found);
return found; return found;
} }