Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-106: replace the local dictionary preload with remove dictionary service.

Browse Source
This commit is contained in:
cheng 2020-07-17 18:31:05 +02:00
parent dd0727e546
commit 4617b3535d
2 changed files with 13 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,8 +1,6 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -11,6 +9,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.configuration.v1.api.model.TypeResponse; import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import feign.FeignException; import feign.FeignException;
@ -30,28 +29,30 @@ public class DictionaryService {
@Getter @Getter
private Map<String, Set<String>> dictionaryEntry = new HashMap<>(); private Map<String, Set<String>> dictionaryEntry = new HashMap<>();
@Getter
private Map<String, float[]> entryColors = new HashMap<>();
public void updateDictionary() { public void updateDictionary() {
long version = 1; // TODO = dictionaryClient.getVersion(); long version = 1; // TODO = dictionaryClient.getVersion();
if (version > dictionaryVersion) { if (version > dictionaryVersion) {
dictionaryVersion = version; dictionaryVersion = version;
dictionaryEntry = retrieveDictionaryEntry(); updateDictionaryEntry();
} }
} }
private Map<String, Set<String>> retrieveDictionaryEntry() { private void updateDictionaryEntry() {
try { try {
TypeResponse typeResponse = dictionaryClient.getAllTypes(); TypeResponse typeResponse = dictionaryClient.getAllTypes();
if (typeResponse == null || CollectionUtils.isEmpty(typeResponse.getTypes())) { if (typeResponse == null || CollectionUtils.isEmpty(typeResponse.getTypes())) {
return Collections.emptyMap(); return;
} else { } else {
List<String> types = typeResponse.getTypes().stream().map(typeResult -> typeResult.getType()).collect(Collectors.toList()); entryColors = typeResponse.getTypes().stream().collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
return types.stream().collect(Collectors.toMap(type -> type, s -> dictionaryClient.getDictionaryForType(s).getEntries().stream().collect(Collectors.toSet()))); dictionaryEntry = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, s -> dictionaryClient.getDictionaryForType(s).getEntries().stream().collect(Collectors.toSet())));
} }
} catch (FeignException e) { } catch (FeignException e) {
log.warn("Got some unknown feignException", e); log.warn("Got some unknown feignException", e);
throw e; throw e;
} }
} }
} }

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.redaction.v1.server.visualization.service; package com.iqser.red.service.redaction.v1.server.visualization.service;
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.ADDRESS_CODE;
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NAME_CODE;
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NO_REDACTION_INDICATOR;
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.VERTEBRATES_CODE;
import java.awt.Color; import java.awt.Color;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -29,6 +24,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@ -40,7 +36,7 @@ import lombok.extern.slf4j.Slf4j;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class AnnotationHighlightService { public class AnnotationHighlightService {
private final DictionaryService dictionaryService;
public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException { public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {
@ -177,36 +173,16 @@ public class AnnotationHighlightService {
if (!entity.isRedaction()) { if (!entity.isRedaction()) {
return false; return false;
} }
if (entity.getType().equals(ADDRESS_CODE)) { return dictionaryService.getDictionaryEntry().keySet().contains(entity.getType());
return true;
}
if (entity.getType().equals(NAME_CODE)) {
return true;
}
return false;
} }
private float[] getColor(Entity entity) { private float[] getColor(Entity entity) {
if (!entity.isRedaction()) { if (!entity.isRedaction()) {
return new float[]{0.627f, 0.627f, 0.627f}; return new float[]{0.627f, 0.627f, 0.627f};
} }
if (entity.getType().equals(VERTEBRATES_CODE)) { return dictionaryService.getEntryColors().get(entity.getType());
return new float[]{0, 1, 0};
}
if (entity.getType().equals(ADDRESS_CODE)) {
return new float[]{0, 1, 1};
}
if (entity.getType().equals(NAME_CODE)) {
return new float[]{1, 1, 0};
}
if (entity.getType().equals(NO_REDACTION_INDICATOR)) {
return new float[]{1, 0.502f, 0};
}
return null;
} }
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException { private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
contentStream.setStrokingColor(Color.LIGHT_GRAY); contentStream.setStrokingColor(Color.LIGHT_GRAY);