Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #232: typeId to type internal refactor

Browse Source 
Merge in RED/redaction-service from persistence-service-integration to master

* commit 'dcd6d9d2c9cb66ad55e08d82a6044ea1b01c54eb':
  fixed pmd
  typeId to type internal refactor
This commit is contained in:
Timo Bejan 2021-09-29 15:57:45 +02:00
commit 10ae7fc20d
7 changed files with 28 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/pom.xml
View File

@ -20,7 +20,7 @@
<dependency> <dependency>
<groupId>com.iqser.red.service</groupId> <groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-api-v1</artifactId> <artifactId>persistence-service-api-v1</artifactId>
<version>0.16.0</version> <version>0.22.0</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -22,7 +22,7 @@ import java.util.Set;
public class RedactionLogEntry { public class RedactionLogEntry {
private String id; private String id;
private String typeId; private String type;
private String value; private String value;
private String reason; private String reason;
private int matchedRule; private int matchedRule;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -109,7 +109,7 @@ public class AnalyzeService {
dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
.getDossierTemplateId(), analyzeRequest.getDossierId()); .getDossierTemplateId());
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary
@ -162,7 +162,7 @@ public class AnalyzeService {
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities); PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
.getDossierTemplateId(), analyzeRequest.getDossierId()); .getDossierTemplateId());
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
@ -254,7 +254,7 @@ public class AnalyzeService {
Rectangle position = entry.getPositions().get(0); Rectangle position = entry.getPositions().get(0);
return Image.builder() return Image.builder()
.type(entry.getTypeId().split(":")[0]) .type(entry.getType())
.position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft() .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft()
.getY(), position.getWidth(), position.getHeight())) .getY(), position.getWidth(), position.getHeight()))
.sectionNumber(entry.getSectionNumber()) .sectionNumber(entry.getSectionNumber())

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -84,7 +84,7 @@ public class DictionaryService {
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId) : dictionaryClient.getAllTypesForDossier(dossierId); var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId) : dictionaryClient.getAllTypesForDossier(dossierId);
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse)) { if (CollectionUtils.isNotEmpty(typeResponse)) {
List<DictionaryModel> dictionary = typeResponse List<DictionaryModel> dictionary = typeResponse
.stream() .stream()

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -1,15 +1,5 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
@ -20,8 +10,12 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@ -30,14 +24,13 @@ public class RedactionLogCreatorService {
private final DictionaryService dictionaryService; private final DictionaryService dictionaryService;
public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages, public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages, String dossierTemplateId) {
String dossierTemplateId, String dossierId) {
List<RedactionLogEntry> entries = new ArrayList<>(); List<RedactionLogEntry> entries = new ArrayList<>();
for (int page = 1; page <= numberOfPages; page++) { for (int page = 1; page <= numberOfPages; page++) {
if (pageEntities.getEntitiesPerPage().get(page) != null) { if (pageEntities.getEntitiesPerPage().get(page) != null) {
entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId, dossierId)); entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId));
} }
if (pageEntities.getImagesPerPage().get(page) != null) { if (pageEntities.getImagesPerPage().get(page) != null) {
@ -62,7 +55,7 @@ public class RedactionLogCreatorService {
.id(id) .id(id)
.color(getColor(image.getType(), dossierTemplateId, image.isRedaction())) .color(getColor(image.getType(), dossierTemplateId, image.isRedaction()))
.isImage(true) .isImage(true)
.typeId(IdBuilder.getTypeId(image.getType(), dossierTemplateId, null)) .type(image.getType())
.redacted(image.isRedaction()) .redacted(image.isRedaction())
.reason(image.getRedactionReason()) .reason(image.getRedactionReason())
.legalBasis(image.getLegalBasis()) .legalBasis(image.getLegalBasis())
@ -86,7 +79,7 @@ public class RedactionLogCreatorService {
} }
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId, String dossierId) { public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>(); List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -98,7 +91,7 @@ public class RedactionLogCreatorService {
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId, dossierId); RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
if (processedIds.contains(entityPositionSequence.getId())) { if (processedIds.contains(entityPositionSequence.getId())) {
// TODO refactor this outer loop jump as soon as we have the time. // TODO refactor this outer loop jump as soon as we have the time.
@ -157,7 +150,7 @@ public class RedactionLogCreatorService {
} }
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId, String dossierId) { private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId()))); entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId())));
@ -167,7 +160,7 @@ public class RedactionLogCreatorService {
.reason(entity.getRedactionReason()) .reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis()) .legalBasis(entity.getLegalBasis())
.value(entity.getWord()) .value(entity.getWord())
.typeId(IdBuilder.getTypeId(entity.getType(), dossierTemplateId, entity.isDossierDictionaryEntry() ? dossierId : null)) .type(entity.getType())
.redacted(entity.isRedaction()) .redacted(entity.isRedaction())
.isHint(isHint(entity.getType(), dossierTemplateId)) .isHint(isHint(entity.getType(), dossierTemplateId))
.isRecommendation(isRecommendation(entity.getType(), dossierTemplateId)) .isRecommendation(isRecommendation(entity.getType(), dossierTemplateId))
@ -207,5 +200,4 @@ public class RedactionLogCreatorService {
} }
} }

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -110,12 +110,12 @@ public class RedactionLogMergeService {
String manualOverrideReason = null; String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) { if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setTypeId(IdBuilder.getTypeId(imageRecategorization.getType(), dossierTemplateId, null)); redactionLogEntry.setType(imageRecategorization.getType());
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
} else { } else {
@ -135,12 +135,12 @@ public class RedactionLogMergeService {
redactionLogEntry.setRedacted(false); redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), true)); .isRedacted(), true));
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
} else { } else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED); redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
@ -160,14 +160,14 @@ public class RedactionLogMergeService {
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) { if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(true); redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else { } else {
@ -191,7 +191,7 @@ public class RedactionLogMergeService {
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else { } else {
@ -258,13 +258,13 @@ public class RedactionLogMergeService {
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(id) .id(id)
.color(getColorForManualAdd(getTypeFromTypeId(manualRedactionEntry.getTypeId()), dossierTemplateId, manualRedactionEntry.getStatus())) .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason()) .reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary()) .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis()) .legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue()) .value(manualRedactionEntry.getValue())
.typeId(manualRedactionEntry.getTypeId()) .type(manualRedactionEntry.getType())
.redacted(true) .redacted(true)
.isHint(false) .isHint(false)
.section(null) .section(null)
@ -324,12 +324,6 @@ public class RedactionLogMergeService {
} }
private String getTypeFromTypeId(String typeId){
return typeId.split(":")[0];
}
} }

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -682,12 +682,6 @@ public class RedactionIntegrationTest {
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID); var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.isImage()) {
System.out.println("---->" + entry.getTypeId());
}
});
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();
System.out.println("first analysis duration: " + (end - start)); System.out.println("first analysis duration: " + (end - start));
@ -848,7 +842,7 @@ public class RedactionIntegrationTest {
manualRedactionEntry.setAnnotationId(manualAddId); manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId"); manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED); manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED);
manualRedactionEntry.setTypeId("name:" + TEST_DOSSIER_TEMPLATE_ID); manualRedactionEntry.setType("name");
manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction"); manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(Rectangle manualRedactionEntry.setPositions(List.of(Rectangle