Merge branch 'RED-7784-additional-fixes' into 'master'
RED-7785 - Fix false positives not getting removed for unprocessed changes & refactor merging unprocessed changes into entitylog logic. Closes RED-7784 See merge request redactmanager/persistence-service!228
This commit is contained in:
Andrei Isvoran
commit
9e04a47e90
@ -0,0 +1,316 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class EntityLogMergeService {
|
||||
|
||||
private final DictionaryPersistenceService dictionaryPersistenceService;
|
||||
|
||||
|
||||
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier) {
|
||||
|
||||
log.info("Merging EntityLog");
|
||||
List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions);
|
||||
|
||||
// Sort manual changes by date so we process them in order of when they were requested
|
||||
allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
|
||||
allManualChanges.forEach(manualChange -> {
|
||||
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
|
||||
if (manualChange instanceof ManualRedactionEntry) {
|
||||
mergeManualRedactionEntries((ManualRedactionEntry) manualChange, unprocessedManualEntities, entityLog, dossier);
|
||||
} else if (manualChange instanceof IdRemoval) {
|
||||
mergeIdsToRemove((IdRemoval) manualChange, entityLog);
|
||||
} else if (manualChange instanceof ManualResizeRedaction) {
|
||||
mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog);
|
||||
} else if (manualChange instanceof ManualLegalBasisChange) {
|
||||
mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog);
|
||||
} else if (manualChange instanceof ManualRecategorization) {
|
||||
mergeRecategorizations((ManualRecategorization) manualChange, entityLog, dossier);
|
||||
} else if (manualChange instanceof ManualForceRedaction) {
|
||||
mergeForceRedactions((ManualForceRedaction) manualChange, entityLog);
|
||||
}
|
||||
});
|
||||
|
||||
log.info("EntityLog merged successfully!");
|
||||
return entityLog;
|
||||
}
|
||||
|
||||
|
||||
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier) {
|
||||
|
||||
UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream()
|
||||
.filter(manualEntity -> manualEntity.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
|
||||
.findFirst().orElseThrow(() -> new NotFoundException("Entry with annotationId " + manualRedactionEntry.getAnnotationId() + " not found"));
|
||||
List<Change> changes = new ArrayList<>();
|
||||
|
||||
changes.add(Change.builder()
|
||||
.analysisNumber(entityLog.getAnalysisNumber())
|
||||
.dateTime(OffsetDateTime.now())
|
||||
.type(ChangeType.ADDED)
|
||||
.build());
|
||||
|
||||
boolean isHint = isHint(manualRedactionEntry.getType(), dossier);
|
||||
|
||||
if (manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE)) {
|
||||
var matchingEntities = entityLog.getEntityLogEntry().stream()
|
||||
.filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0)))
|
||||
.toList();
|
||||
matchingEntities.forEach(matchingEntity -> {
|
||||
mergeFalsePositive(entityLog, matchingEntity);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
var entityLogEntryBuilder = EntityLogEntry.builder()
|
||||
.id(manualRedactionEntry.getAnnotationId())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis() == null || manualRedactionEntry.getLegalBasis().isEmpty() ? unprocessedManualEntity.getLegalBasis() : manualRedactionEntry.getLegalBasis())
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.entryType(isHint ? EntryType.HINT : EntryType.ENTITY)
|
||||
.state(isHint ? EntryState.SKIPPED : EntryState.APPLIED)
|
||||
.imported(false)
|
||||
.matchedRule("")
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.color(unprocessedManualEntity.getColor())
|
||||
.positions(unprocessedManualEntity.getPositions())
|
||||
.textAfter(unprocessedManualEntity.getTextAfter())
|
||||
.textBefore(unprocessedManualEntity.getTextBefore())
|
||||
.startOffset(unprocessedManualEntity.getStartOffset())
|
||||
.endOffset(unprocessedManualEntity.getEndOffset())
|
||||
.containingNodeId(unprocessedManualEntity.getContainingNodeId())
|
||||
.closestHeadline(unprocessedManualEntity.getClosestHeadline())
|
||||
.imageHasTransparency(false)
|
||||
.dictionaryEntry(manualRedactionEntry.isAddToDictionary())
|
||||
.dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
|
||||
.excluded(false)
|
||||
.changes(changes)
|
||||
.manualChanges(List.of(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.ADD_LOCALLY)
|
||||
.requestedDate(manualRedactionEntry.getRequestDate())
|
||||
.processedDate(null)
|
||||
.userId(manualRedactionEntry.getUser())
|
||||
.propertyChanges(Map.of("value", manualRedactionEntry.getValue()))
|
||||
.build()))
|
||||
.engines(new HashSet<>())
|
||||
.reference(new HashSet<>())
|
||||
.importedRedactionIntersections(new HashSet<>());
|
||||
|
||||
entityLog.getEntityLogEntry().add(entityLogEntryBuilder
|
||||
.build());
|
||||
}
|
||||
|
||||
|
||||
private void mergeFalsePositive(EntityLog entityLog, EntityLogEntry existingEntry) {
|
||||
|
||||
existingEntry.setState(EntryState.REMOVED);
|
||||
List<Change> falsePositiveChanges = new ArrayList<>();
|
||||
falsePositiveChanges.add(Change.builder()
|
||||
.analysisNumber(entityLog.getAnalysisNumber())
|
||||
.dateTime(OffsetDateTime.now())
|
||||
.type(ChangeType.REMOVED)
|
||||
.build());
|
||||
if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) {
|
||||
existingEntry.getChanges().addAll(falsePositiveChanges);
|
||||
} else {
|
||||
existingEntry.setChanges(falsePositiveChanges);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLog entityLog) {
|
||||
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setState(EntryState.IGNORED);
|
||||
addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
|
||||
.requestedDate(idRemoval.getRequestDate())
|
||||
.processedDate(null)
|
||||
.userId(idRemoval.getUser()).build());
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog) {
|
||||
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
var newPosition = manualResizeRedaction.getPositions().get(0);
|
||||
entity.get().setPositions(List.of(new Position(
|
||||
newPosition.getTopLeftX(),
|
||||
newPosition.getTopLeftY(),
|
||||
newPosition.getWidth(),
|
||||
newPosition.getHeight(),
|
||||
newPosition.getPage())));
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.RESIZE)
|
||||
.requestedDate(manualResizeRedaction.getRequestDate())
|
||||
.processedDate(null)
|
||||
.propertyChanges(Map.of("value", manualResizeRedaction.getValue()))
|
||||
.userId(manualResizeRedaction.getUser()).build());
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLog entityLog) {
|
||||
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis());
|
||||
entity.get().setSection(manualLegalBasisChange.getSection());
|
||||
entity.get().setValue(manualLegalBasisChange.getValue());
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
|
||||
.requestedDate(manualLegalBasisChange.getRequestDate())
|
||||
.processedDate(null)
|
||||
.propertyChanges(Map.of("value", manualLegalBasisChange.getValue(),
|
||||
"section", manualLegalBasisChange.getSection(),
|
||||
"legalBasis", manualLegalBasisChange.getLegalBasis()))
|
||||
.userId(manualLegalBasisChange.getUser())
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) {
|
||||
|
||||
boolean isHint = isHint(recategorization.getType(), dossier);
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setType(recategorization.getType());
|
||||
entity.get().setEntryType(getEntryType(isHint, recategorization.getType()));
|
||||
entity.get().setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.requestedDate(recategorization.getRequestDate())
|
||||
.processedDate(recategorization.getProcessedDate())
|
||||
.userId(recategorization.getUser())
|
||||
.propertyChanges(Map.of("type", recategorization.getType()))
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static EntryType getEntryType(boolean isHint, String type) {
|
||||
|
||||
if (type.equals("image") || type.equals("logo") || type.equals("signature") || type.equals("formula")) {
|
||||
return isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE;
|
||||
} else {
|
||||
return isHint ? EntryType.HINT : EntryType.ENTITY;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) {
|
||||
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setLegalBasis(forceRedaction.getLegalBasis());
|
||||
entity.get().setState(EntryState.APPLIED);
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
var forceRedactManualChange = ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
|
||||
.requestedDate(forceRedaction.getRequestDate())
|
||||
.processedDate(forceRedaction.getProcessedDate())
|
||||
.userId(forceRedaction.getUser());
|
||||
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
|
||||
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
|
||||
}
|
||||
entity.get().getManualChanges().add(forceRedactManualChange.build());
|
||||
}
|
||||
}
|
||||
|
||||
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber) {
|
||||
|
||||
if (!changes.isEmpty()) {
|
||||
changes.add(Change.builder()
|
||||
.analysisNumber(analysisNumber + 1)
|
||||
.dateTime(OffsetDateTime.now())
|
||||
.type(changeType)
|
||||
.build());
|
||||
} else {
|
||||
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isHint(String type, DossierEntity dossier) {
|
||||
|
||||
String typeId = toTypeId(type, dossier.getDossierTemplateId());
|
||||
TypeEntity typeEntity = dictionaryPersistenceService.getType(typeId);
|
||||
|
||||
if (typeEntity == null) {
|
||||
var optionalType = dictionaryPersistenceService.getAllTypes(false).stream().filter(typeEntity1 -> typeEntity1.getType().equals(type)).findFirst();
|
||||
if (optionalType.isPresent()) {
|
||||
typeEntity = optionalType.get();
|
||||
} else {
|
||||
throw new NotFoundException("TypeEntity could not be found for typeId: " + typeId);
|
||||
}
|
||||
}
|
||||
|
||||
return typeEntity.isHint();
|
||||
}
|
||||
|
||||
private boolean equalPosition(Rectangle position1, Position position2) {
|
||||
|
||||
return position1.getTopLeftX() == position2.x()
|
||||
&& position1.getTopLeftY() == position2.y()
|
||||
&& position1.getWidth() == position2.w()
|
||||
&& position1.getHeight() == position2.h();
|
||||
}
|
||||
|
||||
private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) {
|
||||
|
||||
return Stream.of(manualRedactions.getEntriesToAdd(),
|
||||
manualRedactions.getForceRedactions(),
|
||||
manualRedactions.getResizeRedactions(),
|
||||
manualRedactions.getRecategorizations(),
|
||||
manualRedactions.getIdsToRemove(),
|
||||
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,45 +1,22 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
@ -60,8 +37,7 @@ public class EntityLogService {
|
||||
UnprocessedManualEntityClient unprocessedManualEntityClient;
|
||||
DossierService dossierService;
|
||||
CommentService commentService;
|
||||
DictionaryPersistenceService dictionaryPersistenceService;
|
||||
DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
EntityLogMergeService entityLogMergeService;
|
||||
|
||||
|
||||
public EntityLog getEntityLog(String dossierId, String fileId) {
|
||||
@ -90,7 +66,7 @@ public class EntityLogService {
|
||||
DossierEntity dossier = dossierService.getDossierById(dossierId);
|
||||
ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = getUnprocessedManualEntities(dossierId, fileId, dossier.getDossierTemplateId(), manualRedactions);
|
||||
mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier, fileStatus);
|
||||
entityLogMergeService.mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier);
|
||||
}
|
||||
|
||||
Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);
|
||||
@ -142,232 +118,4 @@ public class EntityLogService {
|
||||
|
||||
}
|
||||
|
||||
public void mergeEntityLog(ManualRedactions manualRedactions, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) {
|
||||
|
||||
log.info("Merging EntityLog");
|
||||
mergeManualRedactionEntries(manualRedactions.getEntriesToAdd(), unprocessedManualEntities, entityLog, dossier, fileStatus);
|
||||
mergeIdsToRemove(manualRedactions.getIdsToRemove(), entityLog);
|
||||
mergeResizeRedactions(manualRedactions.getResizeRedactions(), entityLog);
|
||||
mergeLegalBasisChanges(manualRedactions.getLegalBasisChanges(), entityLog);
|
||||
mergeRecategorizations(manualRedactions.getRecategorizations(), entityLog, dossier, fileStatus);
|
||||
mergeForceRedactions(manualRedactions.getForceRedactions(), entityLog);
|
||||
log.info("EntityLog merged successfully!");
|
||||
}
|
||||
|
||||
|
||||
private void mergeManualRedactionEntries(Set<ManualRedactionEntry> manualRedactionEntries, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) {
|
||||
|
||||
manualRedactionEntries.forEach(manualRedactionEntry -> {
|
||||
UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream()
|
||||
.filter(manualEntity -> manualEntity.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
|
||||
.findFirst().orElseThrow(() -> new NotFoundException("Entry with annotationId " + manualRedactionEntry.getAnnotationId() + " not found"));
|
||||
List<Change> changes = new ArrayList<>();
|
||||
changes.add(Change.builder()
|
||||
.analysisNumber(entityLog.getAnalysisNumber())
|
||||
.dateTime(OffsetDateTime.now())
|
||||
.type(ChangeType.ADDED)
|
||||
.build());
|
||||
boolean isHint = isHint(manualRedactionEntry.getType(), dossier, fileStatus);
|
||||
entityLog.getEntityLogEntry().add(EntityLogEntry.builder()
|
||||
.id(manualRedactionEntry.getAnnotationId())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis() == null || manualRedactionEntry.getLegalBasis().isEmpty() ? unprocessedManualEntity.getLegalBasis() : manualRedactionEntry.getLegalBasis())
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.entryType(isHint ? EntryType.HINT : EntryType.ENTITY)
|
||||
.state(isHint ? EntryState.SKIPPED : EntryState.APPLIED)
|
||||
.imported(false)
|
||||
.matchedRule("")
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.color(unprocessedManualEntity.getColor())
|
||||
.positions(unprocessedManualEntity.getPositions())
|
||||
.textAfter(unprocessedManualEntity.getTextAfter())
|
||||
.textBefore(unprocessedManualEntity.getTextBefore())
|
||||
.startOffset(unprocessedManualEntity.getStartOffset())
|
||||
.endOffset(unprocessedManualEntity.getEndOffset())
|
||||
.containingNodeId(unprocessedManualEntity.getContainingNodeId())
|
||||
.closestHeadline(unprocessedManualEntity.getClosestHeadline())
|
||||
.imageHasTransparency(false)
|
||||
.dictionaryEntry(manualRedactionEntry.isAddToDictionary())
|
||||
.dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
|
||||
.excluded(false)
|
||||
.changes(changes)
|
||||
.manualChanges(List.of(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.ADD_LOCALLY)
|
||||
.requestedDate(manualRedactionEntry.getRequestDate())
|
||||
.processedDate(null)
|
||||
.userId(manualRedactionEntry.getUser())
|
||||
.propertyChanges(Map.of("value", manualRedactionEntry.getValue()))
|
||||
.build()))
|
||||
.engines(new HashSet<>())
|
||||
.reference(new HashSet<>())
|
||||
.importedRedactionIntersections(new HashSet<>())
|
||||
.build());
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private void mergeIdsToRemove(Set<IdRemoval> idRemovals, EntityLog entityLog) {
|
||||
|
||||
idRemovals.forEach(idRemoval -> {
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setState(EntryState.IGNORED);
|
||||
addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
|
||||
.requestedDate(idRemoval.getRequestDate())
|
||||
.processedDate(null)
|
||||
.userId(idRemoval.getUser()).build());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void mergeResizeRedactions(Set<ManualResizeRedaction> manualResizeRedactions, EntityLog entityLog) {
|
||||
|
||||
manualResizeRedactions.forEach(manualResizeRedaction -> {
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
var newPosition = manualResizeRedaction.getPositions().get(0);
|
||||
entity.get().setPositions(List.of(new Position(
|
||||
newPosition.getTopLeftX(),
|
||||
newPosition.getTopLeftY(),
|
||||
newPosition.getWidth(),
|
||||
newPosition.getHeight(),
|
||||
newPosition.getPage())));
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.RESIZE)
|
||||
.requestedDate(manualResizeRedaction.getRequestDate())
|
||||
.processedDate(null)
|
||||
.propertyChanges(Map.of("value", manualResizeRedaction.getValue()))
|
||||
.userId(manualResizeRedaction.getUser()).build());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void mergeLegalBasisChanges(Set<ManualLegalBasisChange> manualLegalBasisChanges, EntityLog entityLog) {
|
||||
|
||||
manualLegalBasisChanges.forEach(manualLegalBasisChange -> {
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis());
|
||||
entity.get().setSection(manualLegalBasisChange.getSection());
|
||||
entity.get().setValue(manualLegalBasisChange.getValue());
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
|
||||
.requestedDate(manualLegalBasisChange.getRequestDate())
|
||||
.processedDate(null)
|
||||
.propertyChanges(Map.of("value", manualLegalBasisChange.getValue(),
|
||||
"section", manualLegalBasisChange.getSection(),
|
||||
"legalBasis", manualLegalBasisChange.getLegalBasis()))
|
||||
.userId(manualLegalBasisChange.getUser())
|
||||
.build());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void mergeRecategorizations(Set<ManualRecategorization> recategorizations, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) {
|
||||
|
||||
recategorizations.forEach(recategorization -> {
|
||||
boolean isHint = isHint(recategorization.getType(), dossier, fileStatus);
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setType(recategorization.getType());
|
||||
entity.get().setEntryType(getEntryType(isHint, recategorization.getType()));
|
||||
entity.get().setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
entity.get().getManualChanges().add(ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.requestedDate(recategorization.getRequestDate())
|
||||
.processedDate(recategorization.getProcessedDate())
|
||||
.userId(recategorization.getUser())
|
||||
.propertyChanges(Map.of("type", recategorization.getType()))
|
||||
.build());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private static EntryType getEntryType(boolean isHint, String type) {
|
||||
|
||||
if (type.equals("image") || type.equals("logo") || type.equals("signature") || type.equals("formula")) {
|
||||
return isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE;
|
||||
} else {
|
||||
return isHint ? EntryType.HINT : EntryType.ENTITY;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void mergeForceRedactions(Set<ManualForceRedaction> forceRedactions, EntityLog entityLog) {
|
||||
|
||||
forceRedactions.forEach(forceRedaction -> {
|
||||
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
|
||||
if (entity.isPresent()) {
|
||||
entity.get().setLegalBasis(forceRedaction.getLegalBasis());
|
||||
entity.get().setState(EntryState.APPLIED);
|
||||
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
|
||||
var forceRedactManualChange = ManualChange.builder()
|
||||
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
|
||||
.requestedDate(forceRedaction.getRequestDate())
|
||||
.processedDate(forceRedaction.getProcessedDate())
|
||||
.userId(forceRedaction.getUser());
|
||||
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
|
||||
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
|
||||
}
|
||||
entity.get().getManualChanges().add(forceRedactManualChange.build());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber) {
|
||||
|
||||
if (!changes.isEmpty()) {
|
||||
changes.add(Change.builder()
|
||||
.analysisNumber(analysisNumber + 1)
|
||||
.dateTime(OffsetDateTime.now())
|
||||
.type(changeType)
|
||||
.build());
|
||||
} else {
|
||||
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isHint(String type, DossierEntity dossier, FileModel fileStatus) {
|
||||
|
||||
String typeId = toTypeId(type, dossier.getDossierTemplateId());
|
||||
TypeEntity typeEntity = dictionaryPersistenceService.getType(typeId);
|
||||
|
||||
if (typeEntity == null) {
|
||||
var optionalType = dictionaryPersistenceService.getAllTypes(false).stream().filter(typeEntity1 -> typeEntity1.getType().equals(type)).findFirst();
|
||||
if (optionalType.isPresent()) {
|
||||
typeEntity = optionalType.get();
|
||||
} else {
|
||||
throw new NotFoundException("TypeEntity could not be found for typeId: " + typeId);
|
||||
}
|
||||
}
|
||||
|
||||
if (Objects.equals(type, "CBI_address")) {
|
||||
|
||||
String vertebrateStudyId = "";
|
||||
var fileAttributesMap = fileStatus.getFileAttributes();
|
||||
var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId());
|
||||
var fileAttributeConfigs = dossierTemplate.getFileAttributeConfigs();
|
||||
var vertebrateStudy = fileAttributeConfigs.stream().filter(fileAttributeConfigEntity -> fileAttributeConfigEntity.getLabel().equals("Vertebrate Study")).findFirst();
|
||||
|
||||
if (vertebrateStudy.isPresent()) {
|
||||
vertebrateStudyId = vertebrateStudy.get().getId();
|
||||
}
|
||||
|
||||
if (!Objects.equals(vertebrateStudyId, "")
|
||||
&& fileAttributesMap.containsKey(vertebrateStudyId)
|
||||
&& (fileAttributesMap.get(vertebrateStudyId) == null || !fileAttributesMap.get(vertebrateStudyId).equals("Yes"))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return typeEntity.isHint();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ import java.util.UUID;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
import org.testcontainers.shaded.com.google.common.collect.Lists;
|
||||
@ -28,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
@ -49,10 +50,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
public class EntityLogTest {
|
||||
public class EntityLogMergeTest {
|
||||
|
||||
@MockBean
|
||||
private FileStatusService fileStatusService;
|
||||
@ -78,12 +80,12 @@ public class EntityLogTest {
|
||||
@MockBean
|
||||
private DossierTemplatePersistenceService dossierTemplatePersistenceService;
|
||||
|
||||
private EntityLogService entityLogService;
|
||||
private EntityLogMergeService entityLogMergeService;
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() {
|
||||
|
||||
entityLogService = new EntityLogService(fileManagementStorageService, fileStatusService, manualRedactionProviderService, unprocessedManualEntityClient, dossierService, commentService, dictionaryPersistenceService, dossierTemplatePersistenceService);
|
||||
entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService);
|
||||
}
|
||||
|
||||
|
||||
@ -127,7 +129,9 @@ public class EntityLogTest {
|
||||
.build());
|
||||
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
|
||||
|
||||
EntityLog response = entityLogService.getEntityLog(dossierId, fileId, null, true);
|
||||
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, List.of(unprocessedManualEntity), entityLog, DossierEntity.builder()
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.build());
|
||||
|
||||
assertNotNull(response);
|
||||
assertFalse(response.getEntityLogEntry().isEmpty());
|
||||
@ -250,11 +254,13 @@ public class EntityLogTest {
|
||||
.addToDossierDictionary(false)
|
||||
.fileId(fileId)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.dictionaryEntryType(DictionaryEntryType.ENTRY)
|
||||
.type("manual")
|
||||
.build()))
|
||||
.idsToRemove(Set.of(
|
||||
IdRemoval.builder()
|
||||
.annotationId(entryToRemoveId)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.build()
|
||||
))
|
||||
.resizeRedactions(Set.of(
|
||||
@ -263,6 +269,7 @@ public class EntityLogTest {
|
||||
.value("Random")
|
||||
.annotationId(entryToResizeId)
|
||||
.positions(positions)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.build()
|
||||
))
|
||||
.legalBasisChanges(Set.of(
|
||||
@ -271,6 +278,7 @@ public class EntityLogTest {
|
||||
.value("Random")
|
||||
.legalBasis("New legal basis")
|
||||
.section("Section")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.build()
|
||||
))
|
||||
.forceRedactions(Set.of(
|
||||
@ -278,8 +286,10 @@ public class EntityLogTest {
|
||||
.annotationId(forceRedactionId)
|
||||
.fileId(fileId)
|
||||
.legalBasis("Force")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
@ -30,5 +31,6 @@ public class ManualRedactionEntry extends BaseAnnotation {
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
private String sourceId;
|
||||
private DictionaryEntryType dictionaryEntryType;
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user