Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7784 - Refactor merge logic to not loop multiple times through the entire entity log entries

Browse Source
This commit is contained in:
Andrei Isvoran 2023-12-20 13:43:16 +01:00 committed by Kilian Schüttler
parent 85fc1a570b
commit 6ba76e66ec
1 changed files with 108 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -10,6 +10,7 @@ import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
@ -66,43 +67,40 @@ public class EntityLogMergeService {
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) { public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) {
log.info("Merging EntityLog"); log.info("Merging EntityLog");
List<BaseAnnotation> allManualChangesExceptAdds = allManualChangesExceptAdds(manualRedactions); List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions);
List<String> manualChangesIds = allManualChanges.stream().map(BaseAnnotation::getAnnotationId).toList();
manualRedactions.getEntriesToAdd().forEach(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier)); List<EntityLogEntry> matchingEntities = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> manualChangesIds.contains(entityLogEntry.getId())).collect(Collectors.toList());
final int analysisNumber = entityLog.getAnalysisNumber();
// Sort manual changes by date, so we process them in order of when they were requested // Sort manual changes by date, so we process them in order of when they were requested
allManualChangesExceptAdds = allManualChangesExceptAdds.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).collect(Collectors.toList()); allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
final int analysisNumber = entityLog.getAnalysisNumber(); allManualChanges.forEach(manualChange -> {
if (!allManualChangesExceptAdds.isEmpty()) { // this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) { if (manualChange instanceof ManualRedactionEntry manualRedactionEntry) {
var optionalManualChange = allManualChangesExceptAdds.stream().filter(manualChange -> manualChange.getAnnotationId().equals(entityLogEntry.getId())).findAny(); var entityLogEntry = mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier);
if (optionalManualChange.isPresent()) { entityLogEntry.ifPresent(matchingEntities::add);
var manualChange = optionalManualChange.get(); } else if (manualChange instanceof IdRemoval idRemoval) {
if (manualChange instanceof IdRemoval idRemoval) { mergeIdsToRemove(idRemoval, matchingEntities, analysisNumber);
mergeIdsToRemove(idRemoval, entityLogEntry, analysisNumber); } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
} else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) { mergeResizeRedactions(manualResizeRedaction, matchingEntities, analysisNumber);
mergeResizeRedactions(manualResizeRedaction, entityLogEntry, analysisNumber); } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
} else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) { mergeLegalBasisChanges(manualLegalBasisChange, matchingEntities, analysisNumber);
mergeLegalBasisChanges(manualLegalBasisChange, entityLogEntry, analysisNumber); } else if (manualChange instanceof ManualRecategorization manualRecategorization) {
} else if (manualChange instanceof ManualRecategorization manualRecategorization) { mergeRecategorizations(manualRecategorization, matchingEntities, dossier, analysisNumber);
mergeRecategorizations(manualRecategorization, entityLogEntry, dossier, analysisNumber); } else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
} else if (manualChange instanceof ManualForceRedaction manualForceRedaction) { mergeForceRedactions(manualForceRedaction, matchingEntities, analysisNumber);
mergeForceRedactions(manualForceRedaction, entityLogEntry, analysisNumber);
}
allManualChangesExceptAdds.remove(manualChange);
}
} }
} });
log.info("EntityLog merged successfully."); log.info("EntityLog merged successfully.");
return entityLog; return entityLog;
} }
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) { private Optional<EntityLogEntry> mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) {
if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) { if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) {
return; return Optional.empty();
} }
List<Change> changes = new ArrayList<>(); List<Change> changes = new ArrayList<>();
@ -120,10 +118,10 @@ public class EntityLogMergeService {
.filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0))) .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0)))
.toList(); .toList();
matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity)); matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity));
return; return Optional.empty();
} }
entityLog.getEntityLogEntry().add(EntityLogEntry.builder() EntityLogEntry entityLogEntry = EntityLogEntry.builder()
.id(manualRedactionEntry.getAnnotationId()) .id(manualRedactionEntry.getAnnotationId())
.type(manualRedactionEntry.getType()) .type(manualRedactionEntry.getType())
.value(manualRedactionEntry.getValue()) .value(manualRedactionEntry.getValue())
@ -152,7 +150,10 @@ public class EntityLogMergeService {
.engines(new HashSet<>()) .engines(new HashSet<>())
.reference(new HashSet<>()) .reference(new HashSet<>())
.importedRedactionIntersections(new HashSet<>()) .importedRedactionIntersections(new HashSet<>())
.build()); .build();
entityLog.getEntityLogEntry().add(entityLogEntry);
return Optional.of(entityLogEntry);
} }
@ -180,19 +181,26 @@ public class EntityLogMergeService {
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) { private void mergeIdsToRemove(IdRemoval idRemoval, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
entityLogEntry.setState(EntryState.IGNORED); var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate()); entity.ifPresent(entityLogEntry -> {
entityLogEntry.getManualChanges().add(ManualChange.builder() entityLogEntry.setState(EntryState.IGNORED);
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
.requestedDate(idRemoval.getRequestDate()) entityLogEntry.getManualChanges()
.processedDate(null) .add(ManualChange.builder()
.userId(idRemoval.getUser()).build()); .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(null)
.userId(idRemoval.getUser())
.build());
});
} }
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) { private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter()); entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore()); entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions())); entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
@ -206,14 +214,32 @@ public class EntityLogMergeService {
manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue())); manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
} }
entityLogEntry.getManualChanges().add(manualChange.build()); entityLogEntry.getManualChanges().add(manualChange.build());
});
} }
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) { private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
Map<String, String> propertyChanges = getPropertyChanges(manualLegalBasisChange);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
});
}
private Map<String, String> getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) {
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
Map<String, String> propertyChanges = new HashMap<>(); Map<String, String> propertyChanges = new HashMap<>();
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) { if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis()); propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
@ -224,31 +250,48 @@ public class EntityLogMergeService {
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) { if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection()); propertyChanges.put("section", manualLegalBasisChange.getSection());
} }
entityLogEntry.getManualChanges().add(ManualChange.builder() return propertyChanges;
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
} }
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
private void mergeRecategorizations(ManualRecategorization recategorization, List<EntityLogEntry> entityLogEntries, DossierEntity dossier, int analysisNumber) {
boolean isHint = isHint(recategorization.getType(), dossier); boolean isHint = isHint(recategorization.getType(), dossier);
entityLogEntry.setType(recategorization.getType()); var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType())); entity.ifPresent(entityLogEntry -> {
entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); entityLogEntry.setType(recategorization.getType());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate()); entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
entityLogEntry.getManualChanges().add(ManualChange.builder() entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
.manualRedactionType(ManualRedactionType.RECATEGORIZE) addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate());
.requestedDate(recategorization.getRequestDate()) entityLogEntry.getManualChanges()
.processedDate(recategorization.getProcessedDate()) .add(ManualChange.builder()
.userId(recategorization.getUser()) .manualRedactionType(ManualRedactionType.RECATEGORIZE)
.propertyChanges(Map.of("type", recategorization.getType())) .requestedDate(recategorization.getRequestDate())
.build()); .processedDate(recategorization.getProcessedDate())
.userId(recategorization.getUser())
.propertyChanges(Map.of("type", recategorization.getType()))
.build());
});
} }
private void mergeForceRedactions(ManualForceRedaction forceRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
});
}
private EntryType getEntryType(boolean isHint, String type) { private EntryType getEntryType(boolean isHint, String type) {
@ -267,23 +310,6 @@ public class EntityLogMergeService {
return ManualRedactionType.ADD_LOCALLY; return ManualRedactionType.ADD_LOCALLY;
} }
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
}
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) { private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) { if (!changes.isEmpty()) {
@ -327,9 +353,10 @@ public class EntityLogMergeService {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList()); return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
} }
private List<BaseAnnotation> allManualChangesExceptAdds(ManualRedactions manualRedactions) { private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getForceRedactions(), return Stream.of(manualRedactions.getEntriesToAdd(),
manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(), manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(), manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(), manualRedactions.getIdsToRemove(),