From 6ba76e66ecec4e782bd3a301b21fb310286aa3e1 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Wed, 20 Dec 2023 13:43:16 +0100 Subject: [PATCH] RED-7784 - Refactor merge logic to not loop multiple times through the entire entity log entries --- .../service/EntityLogMergeService.java | 189 ++++++++++-------- 1 file changed, 108 insertions(+), 81 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index 26ae4580b..7233f0623 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -66,43 +67,40 @@ public class EntityLogMergeService { public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) { log.info("Merging EntityLog"); - List allManualChangesExceptAdds = allManualChangesExceptAdds(manualRedactions); - - manualRedactions.getEntriesToAdd().forEach(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier)); + List allManualChanges = allManualChanges(manualRedactions); + List manualChangesIds = allManualChanges.stream().map(BaseAnnotation::getAnnotationId).toList(); + List matchingEntities = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> manualChangesIds.contains(entityLogEntry.getId())).collect(Collectors.toList()); + final int analysisNumber = entityLog.getAnalysisNumber(); // Sort manual changes by date, so we process them in order of when they were requested - allManualChangesExceptAdds = allManualChangesExceptAdds.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).collect(Collectors.toList()); - final int analysisNumber = entityLog.getAnalysisNumber(); - if (!allManualChangesExceptAdds.isEmpty()) { - for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) { - var optionalManualChange = allManualChangesExceptAdds.stream().filter(manualChange -> manualChange.getAnnotationId().equals(entityLogEntry.getId())).findAny(); - if (optionalManualChange.isPresent()) { - var manualChange = optionalManualChange.get(); - if (manualChange instanceof IdRemoval idRemoval) { - mergeIdsToRemove(idRemoval, entityLogEntry, analysisNumber); - } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) { - mergeResizeRedactions(manualResizeRedaction, entityLogEntry, analysisNumber); - } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) { - mergeLegalBasisChanges(manualLegalBasisChange, entityLogEntry, analysisNumber); - } else if (manualChange instanceof ManualRecategorization manualRecategorization) { - mergeRecategorizations(manualRecategorization, entityLogEntry, dossier, analysisNumber); - } else if (manualChange instanceof ManualForceRedaction manualForceRedaction) { - mergeForceRedactions(manualForceRedaction, entityLogEntry, analysisNumber); - } - allManualChangesExceptAdds.remove(manualChange); - } + allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList(); + allManualChanges.forEach(manualChange -> { + // this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher + if (manualChange instanceof ManualRedactionEntry manualRedactionEntry) { + var entityLogEntry = mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier); + entityLogEntry.ifPresent(matchingEntities::add); + } else if (manualChange instanceof IdRemoval idRemoval) { + mergeIdsToRemove(idRemoval, matchingEntities, analysisNumber); + } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) { + mergeResizeRedactions(manualResizeRedaction, matchingEntities, analysisNumber); + } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) { + mergeLegalBasisChanges(manualLegalBasisChange, matchingEntities, analysisNumber); + } else if (manualChange instanceof ManualRecategorization manualRecategorization) { + mergeRecategorizations(manualRecategorization, matchingEntities, dossier, analysisNumber); + } else if (manualChange instanceof ManualForceRedaction manualForceRedaction) { + mergeForceRedactions(manualForceRedaction, matchingEntities, analysisNumber); } - } + }); log.info("EntityLog merged successfully."); return entityLog; } - private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) { + private Optional mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) { if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) { - return; + return Optional.empty(); } List changes = new ArrayList<>(); @@ -120,10 +118,10 @@ public class EntityLogMergeService { .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0))) .toList(); matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity)); - return; + return Optional.empty(); } - entityLog.getEntityLogEntry().add(EntityLogEntry.builder() + EntityLogEntry entityLogEntry = EntityLogEntry.builder() .id(manualRedactionEntry.getAnnotationId()) .type(manualRedactionEntry.getType()) .value(manualRedactionEntry.getValue()) @@ -152,7 +150,10 @@ public class EntityLogMergeService { .engines(new HashSet<>()) .reference(new HashSet<>()) .importedRedactionIntersections(new HashSet<>()) - .build()); + .build(); + + entityLog.getEntityLogEntry().add(entityLogEntry); + return Optional.of(entityLogEntry); } @@ -180,19 +181,26 @@ public class EntityLogMergeService { - private void mergeIdsToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) { + private void mergeIdsToRemove(IdRemoval idRemoval, List entityLogEntries, int analysisNumber) { - entityLogEntry.setState(EntryState.IGNORED); - addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate()); - entityLogEntry.getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) - .requestedDate(idRemoval.getRequestDate()) - .processedDate(null) - .userId(idRemoval.getUser()).build()); + var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny(); + entity.ifPresent(entityLogEntry -> { + entityLogEntry.setState(EntryState.IGNORED); + addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate()); + entityLogEntry.getManualChanges() + .add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) + .requestedDate(idRemoval.getRequestDate()) + .processedDate(null) + .userId(idRemoval.getUser()) + .build()); + }); } - private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) { + private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, List entityLogEntries, int analysisNumber) { + var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny(); + entity.ifPresent(entityLogEntry -> { entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter()); entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore()); entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions())); @@ -206,14 +214,32 @@ public class EntityLogMergeService { manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue())); } entityLogEntry.getManualChanges().add(manualChange.build()); + }); } - private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) { + private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, List entityLogEntries, int analysisNumber) { + + var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny(); + entity.ifPresent(entityLogEntry -> { + entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + entityLogEntry.setSection(manualLegalBasisChange.getSection()); + entityLogEntry.setValue(manualLegalBasisChange.getValue()); + addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate()); + Map propertyChanges = getPropertyChanges(manualLegalBasisChange); + entityLogEntry.getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) + .requestedDate(manualLegalBasisChange.getRequestDate()) + .processedDate(null) + .propertyChanges(propertyChanges) + .userId(manualLegalBasisChange.getUser()) + .build()); + }); + + } + + + private Map getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) { - entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - entityLogEntry.setSection(manualLegalBasisChange.getSection()); - entityLogEntry.setValue(manualLegalBasisChange.getValue()); - addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate()); Map propertyChanges = new HashMap<>(); if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) { propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis()); @@ -224,31 +250,48 @@ public class EntityLogMergeService { if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) { propertyChanges.put("section", manualLegalBasisChange.getSection()); } - entityLogEntry.getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) - .requestedDate(manualLegalBasisChange.getRequestDate()) - .processedDate(null) - .propertyChanges(propertyChanges) - .userId(manualLegalBasisChange.getUser()) - .build()); + return propertyChanges; } - private void mergeRecategorizations(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) { + + private void mergeRecategorizations(ManualRecategorization recategorization, List entityLogEntries, DossierEntity dossier, int analysisNumber) { boolean isHint = isHint(recategorization.getType(), dossier); - entityLogEntry.setType(recategorization.getType()); - entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType())); - entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); - addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate()); - entityLogEntry.getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.RECATEGORIZE) - .requestedDate(recategorization.getRequestDate()) - .processedDate(recategorization.getProcessedDate()) - .userId(recategorization.getUser()) - .propertyChanges(Map.of("type", recategorization.getType())) - .build()); + var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny(); + entity.ifPresent(entityLogEntry -> { + entityLogEntry.setType(recategorization.getType()); + entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType())); + entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); + addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate()); + entityLogEntry.getManualChanges() + .add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.RECATEGORIZE) + .requestedDate(recategorization.getRequestDate()) + .processedDate(recategorization.getProcessedDate()) + .userId(recategorization.getUser()) + .propertyChanges(Map.of("type", recategorization.getType())) + .build()); + }); } + private void mergeForceRedactions(ManualForceRedaction forceRedaction, List entityLogEntries, int analysisNumber) { + + var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny(); + entity.ifPresent(entityLogEntry -> { + entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis()); + entityLogEntry.setState(EntryState.APPLIED); + addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate()); + var forceRedactManualChange = ManualChange.builder() + .manualRedactionType(ManualRedactionType.FORCE_REDACT) + .requestedDate(forceRedaction.getRequestDate()) + .processedDate(forceRedaction.getProcessedDate()) + .userId(forceRedaction.getUser()); + if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { + forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); + } + entityLogEntry.getManualChanges().add(forceRedactManualChange.build()); + }); + } private EntryType getEntryType(boolean isHint, String type) { @@ -267,23 +310,6 @@ public class EntityLogMergeService { return ManualRedactionType.ADD_LOCALLY; } - - private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) { - - entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis()); - entityLogEntry.setState(EntryState.APPLIED); - addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate()); - var forceRedactManualChange = ManualChange.builder() - .manualRedactionType(ManualRedactionType.FORCE_REDACT) - .requestedDate(forceRedaction.getRequestDate()) - .processedDate(forceRedaction.getProcessedDate()) - .userId(forceRedaction.getUser()); - if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { - forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); - } - entityLogEntry.getManualChanges().add(forceRedactManualChange.build()); - } - private void addChanges(List changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) { if (!changes.isEmpty()) { @@ -327,9 +353,10 @@ public class EntityLogMergeService { return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList()); } - private List allManualChangesExceptAdds(ManualRedactions manualRedactions) { + private List allManualChanges(ManualRedactions manualRedactions) { - return Stream.of(manualRedactions.getForceRedactions(), + return Stream.of(manualRedactions.getEntriesToAdd(), + manualRedactions.getForceRedactions(), manualRedactions.getResizeRedactions(), manualRedactions.getRecategorizations(), manualRedactions.getIdsToRemove(),