Merge branch 'RED-8043' into 'master'

RED-8043 - Refactor entity log merge & don't send surrounding analysis request for dictionary adds

Closes RED-8043

See merge request redactmanager/persistence-service!271
This commit is contained in:
Andrei Isvoran 2023-12-19 16:06:39 +01:00
commit 4c9ffd5f64
4 changed files with 118 additions and 117 deletions

View File

@ -61,46 +61,43 @@ public class EntityLogMergeService {
@Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log")
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier, FileModel fileModel) {
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) {
log.info("Merging EntityLog");
List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions);
ManualRedactions manualRedactionsWhichRequireExtraInformation = ManualRedactions.builder()
.entriesToAdd(new HashSet<>())
.resizeRedactions(new HashSet<>())
.build();
List<BaseAnnotation> allManualChangesExceptAdds = allManualChangesExceptAdds(manualRedactions);
manualRedactions.getEntriesToAdd().forEach(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier));
// Sort manual changes by date, so we process them in order of when they were requested
allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
allManualChanges.forEach(manualChange -> {
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
if (manualChange instanceof ManualRedactionEntry) {
mergeManualRedactionEntries((ManualRedactionEntry) manualChange, entityLog, dossier, manualRedactionsWhichRequireExtraInformation);
} else if (manualChange instanceof IdRemoval) {
mergeIdsToRemove((IdRemoval) manualChange, entityLog);
} else if (manualChange instanceof ManualResizeRedaction) {
mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog, manualRedactionsWhichRequireExtraInformation);
} else if (manualChange instanceof ManualLegalBasisChange) {
mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog);
} else if (manualChange instanceof ManualRecategorization) {
mergeRecategorizations((ManualRecategorization) manualChange, entityLog, dossier);
} else if (manualChange instanceof ManualForceRedaction) {
mergeForceRedactions((ManualForceRedaction) manualChange, entityLog);
allManualChangesExceptAdds = allManualChangesExceptAdds.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).collect(Collectors.toList());
final int analysisNumber = entityLog.getAnalysisNumber();
if (!allManualChangesExceptAdds.isEmpty()) {
for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) {
var optionalManualChange = allManualChangesExceptAdds.stream().filter(manualChange -> manualChange.getAnnotationId().equals(entityLogEntry.getId())).findAny();
if (optionalManualChange.isPresent()) {
var manualChange = optionalManualChange.get();
if (manualChange instanceof IdRemoval idRemoval) {
mergeIdsToRemove(idRemoval, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
mergeResizeRedactions(manualResizeRedaction, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
mergeLegalBasisChanges(manualLegalBasisChange, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualRecategorization manualRecategorization) {
mergeRecategorizations(manualRecategorization, entityLogEntry, dossier, analysisNumber);
} else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
mergeForceRedactions(manualForceRedaction, entityLogEntry, analysisNumber);
}
allManualChangesExceptAdds.remove(manualChange);
}
}
});
if (!manualRedactionsWhichRequireExtraInformation.getEntriesToAdd().isEmpty() || !manualRedactionsWhichRequireExtraInformation.getResizeRedactions().isEmpty()) {
sendToAnalyseQueue(fileModel.getId(), dossier, fileModel, manualRedactionsWhichRequireExtraInformation);
}
log.info("EntityLog merged successfully.");
return entityLog;
}
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier, ManualRedactions manualRedactions) {
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) {
if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) {
return;
@ -110,7 +107,7 @@ public class EntityLogMergeService {
changes.add(Change.builder()
.analysisNumber(entityLog.getAnalysisNumber())
.dateTime(OffsetDateTime.now())
.dateTime(manualRedactionEntry.getRequestDate())
.type(ChangeType.ADDED)
.build());
@ -120,9 +117,7 @@ public class EntityLogMergeService {
var matchingEntities = entityLog.getEntityLogEntry().stream()
.filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0)))
.toList();
matchingEntities.forEach(matchingEntity -> {
mergeFalsePositive(entityLog, matchingEntity);
});
matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity));
return;
}
@ -156,8 +151,6 @@ public class EntityLogMergeService {
.reference(new HashSet<>())
.importedRedactionIntersections(new HashSet<>())
.build());
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
}
@ -185,78 +178,62 @@ public class EntityLogMergeService {
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLog entityLog) {
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setState(EntryState.IGNORED);
addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber());
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(null)
.userId(idRemoval.getUser()).build());
}
entityLogEntry.setState(EntryState.IGNORED);
addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(null)
.userId(idRemoval.getUser()).build());
}
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog, ManualRedactions manualRedactions) {
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
EntityLogEntry entityLogEntry = entity.get();
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RESIZE)
.requestedDate(manualResizeRedaction.getRequestDate())
.processedDate(null)
.propertyChanges(Map.of("value", manualResizeRedaction.getValue()))
.userId(manualResizeRedaction.getUser()).build());
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
}
}
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLog entityLog) {
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis());
entity.get().setSection(manualLegalBasisChange.getSection());
entity.get().setValue(manualLegalBasisChange.getValue());
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(Map.of("value", manualLegalBasisChange.getValue(),
"section", manualLegalBasisChange.getSection(),
"legalBasis", manualLegalBasisChange.getLegalBasis()))
.userId(manualLegalBasisChange.getUser())
.build());
}
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(Map.of("value", manualLegalBasisChange.getValue(),
"section", manualLegalBasisChange.getSection(),
"legalBasis", manualLegalBasisChange.getLegalBasis()))
.userId(manualLegalBasisChange.getUser())
.build());
}
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) {
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
boolean isHint = isHint(recategorization.getType(), dossier);
var optionalEntityLogEntry = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
if (optionalEntityLogEntry.isPresent()) {
var entity = optionalEntityLogEntry.get();
entity.setType(recategorization.getType());
entity.setEntryType(getEntryType(isHint, recategorization.getType()));
entity.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entity.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entity.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
.requestedDate(recategorization.getRequestDate())
.processedDate(recategorization.getProcessedDate())
.userId(recategorization.getUser())
.propertyChanges(Map.of("type", recategorization.getType()))
.build());
}
entityLogEntry.setType(recategorization.getType());
entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
.requestedDate(recategorization.getRequestDate())
.processedDate(recategorization.getProcessedDate())
.userId(recategorization.getUser())
.propertyChanges(Map.of("type", recategorization.getType()))
.build());
}
@ -278,23 +255,20 @@ public class EntityLogMergeService {
}
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) {
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setLegalBasis(forceRedaction.getLegalBasis());
entity.get().setState(EntryState.APPLIED);
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entity.get().getManualChanges().add(forceRedactManualChange.build());
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber);
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
}
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber) {
@ -340,17 +314,16 @@ public class EntityLogMergeService {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
}
private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) {
private List<BaseAnnotation> allManualChangesExceptAdds(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getEntriesToAdd(),
manualRedactions.getForceRedactions(),
return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
}
private void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
public void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
var analyseRequest = AnalyzeRequest.builder()

View File

@ -66,7 +66,7 @@ public class EntityLogService {
if (includeUnprocessed) {
DossierEntity dossier = dossierService.getDossierById(dossierId);
ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier, fileStatus);
entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier);
}
if (fileStatus.getExcludedPages() != null && !fileStatus.getExcludedPages().isEmpty()) {

View File

@ -2,19 +2,23 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
@ -35,6 +39,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -59,17 +66,18 @@ public class ManualRedactionService {
ManualRedactionProviderService manualRedactionProviderService;
AnalysisFlagsCalculationService analysisFlagsCalculationService;
EntityLogService entityLogService;
DictionaryManagementService dictionaryManagementService;
HashFunction hashFunction = Hashing.murmur3_128();
ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
EntityLogMergeService entityLogMergeService;
@Transactional
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
Set<ManualRedactionEntryEntity> manualRedactionEntryEntities = new HashSet<>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId);
for (AddRedactionRequest addRedactionRequest : addRedactionRequests) {
manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue());
@ -77,7 +85,7 @@ public class ManualRedactionService {
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
manualRedactionEntryEntities.add(addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest));
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
addRedactionRequest.getValue(),
@ -94,6 +102,17 @@ public class ManualRedactionService {
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
manualRedactionEntryEntities = manualRedactionEntryEntities.stream()
.filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
.collect(Collectors.toSet());
if (!manualRedactionEntryEntities.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(MagicConverter.convert(manualRedactionEntryEntities, ManualRedactionEntry.class))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
reprocess(dossierId, fileId);
@ -233,12 +252,13 @@ public class ManualRedactionService {
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) {
List<ManualAddResponse> response = new ArrayList<>();
Set<ManualResizeRedactionEntity> manualResizeRedactionEntities = new HashSet<>();
EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId);
for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
manualResizeRedactionEntities.add(resizeRedaction);
if (resizeRedactionRequest.getComment() != null) {
Long commentId = commentService.addCommentAndGetId(fileId,
@ -259,6 +279,19 @@ public class ManualRedactionService {
typeIdsOfModifiedDictionaries);
}
manualResizeRedactionEntities = manualResizeRedactionEntities.stream()
.filter(manualRedactionEntry -> !manualRedactionEntry.getUpdateDictionary())
.collect(Collectors.toSet());
if (!manualResizeRedactionEntities.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder()
.resizeRedactions(MagicConverter.convert(manualResizeRedactionEntities, ManualResizeRedaction.class))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
reprocess(dossierId, fileId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);

View File

@ -133,12 +133,7 @@ public class EntityLogMergeTest {
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder()
.dossierTemplateId(dossierTemplateId)
.build(), FileModel.builder().id(fileId).build());
Mockito.verify(rabbitTemplate).convertAndSend(eq(MessagingConfiguration.REDACTION_PRIORITY_QUEUE), captor.capture());
AnalyzeRequest analyzeRequest = captor.getValue();
assertEquals(analyzeRequest.getManualRedactions().getEntriesToAdd().size(), 1);
assertEquals(analyzeRequest.getManualRedactions().getResizeRedactions().size(), 1);
.build());
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());